diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..244f685 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +.SyncNode.config +/__pycache__ diff --git a/ServerSync.py b/ServerSync.py new file mode 100755 index 0000000..b684ed2 --- /dev/null +++ b/ServerSync.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + + +import sys +import os +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib')) + +import argparse + +parser = argparse.ArgumentParser(usage="A Quick Tool to Sync Files to Local Servers over FTP", description="Idk , i had ADHD and i was too ored to look up a tool for this job , so yea") + +parser.add_argument("Action", help="The action you wanna perform , do Setup to setup") +parser.add_argument("-l","--log", type=int,help="Set the logging mode") +arguments = parser.parse_args() + +match arguments.Action: + + case "Setup": + + print("Setup Mode") + import setup + setup.initialSetup() + + case "init": + import initSync + print("initialising new SyncNode") + initSync.begin() + case "sync": + import sync + print("Syncing...") + sync.handle_upload() + + + + diff --git a/display.py b/display.py new file mode 100644 index 0000000..062d7a6 --- /dev/null +++ b/display.py @@ -0,0 +1,7 @@ +import os + + +def clear(): + os.subprocess('cls' if os.name == 'nt' else 'clear') + + diff --git a/initSync.py b/initSync.py new file mode 100644 index 0000000..bf57175 --- /dev/null +++ b/initSync.py @@ -0,0 +1,109 @@ +from typing import NamedTuple +import lib.questionary as questionary +from pathlib import Path +import re +import json +from lib.questionary.prompts import text +class NodeResult(NamedTuple): + found: bool + path: str | None + matches: list | None + + +def save_config(data, filename=".SyncNode.config"): + with open(filename, "w") as f: + json.dump(data, f, indent=4) + print(f"Config saved to {filename}") +class menu: + @staticmethod + def run_setup_wizard(): + + answers = questionary.form( + protocol=questionary.select( + "Select a protocol:", + choices=["FTP"], + ), + + ip=questionary.text( + "Enter Server IP:", validate=lambda text: True if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", text) else "Enter a Vaid IP" + ), + port=questionary.text( + "Enter Server Port:", + ), + user=questionary.text( + "Enter Server user:", + ), + password=questionary.text( + "Enter user password:", + ), + + directory=questionary.text("Enter Server Directory Target") + + + + + + ).ask() + + save_config(answers) + + print("Writing to .SyncNode.config") + + + + def main(): + print("Checking for existing SyncNode Config") + result = checkForNode(".") + match result: + case NodeResult(found=True): + if result.matches and len(result.matches) > 1: + print("Warning , more than 1 instances of SyncNode configs were found!") + + for path in result.matches: + print("Config @",path) + print(" ") + print("Fix configs or run ServerSync -fix-config ") + exit() + + else: + print("Found a existing SyncNode Configuration!") + print("Run ServerSync config") + exit() + + + case NodeResult(found=False): + print("Clean Dir, Moving on...") + menu.run_setup_wizard() + + + + + + + + + + + +def checkForNode(path): + search = list(Path(path).rglob(".SyncNode.config")) + if search: + # print(NodeResult(True, path, search)) + + return NodeResult(True, path, search) + else: + + # print(NodeResult(False, None, None)) + + + return NodeResult(False, None, None) + + + + + +def begin(): + menu.main() + + + diff --git a/lib/__pycache__/ftp.cpython-314.pyc b/lib/__pycache__/ftp.cpython-314.pyc new file mode 100644 index 0000000..5b7cb89 Binary files /dev/null and b/lib/__pycache__/ftp.cpython-314.pyc differ diff --git a/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so b/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so new file mode 100755 index 0000000..259a8c8 Binary files /dev/null and b/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so differ diff --git a/lib/bcrypt-5.0.0.dist-info/INSTALLER b/lib/bcrypt-5.0.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/bcrypt-5.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/bcrypt-5.0.0.dist-info/METADATA b/lib/bcrypt-5.0.0.dist-info/METADATA new file mode 100644 index 0000000..629a992 --- /dev/null +++ b/lib/bcrypt-5.0.0.dist-info/METADATA @@ -0,0 +1,343 @@ +Metadata-Version: 2.4 +Name: bcrypt +Version: 5.0.0 +Summary: Modern password hashing for your software and your servers +Author-email: The Python Cryptographic Authority developers +License: Apache-2.0 +Project-URL: homepage, https://github.com/pyca/bcrypt/ +Classifier: Development Status :: 5 - Production/Stable +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +License-File: LICENSE +Provides-Extra: tests +Requires-Dist: pytest!=3.3.0,>=3.2.1; extra == "tests" +Provides-Extra: typecheck +Requires-Dist: mypy; extra == "typecheck" +Dynamic: license-file + +bcrypt +====== + +.. image:: https://img.shields.io/pypi/v/bcrypt.svg + :target: https://pypi.org/project/bcrypt/ + :alt: Latest Version + +.. image:: https://github.com/pyca/bcrypt/workflows/CI/badge.svg?branch=main + :target: https://github.com/pyca/bcrypt/actions?query=workflow%3ACI+branch%3Amain + +Acceptable password hashing for your software and your servers (but you should +really use argon2id or scrypt) + + +Installation +============ + +To install bcrypt, simply: + +.. code:: console + + $ pip install bcrypt + +Note that bcrypt should build very easily on Linux provided you have a C +compiler and a Rust compiler (the minimum supported Rust version is 1.56.0). + +For Debian and Ubuntu, the following command will ensure that the required dependencies are installed: + +.. code:: console + + $ sudo apt-get install build-essential cargo + +For Fedora and RHEL-derivatives, the following command will ensure that the required dependencies are installed: + +.. code:: console + + $ sudo yum install gcc cargo + +For Alpine, the following command will ensure that the required dependencies are installed: + +.. code:: console + + $ apk add --update musl-dev gcc cargo + + +Alternatives +============ + +While bcrypt remains an acceptable choice for password storage, depending on your specific use case you may also want to consider using scrypt (either via `standard library`_ or `cryptography`_) or argon2id via `argon2_cffi`_. + +Changelog +========= + +5.0.0 +----- + +* Bumped MSRV to 1.74. +* Added support for Python 3.14 and free-threaded Python 3.14. +* Added support for Windows on ARM. +* Passing ``hashpw`` a password longer than 72 bytes now raises a + ``ValueError``. Previously the password was silently truncated, following the + behavior of the original OpenBSD ``bcrypt`` implementation. + +4.3.0 +----- + +* Dropped support for Python 3.7. +* We now support free-threaded Python 3.13. +* We now support PyPy 3.11. +* We now publish wheels for free-threaded Python 3.13, for PyPy 3.11 on + ``manylinux``, and for ARMv7l on ``manylinux``. + +4.2.1 +----- + +* Bump Rust dependency versions - this should resolve crashes on Python 3.13 + free-threaded builds. +* We no longer build ``manylinux`` wheels for PyPy 3.9. + +4.2.0 +----- + +* Bump Rust dependency versions +* Removed the ``BCRYPT_ALLOW_RUST_163`` environment variable. + +4.1.3 +----- + +* Bump Rust dependency versions + +4.1.2 +----- + +* Publish both ``py37`` and ``py39`` wheels. This should resolve some errors + relating to initializing a module multiple times per process. + +4.1.1 +----- + +* Fixed the type signature on the ``kdf`` method. +* Fixed packaging bug on Windows. +* Fixed incompatibility with passlib package detection assumptions. + +4.1.0 +----- + +* Dropped support for Python 3.6. +* Bumped MSRV to 1.64. (Note: Rust 1.63 can be used by setting the ``BCRYPT_ALLOW_RUST_163`` environment variable) + +4.0.1 +----- + +* We now build PyPy ``manylinux`` wheels. +* Fixed a bug where passing an invalid ``salt`` to ``checkpw`` could result in + a ``pyo3_runtime.PanicException``. It now correctly raises a ``ValueError``. + +4.0.0 +----- + +* ``bcrypt`` is now implemented in Rust. Users building from source will need + to have a Rust compiler available. Nothing will change for users downloading + wheels. +* We no longer ship ``manylinux2010`` wheels. Users should upgrade to the latest + ``pip`` to ensure this doesn’t cause issues downloading wheels on their + platform. We now ship ``manylinux_2_28`` wheels for users on new enough platforms. +* ``NUL`` bytes are now allowed in inputs. + + +3.2.2 +----- + +* Fixed packaging of ``py.typed`` files in wheels so that ``mypy`` works. + +3.2.1 +----- + +* Added support for compilation on z/OS +* The next release of ``bcrypt`` with be 4.0 and it will require Rust at + compile time, for users building from source. There will be no additional + requirement for users who are installing from wheels. Users on most + platforms will be able to obtain a wheel by making sure they have an up to + date ``pip``. The minimum supported Rust version will be 1.56.0. +* This will be the final release for which we ship ``manylinux2010`` wheels. + Going forward the minimum supported manylinux ABI for our wheels will be + ``manylinux2014``. The vast majority of users will continue to receive + ``manylinux`` wheels provided they have an up to date ``pip``. + + +3.2.0 +----- + +* Added typehints for library functions. +* Dropped support for Python versions less than 3.6 (2.7, 3.4, 3.5). +* Shipped ``abi3`` Windows wheels (requires pip >= 20). + +3.1.7 +----- + +* Set a ``setuptools`` lower bound for PEP517 wheel building. +* We no longer distribute 32-bit ``manylinux1`` wheels. Continuing to produce + them was a maintenance burden. + +3.1.6 +----- + +* Added support for compilation on Haiku. + +3.1.5 +----- + +* Added support for compilation on AIX. +* Dropped Python 2.6 and 3.3 support. +* Switched to using ``abi3`` wheels for Python 3. If you are not getting a + wheel on a compatible platform please upgrade your ``pip`` version. + +3.1.4 +----- + +* Fixed compilation with mingw and on illumos. + +3.1.3 +----- +* Fixed a compilation issue on Solaris. +* Added a warning when using too few rounds with ``kdf``. + +3.1.2 +----- +* Fixed a compile issue affecting big endian platforms. +* Fixed invalid escape sequence warnings on Python 3.6. +* Fixed building in non-UTF8 environments on Python 2. + +3.1.1 +----- +* Resolved a ``UserWarning`` when used with ``cffi`` 1.8.3. + +3.1.0 +----- +* Added support for ``checkpw``, a convenience method for verifying a password. +* Ensure that you get a ``$2y$`` hash when you input a ``$2y$`` salt. +* Fixed a regression where ``$2a`` hashes were vulnerable to a wraparound bug. +* Fixed compilation under Alpine Linux. + +3.0.0 +----- +* Switched the C backend to code obtained from the OpenBSD project rather than + openwall. +* Added support for ``bcrypt_pbkdf`` via the ``kdf`` function. + +2.0.0 +----- +* Added support for an adjustible prefix when calling ``gensalt``. +* Switched to CFFI 1.0+ + +Usage +----- + +Password Hashing +~~~~~~~~~~~~~~~~ + +Hashing and then later checking that a password matches the previous hashed +password is very simple: + +.. code:: pycon + + >>> import bcrypt + >>> password = b"super secret password" + >>> # Hash a password for the first time, with a randomly-generated salt + >>> hashed = bcrypt.hashpw(password, bcrypt.gensalt()) + >>> # Check that an unhashed password matches one that has previously been + >>> # hashed + >>> if bcrypt.checkpw(password, hashed): + ... print("It Matches!") + ... else: + ... print("It Does not Match :(") + +KDF +~~~ + +As of 3.0.0 ``bcrypt`` now offers a ``kdf`` function which does ``bcrypt_pbkdf``. +This KDF is used in OpenSSH's newer encrypted private key format. + +.. code:: pycon + + >>> import bcrypt + >>> key = bcrypt.kdf( + ... password=b'password', + ... salt=b'salt', + ... desired_key_bytes=32, + ... rounds=100) + + +Adjustable Work Factor +~~~~~~~~~~~~~~~~~~~~~~ +One of bcrypt's features is an adjustable logarithmic work factor. To adjust +the work factor merely pass the desired number of rounds to +``bcrypt.gensalt(rounds=12)`` which defaults to 12): + +.. code:: pycon + + >>> import bcrypt + >>> password = b"super secret password" + >>> # Hash a password for the first time, with a certain number of rounds + >>> hashed = bcrypt.hashpw(password, bcrypt.gensalt(14)) + >>> # Check that a unhashed password matches one that has previously been + >>> # hashed + >>> if bcrypt.checkpw(password, hashed): + ... print("It Matches!") + ... else: + ... print("It Does not Match :(") + + +Adjustable Prefix +~~~~~~~~~~~~~~~~~ + +Another one of bcrypt's features is an adjustable prefix to let you define what +libraries you'll remain compatible with. To adjust this, pass either ``2a`` or +``2b`` (the default) to ``bcrypt.gensalt(prefix=b"2b")`` as a bytes object. + +As of 3.0.0 the ``$2y$`` prefix is still supported in ``hashpw`` but deprecated. + +Maximum Password Length +~~~~~~~~~~~~~~~~~~~~~~~ + +The bcrypt algorithm only handles passwords up to 72 characters, any characters +beyond that are ignored. To work around this, a common approach is to hash a +password with a cryptographic hash (such as ``sha256``) and then base64 +encode it to prevent NULL byte problems before hashing the result with +``bcrypt``: + +.. code:: pycon + + >>> password = b"an incredibly long password" * 10 + >>> hashed = bcrypt.hashpw( + ... base64.b64encode(hashlib.sha256(password).digest()), + ... bcrypt.gensalt() + ... ) + +Compatibility +------------- + +This library should be compatible with py-bcrypt and it will run on Python +3.8+ (including free-threaded builds), and PyPy 3. + +Security +-------- + +``bcrypt`` follows the `same security policy as cryptography`_, if you +identify a vulnerability, we ask you to contact us privately. + +.. _`same security policy as cryptography`: https://cryptography.io/en/latest/security.html +.. _`standard library`: https://docs.python.org/3/library/hashlib.html#hashlib.scrypt +.. _`argon2_cffi`: https://argon2-cffi.readthedocs.io +.. _`cryptography`: https://cryptography.io/en/latest/hazmat/primitives/key-derivation-functions/#cryptography.hazmat.primitives.kdf.scrypt.Scrypt diff --git a/lib/bcrypt-5.0.0.dist-info/RECORD b/lib/bcrypt-5.0.0.dist-info/RECORD new file mode 100644 index 0000000..04009c6 --- /dev/null +++ b/lib/bcrypt-5.0.0.dist-info/RECORD @@ -0,0 +1,11 @@ +bcrypt-5.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +bcrypt-5.0.0.dist-info/METADATA,sha256=yV1BfLlI6udlVy23eNbzDa62DSEbUrlWvlLBCI6UAdI,10524 +bcrypt-5.0.0.dist-info/RECORD,, +bcrypt-5.0.0.dist-info/WHEEL,sha256=WieEZvWpc0Erab6-NfTu9412g-GcE58js6gvBn3Q7B4,111 +bcrypt-5.0.0.dist-info/licenses/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850 +bcrypt-5.0.0.dist-info/top_level.txt,sha256=BkR_qBzDbSuycMzHWE1vzXrfYecAzUVmQs6G2CukqNI,7 +bcrypt/__init__.py,sha256=cv-NupIX6P7o6A4PK_F0ur6IZoDr3GnvyzFO9k16wKQ,1000 +bcrypt/__init__.pyi,sha256=ITUCB9mPVU8sKUbJQMDUH5YfQXZb1O55F9qvKZR_o8I,333 +bcrypt/__pycache__/__init__.cpython-314.pyc,, +bcrypt/_bcrypt.abi3.so,sha256=oFwJu4Gq44FqJDttx_oWpypfuUQ30BkCWzD2FhojdYw,631768 +bcrypt/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/lib/bcrypt-5.0.0.dist-info/WHEEL b/lib/bcrypt-5.0.0.dist-info/WHEEL new file mode 100644 index 0000000..eb203c1 --- /dev/null +++ b/lib/bcrypt-5.0.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: false +Tag: cp39-abi3-manylinux_2_34_x86_64 + diff --git a/lib/bcrypt-5.0.0.dist-info/licenses/LICENSE b/lib/bcrypt-5.0.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000..11069ed --- /dev/null +++ b/lib/bcrypt-5.0.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/lib/bcrypt-5.0.0.dist-info/top_level.txt b/lib/bcrypt-5.0.0.dist-info/top_level.txt new file mode 100644 index 0000000..7f0b6e7 --- /dev/null +++ b/lib/bcrypt-5.0.0.dist-info/top_level.txt @@ -0,0 +1 @@ +bcrypt diff --git a/lib/bcrypt/__init__.py b/lib/bcrypt/__init__.py new file mode 100644 index 0000000..81a92fd --- /dev/null +++ b/lib/bcrypt/__init__.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._bcrypt import ( + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + checkpw, + gensalt, + hashpw, + kdf, +) +from ._bcrypt import ( + __version_ex__ as __version__, +) + +__all__ = [ + "__author__", + "__copyright__", + "__email__", + "__license__", + "__summary__", + "__title__", + "__uri__", + "__version__", + "checkpw", + "gensalt", + "hashpw", + "kdf", +] diff --git a/lib/bcrypt/__init__.pyi b/lib/bcrypt/__init__.pyi new file mode 100644 index 0000000..12e4a2e --- /dev/null +++ b/lib/bcrypt/__init__.pyi @@ -0,0 +1,10 @@ +def gensalt(rounds: int = 12, prefix: bytes = b"2b") -> bytes: ... +def hashpw(password: bytes, salt: bytes) -> bytes: ... +def checkpw(password: bytes, hashed_password: bytes) -> bool: ... +def kdf( + password: bytes, + salt: bytes, + desired_key_bytes: int, + rounds: int, + ignore_few_rounds: bool = False, +) -> bytes: ... diff --git a/lib/bcrypt/__pycache__/__init__.cpython-314.pyc b/lib/bcrypt/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..c111cd2 Binary files /dev/null and b/lib/bcrypt/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/bcrypt/_bcrypt.abi3.so b/lib/bcrypt/_bcrypt.abi3.so new file mode 100755 index 0000000..4806fec Binary files /dev/null and b/lib/bcrypt/_bcrypt.abi3.so differ diff --git a/lib/bcrypt/py.typed b/lib/bcrypt/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lib/bin/markdown-it b/lib/bin/markdown-it new file mode 100755 index 0000000..0b55dc9 --- /dev/null +++ b/lib/bin/markdown-it @@ -0,0 +1,7 @@ +#!/usr/bin/python +import sys +from markdown_it.cli.parse import main +if __name__ == '__main__': + if sys.argv[0].endswith('.exe'): + sys.argv[0] = sys.argv[0][:-4] + sys.exit(main()) diff --git a/lib/bin/pygmentize b/lib/bin/pygmentize new file mode 100755 index 0000000..2dd30f1 --- /dev/null +++ b/lib/bin/pygmentize @@ -0,0 +1,7 @@ +#!/usr/bin/python +import sys +from pygments.cmdline import main +if __name__ == '__main__': + if sys.argv[0].endswith('.exe'): + sys.argv[0] = sys.argv[0][:-4] + sys.exit(main()) diff --git a/lib/cffi-2.0.0.dist-info/INSTALLER b/lib/cffi-2.0.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/cffi-2.0.0.dist-info/METADATA b/lib/cffi-2.0.0.dist-info/METADATA new file mode 100644 index 0000000..67508e5 --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/METADATA @@ -0,0 +1,68 @@ +Metadata-Version: 2.4 +Name: cffi +Version: 2.0.0 +Summary: Foreign Function Interface for Python calling C code. +Author: Armin Rigo, Maciej Fijalkowski +Maintainer: Matt Davis, Matt Clay, Matti Picus +License-Expression: MIT +Project-URL: Documentation, https://cffi.readthedocs.io/ +Project-URL: Changelog, https://cffi.readthedocs.io/en/latest/whatsnew.html +Project-URL: Downloads, https://github.com/python-cffi/cffi/releases +Project-URL: Contact, https://groups.google.com/forum/#!forum/python-cffi +Project-URL: Source Code, https://github.com/python-cffi/cffi +Project-URL: Issue Tracker, https://github.com/python-cffi/cffi/issues +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta +Classifier: Programming Language :: Python :: Implementation :: CPython +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-File: LICENSE +License-File: AUTHORS +Requires-Dist: pycparser; implementation_name != "PyPy" +Dynamic: license-file + +[![GitHub Actions Status](https://github.com/python-cffi/cffi/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/python-cffi/cffi/actions/workflows/ci.yaml?query=branch%3Amain++) +[![PyPI version](https://img.shields.io/pypi/v/cffi.svg)](https://pypi.org/project/cffi) +[![Read the Docs](https://img.shields.io/badge/docs-latest-blue.svg)][Documentation] + + +CFFI +==== + +Foreign Function Interface for Python calling C code. + +Please see the [Documentation] or uncompiled in the `doc/` subdirectory. + +Download +-------- + +[Download page](https://github.com/python-cffi/cffi/releases) + +Source Code +----------- + +Source code is publicly available on +[GitHub](https://github.com/python-cffi/cffi). + +Contact +------- + +[Mailing list](https://groups.google.com/forum/#!forum/python-cffi) + +Testing/development tips +------------------------ + +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: + + pip install pytest + pip install -e . # editable install of CFFI for local development + pytest src/c/ testing/ + +[Documentation]: http://cffi.readthedocs.org/ diff --git a/lib/cffi-2.0.0.dist-info/RECORD b/lib/cffi-2.0.0.dist-info/RECORD new file mode 100644 index 0000000..0bff34c --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/RECORD @@ -0,0 +1,49 @@ +_cffi_backend.cpython-314-x86_64-linux-gnu.so,sha256=IC779uqgG6Lc-DPt98a_lEnFmcODSBaaWDhkt5YCnYc,344664 +cffi-2.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +cffi-2.0.0.dist-info/METADATA,sha256=uYzn40F68Im8EtXHNBLZs7FoPM-OxzyYbDWsjJvhujk,2559 +cffi-2.0.0.dist-info/RECORD,, +cffi-2.0.0.dist-info/WHEEL,sha256=89M_z4WU4NdsrUE_fycOqZb9IBH4u1jBOWPYOzOuUsU,151 +cffi-2.0.0.dist-info/entry_points.txt,sha256=y6jTxnyeuLnL-XJcDv8uML3n6wyYiGRg8MTp_QGJ9Ho,75 +cffi-2.0.0.dist-info/licenses/AUTHORS,sha256=KmemC7-zN1nWfWRf8TG45ta8TK_CMtdR_Kw-2k0xTMg,208 +cffi-2.0.0.dist-info/licenses/LICENSE,sha256=W6JN3FcGf5JJrdZEw6_EGl1tw34jQz73Wdld83Cwr2M,1123 +cffi-2.0.0.dist-info/top_level.txt,sha256=rE7WR3rZfNKxWI9-jn6hsHCAl7MDkB-FmuQbxWjFehQ,19 +cffi/__init__.py,sha256=-ksBQ7MfDzVvbBlV_ftYBWAmEqfA86ljIzMxzaZeAlI,511 +cffi/__pycache__/__init__.cpython-314.pyc,, +cffi/__pycache__/_imp_emulation.cpython-314.pyc,, +cffi/__pycache__/_shimmed_dist_utils.cpython-314.pyc,, +cffi/__pycache__/api.cpython-314.pyc,, +cffi/__pycache__/backend_ctypes.cpython-314.pyc,, +cffi/__pycache__/cffi_opcode.cpython-314.pyc,, +cffi/__pycache__/commontypes.cpython-314.pyc,, +cffi/__pycache__/cparser.cpython-314.pyc,, +cffi/__pycache__/error.cpython-314.pyc,, +cffi/__pycache__/ffiplatform.cpython-314.pyc,, +cffi/__pycache__/lock.cpython-314.pyc,, +cffi/__pycache__/model.cpython-314.pyc,, +cffi/__pycache__/pkgconfig.cpython-314.pyc,, +cffi/__pycache__/recompiler.cpython-314.pyc,, +cffi/__pycache__/setuptools_ext.cpython-314.pyc,, +cffi/__pycache__/vengine_cpy.cpython-314.pyc,, +cffi/__pycache__/vengine_gen.cpython-314.pyc,, +cffi/__pycache__/verifier.cpython-314.pyc,, +cffi/_cffi_errors.h,sha256=zQXt7uR_m8gUW-fI2hJg0KoSkJFwXv8RGUkEDZ177dQ,3908 +cffi/_cffi_include.h,sha256=Exhmgm9qzHWzWivjfTe0D7Xp4rPUkVxdNuwGhMTMzbw,15055 +cffi/_embedding.h,sha256=Ai33FHblE7XSpHOCp8kPcWwN5_9BV14OvN0JVa6ITpw,18786 +cffi/_imp_emulation.py,sha256=RxREG8zAbI2RPGBww90u_5fi8sWdahpdipOoPzkp7C0,2960 +cffi/_shimmed_dist_utils.py,sha256=Bjj2wm8yZbvFvWEx5AEfmqaqZyZFhYfoyLLQHkXZuao,2230 +cffi/api.py,sha256=alBv6hZQkjpmZplBphdaRn2lPO9-CORs_M7ixabvZWI,42169 +cffi/backend_ctypes.py,sha256=h5ZIzLc6BFVXnGyc9xPqZWUS7qGy7yFSDqXe68Sa8z4,42454 +cffi/cffi_opcode.py,sha256=JDV5l0R0_OadBX_uE7xPPTYtMdmpp8I9UYd6av7aiDU,5731 +cffi/commontypes.py,sha256=7N6zPtCFlvxXMWhHV08psUjdYIK2XgsN3yo5dgua_v4,2805 +cffi/cparser.py,sha256=QUTfmlL-aO-MYR8bFGlvAUHc36OQr7XYLe0WLkGFjRo,44790 +cffi/error.py,sha256=v6xTiS4U0kvDcy4h_BDRo5v39ZQuj-IMRYLv5ETddZs,877 +cffi/ffiplatform.py,sha256=avxFjdikYGJoEtmJO7ewVmwG_VEVl6EZ_WaNhZYCqv4,3584 +cffi/lock.py,sha256=l9TTdwMIMpi6jDkJGnQgE9cvTIR7CAntIJr8EGHt3pY,747 +cffi/model.py,sha256=W30UFQZE73jL5Mx5N81YT77us2W2iJjTm0XYfnwz1cg,21797 +cffi/parse_c_type.h,sha256=OdwQfwM9ktq6vlCB43exFQmxDBtj2MBNdK8LYl15tjw,5976 +cffi/pkgconfig.py,sha256=LP1w7vmWvmKwyqLaU1Z243FOWGNQMrgMUZrvgFuOlco,4374 +cffi/recompiler.py,sha256=78J6lMEEOygXNmjN9-fOFFO3j7eW-iFxSrxfvQb54bY,65509 +cffi/setuptools_ext.py,sha256=0rCwBJ1W7FHWtiMKfNXsSST88V8UXrui5oeXFlDNLG8,9411 +cffi/vengine_cpy.py,sha256=oyQKD23kpE0aChUKA8Jg0e723foPiYzLYEdb-J0MiNs,43881 +cffi/vengine_gen.py,sha256=DUlEIrDiVin1Pnhn1sfoamnS5NLqfJcOdhRoeSNeJRg,26939 +cffi/verifier.py,sha256=oX8jpaohg2Qm3aHcznidAdvrVm5N4sQYG0a3Eo5mIl4,11182 diff --git a/lib/cffi-2.0.0.dist-info/WHEEL b/lib/cffi-2.0.0.dist-info/WHEEL new file mode 100644 index 0000000..8c7218a --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: false +Tag: cp314-cp314-manylinux_2_17_x86_64 +Tag: cp314-cp314-manylinux2014_x86_64 + diff --git a/lib/cffi-2.0.0.dist-info/entry_points.txt b/lib/cffi-2.0.0.dist-info/entry_points.txt new file mode 100644 index 0000000..4b0274f --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[distutils.setup_keywords] +cffi_modules = cffi.setuptools_ext:cffi_modules diff --git a/lib/cffi-2.0.0.dist-info/licenses/AUTHORS b/lib/cffi-2.0.0.dist-info/licenses/AUTHORS new file mode 100644 index 0000000..370a25d --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/licenses/AUTHORS @@ -0,0 +1,8 @@ +This package has been mostly done by Armin Rigo with help from +Maciej Fijałkowski. The idea is heavily based (although not directly +copied) from LuaJIT ffi by Mike Pall. + + +Other contributors: + + Google Inc. diff --git a/lib/cffi-2.0.0.dist-info/licenses/LICENSE b/lib/cffi-2.0.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000..0a1dbfb --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/licenses/LICENSE @@ -0,0 +1,23 @@ + +Except when otherwise stated (look for LICENSE files in directories or +information at the beginning of each file) all software and +documentation is licensed as follows: + + MIT No Attribution + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + diff --git a/lib/cffi-2.0.0.dist-info/top_level.txt b/lib/cffi-2.0.0.dist-info/top_level.txt new file mode 100644 index 0000000..f645779 --- /dev/null +++ b/lib/cffi-2.0.0.dist-info/top_level.txt @@ -0,0 +1,2 @@ +_cffi_backend +cffi diff --git a/lib/cffi/__init__.py b/lib/cffi/__init__.py new file mode 100644 index 0000000..c99ec3d --- /dev/null +++ b/lib/cffi/__init__.py @@ -0,0 +1,14 @@ +__all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', + 'FFIError'] + +from .api import FFI +from .error import CDefError, FFIError, VerificationError, VerificationMissing +from .error import PkgConfigError + +__version__ = "2.0.0" +__version_info__ = (2, 0, 0) + +# The verifier module file names are based on the CRC32 of a string that +# contains the following version number. It may be older than __version__ +# if nothing is clearly incompatible. +__version_verifier_modules__ = "0.8.6" diff --git a/lib/cffi/__pycache__/__init__.cpython-314.pyc b/lib/cffi/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..f4b37b8 Binary files /dev/null and b/lib/cffi/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/_imp_emulation.cpython-314.pyc b/lib/cffi/__pycache__/_imp_emulation.cpython-314.pyc new file mode 100644 index 0000000..f763cd6 Binary files /dev/null and b/lib/cffi/__pycache__/_imp_emulation.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/_shimmed_dist_utils.cpython-314.pyc b/lib/cffi/__pycache__/_shimmed_dist_utils.cpython-314.pyc new file mode 100644 index 0000000..9b3c330 Binary files /dev/null and b/lib/cffi/__pycache__/_shimmed_dist_utils.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/api.cpython-314.pyc b/lib/cffi/__pycache__/api.cpython-314.pyc new file mode 100644 index 0000000..ae7f215 Binary files /dev/null and b/lib/cffi/__pycache__/api.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/backend_ctypes.cpython-314.pyc b/lib/cffi/__pycache__/backend_ctypes.cpython-314.pyc new file mode 100644 index 0000000..f0edd5e Binary files /dev/null and b/lib/cffi/__pycache__/backend_ctypes.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/cffi_opcode.cpython-314.pyc b/lib/cffi/__pycache__/cffi_opcode.cpython-314.pyc new file mode 100644 index 0000000..22c6219 Binary files /dev/null and b/lib/cffi/__pycache__/cffi_opcode.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/commontypes.cpython-314.pyc b/lib/cffi/__pycache__/commontypes.cpython-314.pyc new file mode 100644 index 0000000..a57db0c Binary files /dev/null and b/lib/cffi/__pycache__/commontypes.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/cparser.cpython-314.pyc b/lib/cffi/__pycache__/cparser.cpython-314.pyc new file mode 100644 index 0000000..af42abc Binary files /dev/null and b/lib/cffi/__pycache__/cparser.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/error.cpython-314.pyc b/lib/cffi/__pycache__/error.cpython-314.pyc new file mode 100644 index 0000000..54771a3 Binary files /dev/null and b/lib/cffi/__pycache__/error.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/ffiplatform.cpython-314.pyc b/lib/cffi/__pycache__/ffiplatform.cpython-314.pyc new file mode 100644 index 0000000..3566c41 Binary files /dev/null and b/lib/cffi/__pycache__/ffiplatform.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/lock.cpython-314.pyc b/lib/cffi/__pycache__/lock.cpython-314.pyc new file mode 100644 index 0000000..4df73e4 Binary files /dev/null and b/lib/cffi/__pycache__/lock.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/model.cpython-314.pyc b/lib/cffi/__pycache__/model.cpython-314.pyc new file mode 100644 index 0000000..50f9fc2 Binary files /dev/null and b/lib/cffi/__pycache__/model.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/pkgconfig.cpython-314.pyc b/lib/cffi/__pycache__/pkgconfig.cpython-314.pyc new file mode 100644 index 0000000..60a29a7 Binary files /dev/null and b/lib/cffi/__pycache__/pkgconfig.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/recompiler.cpython-314.pyc b/lib/cffi/__pycache__/recompiler.cpython-314.pyc new file mode 100644 index 0000000..83e160b Binary files /dev/null and b/lib/cffi/__pycache__/recompiler.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/setuptools_ext.cpython-314.pyc b/lib/cffi/__pycache__/setuptools_ext.cpython-314.pyc new file mode 100644 index 0000000..191ce66 Binary files /dev/null and b/lib/cffi/__pycache__/setuptools_ext.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/vengine_cpy.cpython-314.pyc b/lib/cffi/__pycache__/vengine_cpy.cpython-314.pyc new file mode 100644 index 0000000..dd61e60 Binary files /dev/null and b/lib/cffi/__pycache__/vengine_cpy.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/vengine_gen.cpython-314.pyc b/lib/cffi/__pycache__/vengine_gen.cpython-314.pyc new file mode 100644 index 0000000..5d8eb97 Binary files /dev/null and b/lib/cffi/__pycache__/vengine_gen.cpython-314.pyc differ diff --git a/lib/cffi/__pycache__/verifier.cpython-314.pyc b/lib/cffi/__pycache__/verifier.cpython-314.pyc new file mode 100644 index 0000000..90c94f7 Binary files /dev/null and b/lib/cffi/__pycache__/verifier.cpython-314.pyc differ diff --git a/lib/cffi/_cffi_errors.h b/lib/cffi/_cffi_errors.h new file mode 100644 index 0000000..158e059 --- /dev/null +++ b/lib/cffi/_cffi_errors.h @@ -0,0 +1,149 @@ +#ifndef CFFI_MESSAGEBOX +# ifdef _MSC_VER +# define CFFI_MESSAGEBOX 1 +# else +# define CFFI_MESSAGEBOX 0 +# endif +#endif + + +#if CFFI_MESSAGEBOX +/* Windows only: logic to take the Python-CFFI embedding logic + initialization errors and display them in a background thread + with MessageBox. The idea is that if the whole program closes + as a result of this problem, then likely it is already a console + program and you can read the stderr output in the console too. + If it is not a console program, then it will likely show its own + dialog to complain, or generally not abruptly close, and for this + case the background thread should stay alive. +*/ +static void *volatile _cffi_bootstrap_text; + +static PyObject *_cffi_start_error_capture(void) +{ + PyObject *result = NULL; + PyObject *x, *m, *bi; + + if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text, + (void *)1, NULL) != NULL) + return (PyObject *)1; + + m = PyImport_AddModule("_cffi_error_capture"); + if (m == NULL) + goto error; + + result = PyModule_GetDict(m); + if (result == NULL) + goto error; + +#if PY_MAJOR_VERSION >= 3 + bi = PyImport_ImportModule("builtins"); +#else + bi = PyImport_ImportModule("__builtin__"); +#endif + if (bi == NULL) + goto error; + PyDict_SetItemString(result, "__builtins__", bi); + Py_DECREF(bi); + + x = PyRun_String( + "import sys\n" + "class FileLike:\n" + " def write(self, x):\n" + " try:\n" + " of.write(x)\n" + " except: pass\n" + " self.buf += x\n" + " def flush(self):\n" + " pass\n" + "fl = FileLike()\n" + "fl.buf = ''\n" + "of = sys.stderr\n" + "sys.stderr = fl\n" + "def done():\n" + " sys.stderr = of\n" + " return fl.buf\n", /* make sure the returned value stays alive */ + Py_file_input, + result, result); + Py_XDECREF(x); + + error: + if (PyErr_Occurred()) + { + PyErr_WriteUnraisable(Py_None); + PyErr_Clear(); + } + return result; +} + +#pragma comment(lib, "user32.lib") + +static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored) +{ + Sleep(666); /* may be interrupted if the whole process is closing */ +#if PY_MAJOR_VERSION >= 3 + MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text, + L"Python-CFFI error", + MB_OK | MB_ICONERROR); +#else + MessageBoxA(NULL, (char *)_cffi_bootstrap_text, + "Python-CFFI error", + MB_OK | MB_ICONERROR); +#endif + _cffi_bootstrap_text = NULL; + return 0; +} + +static void _cffi_stop_error_capture(PyObject *ecap) +{ + PyObject *s; + void *text; + + if (ecap == (PyObject *)1) + return; + + if (ecap == NULL) + goto error; + + s = PyRun_String("done()", Py_eval_input, ecap, ecap); + if (s == NULL) + goto error; + + /* Show a dialog box, but in a background thread, and + never show multiple dialog boxes at once. */ +#if PY_MAJOR_VERSION >= 3 + text = PyUnicode_AsWideCharString(s, NULL); +#else + text = PyString_AsString(s); +#endif + + _cffi_bootstrap_text = text; + + if (text != NULL) + { + HANDLE h; + h = CreateThread(NULL, 0, _cffi_bootstrap_dialog, + NULL, 0, NULL); + if (h != NULL) + CloseHandle(h); + } + /* decref the string, but it should stay alive as 'fl.buf' + in the small module above. It will really be freed only if + we later get another similar error. So it's a leak of at + most one copy of the small module. That's fine for this + situation which is usually a "fatal error" anyway. */ + Py_DECREF(s); + PyErr_Clear(); + return; + + error: + _cffi_bootstrap_text = NULL; + PyErr_Clear(); +} + +#else + +static PyObject *_cffi_start_error_capture(void) { return NULL; } +static void _cffi_stop_error_capture(PyObject *ecap) { } + +#endif diff --git a/lib/cffi/_cffi_include.h b/lib/cffi/_cffi_include.h new file mode 100644 index 0000000..908a1d7 --- /dev/null +++ b/lib/cffi/_cffi_include.h @@ -0,0 +1,389 @@ +#define _CFFI_ + +/* We try to define Py_LIMITED_API before including Python.h. + + Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and + Py_REF_DEBUG are not defined. This is a best-effort approximation: + we can learn about Py_DEBUG from pyconfig.h, but it is unclear if + the same works for the other two macros. Py_DEBUG implies them, + but not the other way around. + + The implementation is messy (issue #350): on Windows, with _MSC_VER, + we have to define Py_LIMITED_API even before including pyconfig.h. + In that case, we guess what pyconfig.h will do to the macros above, + and check our guess after the #include. + + Note that on Windows, with CPython 3.x, you need >= 3.5 and virtualenv + version >= 16.0.0. With older versions of either, you don't get a + copy of PYTHON3.DLL in the virtualenv. We can't check the version of + CPython *before* we even include pyconfig.h. ffi.set_source() puts + a ``#define _CFFI_NO_LIMITED_API'' at the start of this file if it is + running on Windows < 3.5, as an attempt at fixing it, but that's + arguably wrong because it may not be the target version of Python. + Still better than nothing I guess. As another workaround, you can + remove the definition of Py_LIMITED_API here. + + See also 'py_limited_api' in cffi/setuptools_ext.py. +*/ +#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) +# define Py_LIMITED_API +# endif +# endif +#endif + +#include +#ifdef __cplusplus +extern "C" { +#endif +#include +#include "parse_c_type.h" + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +# define _cffi_float_complex_t _Fcomplex /* include for it */ +# define _cffi_double_complex_t _Dcomplex /* include for it */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +# define _cffi_float_complex_t float _Complex +# define _cffi_double_complex_t double _Complex +#endif + +#ifdef __GNUC__ +# define _CFFI_UNUSED_FN __attribute__((unused)) +#else +# define _CFFI_UNUSED_FN /* nothing */ +#endif + +#ifdef __cplusplus +# ifndef _Bool + typedef bool _Bool; /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + + +#if PY_MAJOR_VERSION >= 3 +# define PyInt_FromLong PyLong_FromLong +#endif + +#define _cffi_from_c_double PyFloat_FromDouble +#define _cffi_from_c_float PyFloat_FromDouble +#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_ulong PyLong_FromUnsignedLong +#define _cffi_from_c_longlong PyLong_FromLongLong +#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong +#define _cffi_from_c__Bool PyBool_FromLong + +#define _cffi_to_c_double PyFloat_AsDouble +#define _cffi_to_c_float PyFloat_AsDouble + +#define _cffi_from_c_int(x, type) \ + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) + +#define _cffi_to_c_int(o, type) \ + ((type)( \ + sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ + (Py_FatalError("unsupported size for type " #type), (type)0))) + +#define _cffi_to_c_i8 \ + ((int(*)(PyObject *))_cffi_exports[1]) +#define _cffi_to_c_u8 \ + ((int(*)(PyObject *))_cffi_exports[2]) +#define _cffi_to_c_i16 \ + ((int(*)(PyObject *))_cffi_exports[3]) +#define _cffi_to_c_u16 \ + ((int(*)(PyObject *))_cffi_exports[4]) +#define _cffi_to_c_i32 \ + ((int(*)(PyObject *))_cffi_exports[5]) +#define _cffi_to_c_u32 \ + ((unsigned int(*)(PyObject *))_cffi_exports[6]) +#define _cffi_to_c_i64 \ + ((long long(*)(PyObject *))_cffi_exports[7]) +#define _cffi_to_c_u64 \ + ((unsigned long long(*)(PyObject *))_cffi_exports[8]) +#define _cffi_to_c_char \ + ((int(*)(PyObject *))_cffi_exports[9]) +#define _cffi_from_c_pointer \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10]) +#define _cffi_to_c_pointer \ + ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11]) +#define _cffi_get_struct_layout \ + not used any more +#define _cffi_restore_errno \ + ((void(*)(void))_cffi_exports[13]) +#define _cffi_save_errno \ + ((void(*)(void))_cffi_exports[14]) +#define _cffi_from_c_char \ + ((PyObject *(*)(char))_cffi_exports[15]) +#define _cffi_from_c_deref \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16]) +#define _cffi_to_c \ + ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17]) +#define _cffi_from_c_struct \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18]) +#define _cffi_to_c_wchar_t \ + ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19]) +#define _cffi_from_c_wchar_t \ + ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20]) +#define _cffi_to_c_long_double \ + ((long double(*)(PyObject *))_cffi_exports[21]) +#define _cffi_to_c__Bool \ + ((_Bool(*)(PyObject *))_cffi_exports[22]) +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(struct _cffi_ctypedescr *, \ + PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24]) +#define _CFFI_CPIDX 25 +#define _cffi_call_python \ + ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX]) +#define _cffi_to_c_wchar3216_t \ + ((int(*)(PyObject *))_cffi_exports[26]) +#define _cffi_from_c_wchar3216_t \ + ((PyObject *(*)(int))_cffi_exports[27]) +#define _CFFI_NUM_EXPORTS 28 + +struct _cffi_ctypedescr; + +static void *_cffi_exports[_CFFI_NUM_EXPORTS]; + +#define _cffi_type(index) ( \ + assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \ + (struct _cffi_ctypedescr *)_cffi_types[index]) + +static PyObject *_cffi_init(const char *module_name, Py_ssize_t version, + const struct _cffi_type_context_s *ctx) +{ + PyObject *module, *o_arg, *new_module; + void *raw[] = { + (void *)module_name, + (void *)version, + (void *)_cffi_exports, + (void *)ctx, + }; + + module = PyImport_ImportModule("_cffi_backend"); + if (module == NULL) + goto failure; + + o_arg = PyLong_FromVoidPtr((void *)raw); + if (o_arg == NULL) + goto failure; + + new_module = PyObject_CallMethod( + module, (char *)"_init_cffi_1_0_external_module", (char *)"O", o_arg); + + Py_DECREF(o_arg); + Py_DECREF(module); + return new_module; + + failure: + Py_XDECREF(module); + return NULL; +} + + +#ifdef HAVE_WCHAR_H +typedef wchar_t _cffi_wchar_t; +#else +typedef uint16_t _cffi_wchar_t; /* same random pick as _cffi_backend.c */ +#endif + +_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o) +{ + if (sizeof(_cffi_wchar_t) == 2) + return (uint16_t)_cffi_to_c_wchar_t(o); + else + return (uint16_t)_cffi_to_c_wchar3216_t(o); +} + +_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x) +{ + if (sizeof(_cffi_wchar_t) == 2) + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); + else + return _cffi_from_c_wchar3216_t((int)x); +} + +_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o) +{ + if (sizeof(_cffi_wchar_t) == 4) + return (int)_cffi_to_c_wchar_t(o); + else + return (int)_cffi_to_c_wchar3216_t(o); +} + +_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x) +{ + if (sizeof(_cffi_wchar_t) == 4) + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); + else + return _cffi_from_c_wchar3216_t((int)x); +} + +union _cffi_union_alignment_u { + unsigned char m_char; + unsigned short m_short; + unsigned int m_int; + unsigned long m_long; + unsigned long long m_longlong; + float m_float; + double m_double; + long double m_longdouble; +}; + +struct _cffi_freeme_s { + struct _cffi_freeme_s *next; + union _cffi_union_alignment_u alignment; +}; + +_CFFI_UNUSED_FN static int +_cffi_convert_array_argument(struct _cffi_ctypedescr *ctptr, PyObject *arg, + char **output_data, Py_ssize_t datasize, + struct _cffi_freeme_s **freeme) +{ + char *p; + if (datasize < 0) + return -1; + + p = *output_data; + if (p == NULL) { + struct _cffi_freeme_s *fp = (struct _cffi_freeme_s *)PyObject_Malloc( + offsetof(struct _cffi_freeme_s, alignment) + (size_t)datasize); + if (fp == NULL) + return -1; + fp->next = *freeme; + *freeme = fp; + p = *output_data = (char *)&fp->alignment; + } + memset((void *)p, 0, (size_t)datasize); + return _cffi_convert_array_from_object(p, ctptr, arg); +} + +_CFFI_UNUSED_FN static void +_cffi_free_array_arguments(struct _cffi_freeme_s *freeme) +{ + do { + void *p = (void *)freeme; + freeme = freeme->next; + PyObject_Free(p); + } while (freeme != NULL); +} + +/********** end CPython-specific section **********/ +#else +_CFFI_UNUSED_FN +static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *); +# define _cffi_call_python _cffi_call_python_org +#endif + + +#define _cffi_array_len(array) (sizeof(array) / sizeof((array)[0])) + +#define _cffi_prim_int(size, sign) \ + ((size) == 1 ? ((sign) ? _CFFI_PRIM_INT8 : _CFFI_PRIM_UINT8) : \ + (size) == 2 ? ((sign) ? _CFFI_PRIM_INT16 : _CFFI_PRIM_UINT16) : \ + (size) == 4 ? ((sign) ? _CFFI_PRIM_INT32 : _CFFI_PRIM_UINT32) : \ + (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ + _CFFI__UNKNOWN_PRIM) + +#define _cffi_prim_float(size) \ + ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT : \ + (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE : \ + (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE : \ + _CFFI__UNKNOWN_FLOAT_PRIM) + +#define _cffi_check_int(got, got_nonpos, expected) \ + ((got_nonpos) == (expected <= 0) && \ + (got) == (unsigned long long)expected) + +#ifdef MS_WIN32 +# define _cffi_stdcall __stdcall +#else +# define _cffi_stdcall /* nothing */ +#endif + +#ifdef __cplusplus +} +#endif diff --git a/lib/cffi/_embedding.h b/lib/cffi/_embedding.h new file mode 100644 index 0000000..64c04f6 --- /dev/null +++ b/lib/cffi/_embedding.h @@ -0,0 +1,550 @@ + +/***** Support code for embedding *****/ + +#ifdef __cplusplus +extern "C" { +#endif + + +#if defined(_WIN32) +# define CFFI_DLLEXPORT __declspec(dllexport) +#elif defined(__GNUC__) +# define CFFI_DLLEXPORT __attribute__((visibility("default"))) +#else +# define CFFI_DLLEXPORT /* nothing */ +#endif + + +/* There are two global variables of type _cffi_call_python_fnptr: + + * _cffi_call_python, which we declare just below, is the one called + by ``extern "Python"`` implementations. + + * _cffi_call_python_org, which on CPython is actually part of the + _cffi_exports[] array, is the function pointer copied from + _cffi_backend. If _cffi_start_python() fails, then this is set + to NULL; otherwise, it should never be NULL. + + After initialization is complete, both are equal. However, the + first one remains equal to &_cffi_start_and_call_python until the + very end of initialization, when we are (or should be) sure that + concurrent threads also see a completely initialized world, and + only then is it changed. +*/ +#undef _cffi_call_python +typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *); +static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *); +static _cffi_call_python_fnptr _cffi_call_python = &_cffi_start_and_call_python; + + +#ifndef _MSC_VER + /* --- Assuming a GCC not infinitely old --- */ +# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n) +# define cffi_write_barrier() __sync_synchronize() +# if !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386) +# define cffi_read_barrier() __sync_synchronize() +# else +# define cffi_read_barrier() (void)0 +# endif +#else + /* --- Windows threads version --- */ +# include +# define cffi_compare_and_swap(l,o,n) \ + (InterlockedCompareExchangePointer(l,n,o) == (o)) +# define cffi_write_barrier() InterlockedCompareExchange(&_cffi_dummy,0,0) +# define cffi_read_barrier() (void)0 +static volatile LONG _cffi_dummy; +#endif + +#ifdef WITH_THREAD +# ifndef _MSC_VER +# include + static pthread_mutex_t _cffi_embed_startup_lock; +# else + static CRITICAL_SECTION _cffi_embed_startup_lock; +# endif + static char _cffi_embed_startup_lock_ready = 0; +#endif + +static void _cffi_acquire_reentrant_mutex(void) +{ + static void *volatile lock = NULL; + + while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) { + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: pthread_mutex_init() should be very fast, and + this is only run at start-up anyway. */ + } + +#ifdef WITH_THREAD + if (!_cffi_embed_startup_lock_ready) { +# ifndef _MSC_VER + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_cffi_embed_startup_lock, &attr); +# else + InitializeCriticalSection(&_cffi_embed_startup_lock); +# endif + _cffi_embed_startup_lock_ready = 1; + } +#endif + + while (!cffi_compare_and_swap(&lock, (void *)1, NULL)) + ; + +#ifndef _MSC_VER + pthread_mutex_lock(&_cffi_embed_startup_lock); +#else + EnterCriticalSection(&_cffi_embed_startup_lock); +#endif +} + +static void _cffi_release_reentrant_mutex(void) +{ +#ifndef _MSC_VER + pthread_mutex_unlock(&_cffi_embed_startup_lock); +#else + LeaveCriticalSection(&_cffi_embed_startup_lock); +#endif +} + + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + +#include "_cffi_errors.h" + + +#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX] + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */ + +static void _cffi_py_initialize(void) +{ + /* XXX use initsigs=0, which "skips initialization registration of + signal handlers, which might be useful when Python is + embedded" according to the Python docs. But review and think + if it should be a user-controllable setting. + + XXX we should also give a way to write errors to a buffer + instead of to stderr. + + XXX if importing 'site' fails, CPython (any version) calls + exit(). Should we try to work around this behavior here? + */ + Py_InitializeEx(0); +} + +static int _cffi_initialize_python(void) +{ + /* This initializes Python, imports _cffi_backend, and then the + present .dll/.so is set up as a CPython C extension module. + */ + int result; + PyGILState_STATE state; + PyObject *pycode=NULL, *global_dict=NULL, *x; + PyObject *builtins; + + state = PyGILState_Ensure(); + + /* Call the initxxx() function from the present module. It will + create and initialize us as a CPython extension module, instead + of letting the startup Python code do it---it might reimport + the same .dll/.so and get maybe confused on some platforms. + It might also have troubles locating the .dll/.so again for all + I know. + */ + (void)_CFFI_PYTHON_STARTUP_FUNC(); + if (PyErr_Occurred()) + goto error; + + /* Now run the Python code provided to ffi.embedding_init_code(). + */ + pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE, + "", + Py_file_input); + if (pycode == NULL) + goto error; + global_dict = PyDict_New(); + if (global_dict == NULL) + goto error; + builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) + goto error; + x = PyEval_EvalCode( +#if PY_MAJOR_VERSION < 3 + (PyCodeObject *) +#endif + pycode, global_dict, global_dict); + if (x == NULL) + goto error; + Py_DECREF(x); + + /* Done! Now if we've been called from + _cffi_start_and_call_python() in an ``extern "Python"``, we can + only hope that the Python code did correctly set up the + corresponding @ffi.def_extern() function. Otherwise, the + general logic of ``extern "Python"`` functions (inside the + _cffi_backend module) will find that the reference is still + missing and print an error. + */ + result = 0; + done: + Py_XDECREF(pycode); + Py_XDECREF(global_dict); + PyGILState_Release(state); + return result; + + error:; + { + /* Print as much information as potentially useful. + Debugging load-time failures with embedding is not fun + */ + PyObject *ecap; + PyObject *exception, *v, *tb, *f, *modules, *mod; + PyErr_Fetch(&exception, &v, &tb); + ecap = _cffi_start_error_capture(); + f = PySys_GetObject((char *)"stderr"); + if (f != NULL && f != Py_None) { + PyFile_WriteString( + "Failed to initialize the Python-CFFI embedding logic:\n\n", f); + } + + if (exception != NULL) { + PyErr_NormalizeException(&exception, &v, &tb); + PyErr_Display(exception, v, tb); + } + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + if (f != NULL && f != Py_None) { + PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME + "\ncompiled with cffi version: 2.0.0" + "\n_cffi_backend module: ", f); + modules = PyImport_GetModuleDict(); + mod = PyDict_GetItemString(modules, "_cffi_backend"); + if (mod == NULL) { + PyFile_WriteString("not loaded", f); + } + else { + v = PyObject_GetAttrString(mod, "__file__"); + PyFile_WriteObject(v, f, 0); + Py_XDECREF(v); + } + PyFile_WriteString("\nsys.path: ", f); + PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0); + PyFile_WriteString("\n\n", f); + } + _cffi_stop_error_capture(ecap); + } + result = -1; + goto done; +} + +#if PY_VERSION_HEX < 0x03080000 +PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ +#endif + +static int _cffi_carefully_make_gil(void) +{ + /* This does the basic initialization of Python. It can be called + completely concurrently from unrelated threads. It assumes + that we don't hold the GIL before (if it exists), and we don't + hold it afterwards. + + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) + + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. + So we use a global variable as a simple spin lock. This global + variable must be from 'libpythonX.Y.so', not from this + cffi-based extension module, because it must be shared from + different cffi-based extension modules. + + In Python < 3.8, we choose + _PyParser_TokenNames[0] as a completely arbitrary pointer value + that is never written to. The default is to point to the + string "ENDMARKER". We change it temporarily to point to the + next character in that string. (Yes, I know it's REALLY + obscure.) + + In Python >= 3.8, this string array is no longer writable, so + instead we pick PyCapsuleType.tp_version_tag. We can't change + Python < 3.8 because someone might use a mixture of cffi + embedded modules, some of which were compiled before this file + changed. + + In Python >= 3.12, this stopped working because that particular + tp_version_tag gets modified during interpreter startup. It's + arguably a bad idea before 3.12 too, but again we can't change + that because someone might use a mixture of cffi embedded + modules, and no-one reported a bug so far. In Python >= 3.12 + we go instead for PyCapsuleType.tp_as_buffer, which is supposed + to always be NULL. We write to it temporarily a pointer to + a struct full of NULLs, which is semantically the same. + */ + +#ifdef WITH_THREAD +# if PY_VERSION_HEX < 0x03080000 + char *volatile *lock = (char *volatile *)_PyParser_TokenNames; + char *old_value, *locked_value; + + while (1) { /* spin loop */ + old_value = *lock; + locked_value = old_value + 1; + if (old_value[0] == 'E') { + assert(old_value[1] == 'N'); + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { + assert(old_value[0] == 'N'); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +# else +# if PY_VERSION_HEX < 0x030C0000 + int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; + int old_value, locked_value = -42; + assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG)); +# else + static struct ebp_s { PyBufferProcs buf; int mark; } empty_buffer_procs; + empty_buffer_procs.mark = -42; + PyBufferProcs *volatile *lock = (PyBufferProcs *volatile *) + &PyCapsule_Type.tp_as_buffer; + PyBufferProcs *old_value, *locked_value = &empty_buffer_procs.buf; +# endif + + while (1) { /* spin loop */ + old_value = *lock; + if (old_value == 0) { + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { +# if PY_VERSION_HEX < 0x030C0000 + assert(old_value == locked_value); +# else + /* The pointer should point to a possibly different + empty_buffer_procs from another C extension module */ + assert(((struct ebp_s *)old_value)->mark == -42); +# endif + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +# endif +#endif + + /* call Py_InitializeEx() */ + if (!Py_IsInitialized()) { + _cffi_py_initialize(); +#if PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +#endif + PyEval_SaveThread(); /* release the GIL */ + /* the returned tstate must be the one that has been stored into the + autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */ + } + else { +#if PY_VERSION_HEX < 0x03070000 + /* PyEval_InitThreads() is always a no-op from CPython 3.7 */ + PyGILState_STATE state = PyGILState_Ensure(); + PyEval_InitThreads(); + PyGILState_Release(state); +#endif + } + +#ifdef WITH_THREAD + /* release the lock */ + while (!cffi_compare_and_swap(lock, locked_value, old_value)) + ; +#endif + + return 0; +} + +/********** end CPython-specific section **********/ + + +#else + + +/********** PyPy-specific section **********/ + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ + +static struct _cffi_pypy_init_s { + const char *name; + void *func; /* function pointer */ + const char *code; +} _cffi_pypy_init = { + _CFFI_MODULE_NAME, + _CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_CODE, +}; + +extern int pypy_carefully_make_gil(const char *); +extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *); + +static int _cffi_carefully_make_gil(void) +{ + return pypy_carefully_make_gil(_CFFI_MODULE_NAME); +} + +static int _cffi_initialize_python(void) +{ + return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init); +} + +/********** end PyPy-specific section **********/ + + +#endif + + +#ifdef __GNUC__ +__attribute__((noinline)) +#endif +static _cffi_call_python_fnptr _cffi_start_python(void) +{ + /* Delicate logic to initialize Python. This function can be + called multiple times concurrently, e.g. when the process calls + its first ``extern "Python"`` functions in multiple threads at + once. It can also be called recursively, in which case we must + ignore it. We also have to consider what occurs if several + different cffi-based extensions reach this code in parallel + threads---it is a different copy of the code, then, and we + can't have any shared global variable unless it comes from + 'libpythonX.Y.so'. + + Idea: + + * _cffi_carefully_make_gil(): "carefully" call + PyEval_InitThreads() (possibly with Py_InitializeEx() first). + + * then we use a (local) custom lock to make sure that a call to this + cffi-based extension will wait if another call to the *same* + extension is running the initialization in another thread. + It is reentrant, so that a recursive call will not block, but + only one from a different thread. + + * then we grab the GIL and (Python 2) we call Py_InitializeEx(). + At this point, concurrent calls to Py_InitializeEx() are not + possible: we have the GIL. + + * do the rest of the specific initialization, which may + temporarily release the GIL but not the custom lock. + Only release the custom lock when we are done. + */ + static char called = 0; + + if (_cffi_carefully_make_gil() != 0) + return NULL; + + _cffi_acquire_reentrant_mutex(); + + /* Here the GIL exists, but we don't have it. We're only protected + from concurrency by the reentrant mutex. */ + + /* This file only initializes the embedded module once, the first + time this is called, even if there are subinterpreters. */ + if (!called) { + called = 1; /* invoke _cffi_initialize_python() only once, + but don't set '_cffi_call_python' right now, + otherwise concurrent threads won't call + this function at all (we need them to wait) */ + if (_cffi_initialize_python() == 0) { + /* now initialization is finished. Switch to the fast-path. */ + + /* We would like nobody to see the new value of + '_cffi_call_python' without also seeing the rest of the + data initialized. However, this is not possible. But + the new value of '_cffi_call_python' is the function + 'cffi_call_python()' from _cffi_backend. So: */ + cffi_write_barrier(); + /* ^^^ we put a write barrier here, and a corresponding + read barrier at the start of cffi_call_python(). This + ensures that after that read barrier, we see everything + done here before the write barrier. + */ + + assert(_cffi_call_python_org != NULL); + _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org; + } + else { + /* initialization failed. Reset this to NULL, even if it was + already set to some other value. Future calls to + _cffi_start_python() are still forced to occur, and will + always return NULL from now on. */ + _cffi_call_python_org = NULL; + } + } + + _cffi_release_reentrant_mutex(); + + return (_cffi_call_python_fnptr)_cffi_call_python_org; +} + +static +void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args) +{ + _cffi_call_python_fnptr fnptr; + int current_err = errno; +#ifdef _MSC_VER + int current_lasterr = GetLastError(); +#endif + fnptr = _cffi_start_python(); + if (fnptr == NULL) { + fprintf(stderr, "function %s() called, but initialization code " + "failed. Returning 0.\n", externpy->name); + memset(args, 0, externpy->size_of_result); + } +#ifdef _MSC_VER + SetLastError(current_lasterr); +#endif + errno = current_err; + + if (fnptr != NULL) + fnptr(externpy, args); +} + + +/* The cffi_start_python() function makes sure Python is initialized + and our cffi module is set up. It can be called manually from the + user C code. The same effect is obtained automatically from any + dll-exported ``extern "Python"`` function. This function returns + -1 if initialization failed, 0 if all is OK. */ +_CFFI_UNUSED_FN +static int cffi_start_python(void) +{ + if (_cffi_call_python == &_cffi_start_and_call_python) { + if (_cffi_start_python() == NULL) + return -1; + } + cffi_read_barrier(); + return 0; +} + +#undef cffi_compare_and_swap +#undef cffi_write_barrier +#undef cffi_read_barrier + +#ifdef __cplusplus +} +#endif diff --git a/lib/cffi/_imp_emulation.py b/lib/cffi/_imp_emulation.py new file mode 100644 index 0000000..136abdd --- /dev/null +++ b/lib/cffi/_imp_emulation.py @@ -0,0 +1,83 @@ + +try: + # this works on Python < 3.12 + from imp import * + +except ImportError: + # this is a limited emulation for Python >= 3.12. + # Note that this is used only for tests or for the old ffi.verify(). + # This is copied from the source code of Python 3.11. + + from _imp import (acquire_lock, release_lock, + is_builtin, is_frozen) + + from importlib._bootstrap import _load + + from importlib import machinery + import os + import sys + import tokenize + + SEARCH_ERROR = 0 + PY_SOURCE = 1 + PY_COMPILED = 2 + C_EXTENSION = 3 + PY_RESOURCE = 4 + PKG_DIRECTORY = 5 + C_BUILTIN = 6 + PY_FROZEN = 7 + PY_CODERESOURCE = 8 + IMP_HOOK = 9 + + def get_suffixes(): + extensions = [(s, 'rb', C_EXTENSION) + for s in machinery.EXTENSION_SUFFIXES] + source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] + bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] + return extensions + source + bytecode + + def find_module(name, path=None): + if not isinstance(name, str): + raise TypeError("'name' must be a str, not {}".format(type(name))) + elif not isinstance(path, (type(None), list)): + # Backwards-compatibility + raise RuntimeError("'path' must be None or a list, " + "not {}".format(type(path))) + + if path is None: + if is_builtin(name): + return None, None, ('', '', C_BUILTIN) + elif is_frozen(name): + return None, None, ('', '', PY_FROZEN) + else: + path = sys.path + + for entry in path: + package_directory = os.path.join(entry, name) + for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: + package_file_name = '__init__' + suffix + file_path = os.path.join(package_directory, package_file_name) + if os.path.isfile(file_path): + return None, package_directory, ('', '', PKG_DIRECTORY) + for suffix, mode, type_ in get_suffixes(): + file_name = name + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + break + else: + continue + break # Break out of outer loop when breaking out of inner loop. + else: + raise ImportError(name, name=name) + + encoding = None + if 'b' not in mode: + with open(file_path, 'rb') as file: + encoding = tokenize.detect_encoding(file.readline)[0] + file = open(file_path, mode, encoding=encoding) + return file, file_path, (suffix, mode, type_) + + def load_dynamic(name, path, file=None): + loader = machinery.ExtensionFileLoader(name, path) + spec = machinery.ModuleSpec(name=name, loader=loader, origin=path) + return _load(spec) diff --git a/lib/cffi/_shimmed_dist_utils.py b/lib/cffi/_shimmed_dist_utils.py new file mode 100644 index 0000000..c3d2312 --- /dev/null +++ b/lib/cffi/_shimmed_dist_utils.py @@ -0,0 +1,45 @@ +""" +Temporary shim module to indirect the bits of distutils we need from setuptools/distutils while providing useful +error messages beyond `No module named 'distutils' on Python >= 3.12, or when setuptools' vendored distutils is broken. + +This is a compromise to avoid a hard-dep on setuptools for Python >= 3.12, since many users don't need runtime compilation support from CFFI. +""" +import sys + +try: + # import setuptools first; this is the most robust way to ensure its embedded distutils is available + # (the .pth shim should usually work, but this is even more robust) + import setuptools +except Exception as ex: + if sys.version_info >= (3, 12): + # Python 3.12 has no built-in distutils to fall back on, so any import problem is fatal + raise Exception("This CFFI feature requires setuptools on Python >= 3.12. The setuptools module is missing or non-functional.") from ex + + # silently ignore on older Pythons (support fallback to stdlib distutils where available) +else: + del setuptools + +try: + # bring in just the bits of distutils we need, whether they really came from setuptools or stdlib-embedded distutils + from distutils import log, sysconfig + from distutils.ccompiler import CCompiler + from distutils.command.build_ext import build_ext + from distutils.core import Distribution, Extension + from distutils.dir_util import mkpath + from distutils.errors import DistutilsSetupError, CompileError, LinkError + from distutils.log import set_threshold, set_verbosity + + if sys.platform == 'win32': + try: + # FUTURE: msvc9compiler module was removed in setuptools 74; consider removing, as it's only used by an ancient patch in `recompiler` + from distutils.msvc9compiler import MSVCCompiler + except ImportError: + MSVCCompiler = None +except Exception as ex: + if sys.version_info >= (3, 12): + raise Exception("This CFFI feature requires setuptools on Python >= 3.12. Please install the setuptools package.") from ex + + # anything older, just let the underlying distutils import error fly + raise Exception("This CFFI feature requires distutils. Please install the distutils or setuptools package.") from ex + +del sys diff --git a/lib/cffi/api.py b/lib/cffi/api.py new file mode 100644 index 0000000..5a474f3 --- /dev/null +++ b/lib/cffi/api.py @@ -0,0 +1,967 @@ +import sys, types +from .lock import allocate_lock +from .error import CDefError +from . import model + +try: + callable +except NameError: + # Python 3.1 + from collections import Callable + callable = lambda x: isinstance(x, Callable) + +try: + basestring +except NameError: + # Python 3.x + basestring = str + +_unspecified = object() + + + +class FFI(object): + r''' + The main top-level class that you instantiate once, or once per module. + + Example usage: + + ffi = FFI() + ffi.cdef(""" + int printf(const char *, ...); + """) + + C = ffi.dlopen(None) # standard library + -or- + C = ffi.verify() # use a C compiler: verify the decl above is right + + C.printf("hello, %s!\n", ffi.new("char[]", "world")) + ''' + + def __init__(self, backend=None): + """Create an FFI instance. The 'backend' argument is used to + select a non-default backend, mostly for tests. + """ + if backend is None: + # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with + # _cffi_backend.so compiled. + import _cffi_backend as backend + from . import __version__ + if backend.__version__ != __version__: + # bad version! Try to be as explicit as possible. + if hasattr(backend, '__file__'): + # CPython + raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r. When we import the top-level '_cffi_backend' extension module, we get version %s, located in %r. The two versions should be equal; check your installation." % ( + __version__, __file__, + backend.__version__, backend.__file__)) + else: + # PyPy + raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r. This interpreter comes with a built-in '_cffi_backend' module, which is version %s. The two versions should be equal; check your installation." % ( + __version__, __file__, backend.__version__)) + # (If you insist you can also try to pass the option + # 'backend=backend_ctypes.CTypesBackend()', but don't + # rely on it! It's probably not going to work well.) + + from . import cparser + self._backend = backend + self._lock = allocate_lock() + self._parser = cparser.Parser() + self._cached_btypes = {} + self._parsed_types = types.ModuleType('parsed_types').__dict__ + self._new_types = types.ModuleType('new_types').__dict__ + self._function_caches = [] + self._libraries = [] + self._cdefsources = [] + self._included_ffis = [] + self._windows_unicode = None + self._init_once_cache = {} + self._cdef_version = None + self._embedding = None + self._typecache = model.get_typecache(backend) + if hasattr(backend, 'set_ffi'): + backend.set_ffi(self) + for name in list(backend.__dict__): + if name.startswith('RTLD_'): + setattr(self, name, getattr(backend, name)) + # + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) + self.BCharA = self._get_cached_btype(model.char_array_type) + if isinstance(backend, types.ModuleType): + # _cffi_backend: attach these constants to the class + if not hasattr(FFI, 'NULL'): + FFI.NULL = self.cast(self.BVoidP, 0) + FFI.CData, FFI.CType = backend._get_types() + else: + # ctypes backend: attach these constants to the instance + self.NULL = self.cast(self.BVoidP, 0) + self.CData, self.CType = backend._get_types() + self.buffer = backend.buffer + + def cdef(self, csource, override=False, packed=False, pack=None): + """Parse the given C source. This registers all declared functions, + types, and global variables. The functions and global variables can + then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. + The types can be used in 'ffi.new()' and other functions. + If 'packed' is specified as True, all structs declared inside this + cdef are packed, i.e. laid out without any field alignment at all. + Alternatively, 'pack' can be a small integer, and requests for + alignment greater than that are ignored (pack=1 is equivalent to + packed=True). + """ + self._cdef(csource, override=override, packed=packed, pack=pack) + + def embedding_api(self, csource, packed=False, pack=None): + self._cdef(csource, packed=packed, pack=pack, dllexport=True) + if self._embedding is None: + self._embedding = '' + + def _cdef(self, csource, override=False, **options): + if not isinstance(csource, str): # unicode, on Python 2 + if not isinstance(csource, basestring): + raise TypeError("cdef() argument must be a string") + csource = csource.encode('ascii') + with self._lock: + self._cdef_version = object() + self._parser.parse(csource, override=override, **options) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() + finishlist = self._parser._recomplete + if finishlist: + self._parser._recomplete = [] + for tp in finishlist: + tp.finish_backend_type(self, finishlist) + + def dlopen(self, name, flags=0): + """Load and return a dynamic library identified by 'name'. + The standard C library can be loaded by passing None. + Note that functions and types declared by 'ffi.cdef()' are not + linked to a particular library, just like C headers; in the + library we only look for the actual (untyped) symbols. + """ + if not (isinstance(name, basestring) or + name is None or + isinstance(name, self.CData)): + raise TypeError("dlopen(name): name must be a file name, None, " + "or an already-opened 'void *' handle") + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) + return lib + + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + + def _typeof(self, cdecl, consider_function_as_funcptr=False): + # string -> ctype object + try: + result = self._parsed_types[cdecl] + except KeyError: + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) + return btype + + def typeof(self, cdecl): + """Parse the C type given as a string and return the + corresponding object. + It can also be used on 'cdata' instance to get its C type. + """ + if isinstance(cdecl, basestring): + return self._typeof(cdecl) + if isinstance(cdecl, self.CData): + return self._backend.typeof(cdecl) + if isinstance(cdecl, types.BuiltinFunctionType): + res = _builtin_function_type(cdecl) + if res is not None: + return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) + raise TypeError(type(cdecl)) + + def sizeof(self, cdecl): + """Return the size in bytes of the argument. It can be a + string naming a C type, or a 'cdata' instance. + """ + if isinstance(cdecl, basestring): + BType = self._typeof(cdecl) + return self._backend.sizeof(BType) + else: + return self._backend.sizeof(cdecl) + + def alignof(self, cdecl): + """Return the natural alignment size in bytes of the C type + given as a string. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.alignof(cdecl) + + def offsetof(self, cdecl, *fields_or_indexes): + """Return the offset of the named field inside the given + structure or array, which must be given as a C type name. + You can give several field names in case of nested structures. + You can also give numeric values which correspond to array + items, in case of an array type. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._typeoffsetof(cdecl, *fields_or_indexes)[1] + + def new(self, cdecl, init=None): + """Allocate an instance according to the specified C type and + return a pointer to it. The specified C type must be either a + pointer or an array: ``new('X *')`` allocates an X and returns + a pointer to it, whereas ``new('X[n]')`` allocates an array of + n X'es and returns an array referencing it (which works + mostly like a pointer, like in C). You can also use + ``new('X[]', n)`` to allocate an array of a non-constant + length n. + + The memory is initialized following the rules of declaring a + global variable in C: by default it is zero-initialized, but + an explicit initializer can be given which can be used to + fill all or part of the memory. + + When the returned object goes out of scope, the memory + is freed. In other words the returned object has + ownership of the value of type 'cdecl' that it points to. This + means that the raw data can be used as long as this object is + kept alive, but must not be used for a longer time. Be careful + about that when copying the pointer to the memory somewhere + else, e.g. into another structure. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.newp(cdecl, init) + + def new_allocator(self, alloc=None, free=None, + should_clear_after_alloc=True): + """Return a new allocator, i.e. a function that behaves like ffi.new() + but uses the provided low-level 'alloc' and 'free' functions. + + 'alloc' is called with the size as argument. If it returns NULL, a + MemoryError is raised. 'free' is called with the result of 'alloc' + as argument. Both can be either Python function or directly C + functions. If 'free' is None, then no free function is called. + If both 'alloc' and 'free' are None, the default is used. + + If 'should_clear_after_alloc' is set to False, then the memory + returned by 'alloc' is assumed to be already cleared (or you are + fine with garbage); otherwise CFFI will clear it. + """ + compiled_ffi = self._backend.FFI() + allocator = compiled_ffi.new_allocator(alloc, free, + should_clear_after_alloc) + def allocate(cdecl, init=None): + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return allocator(cdecl, init) + return allocate + + def cast(self, cdecl, source): + """Similar to a C cast: returns an instance of the named C + type initialized with the given 'source'. The source is + casted between integers or pointers of any type. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.cast(cdecl, source) + + def string(self, cdata, maxlen=-1): + """Return a Python string (or unicode string) from the 'cdata'. + If 'cdata' is a pointer or array of characters or bytes, returns + the null-terminated string. The returned string extends until + the first null character, or at most 'maxlen' characters. If + 'cdata' is an array then 'maxlen' defaults to its length. + + If 'cdata' is a pointer or array of wchar_t, returns a unicode + string following the same rules. + + If 'cdata' is a single character or byte or a wchar_t, returns + it as a string or unicode string. + + If 'cdata' is an enum, returns the value of the enumerator as a + string, or 'NUMBER' if the value is out of range. + """ + return self._backend.string(cdata, maxlen) + + def unpack(self, cdata, length): + """Unpack an array of C data of the given length, + returning a Python string/unicode/list. + + If 'cdata' is a pointer to 'char', returns a byte string. + It does not stop at the first null. This is equivalent to: + ffi.buffer(cdata, length)[:] + + If 'cdata' is a pointer to 'wchar_t', returns a unicode string. + 'length' is measured in wchar_t's; it is not the size in bytes. + + If 'cdata' is a pointer to anything else, returns a list of + 'length' items. This is a faster equivalent to: + [cdata[i] for i in range(length)] + """ + return self._backend.unpack(cdata, length) + + #def buffer(self, cdata, size=-1): + # """Return a read-write buffer object that references the raw C data + # pointed to by the given 'cdata'. The 'cdata' must be a pointer or + # an array. Can be passed to functions expecting a buffer, or directly + # manipulated with: + # + # buf[:] get a copy of it in a regular string, or + # buf[idx] as a single character + # buf[:] = ... + # buf[idx] = ... change the content + # """ + # note that 'buffer' is a type, set on this instance by __init__ + + def from_buffer(self, cdecl, python_buffer=_unspecified, + require_writable=False): + """Return a cdata of the given type pointing to the data of the + given Python object, which must support the buffer interface. + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) + but only on objects containing large quantities of raw data + in some other format, like 'array.array' or numpy arrays. + + The first argument is optional and default to 'char[]'. + """ + if python_buffer is _unspecified: + cdecl, python_buffer = self.BCharA, cdecl + elif isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.from_buffer(cdecl, python_buffer, + require_writable) + + def memmove(self, dest, src, n): + """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. + + Like the C function memmove(), the memory areas may overlap; + apart from that it behaves like the C function memcpy(). + + 'src' can be any cdata ptr or array, or any Python buffer object. + 'dest' can be any cdata ptr or array, or a writable Python buffer + object. The size to copy, 'n', is always measured in bytes. + + Unlike other methods, this one supports all Python buffer including + byte strings and bytearrays---but it still does not support + non-contiguous buffers. + """ + return self._backend.memmove(dest, src, n) + + def callback(self, cdecl, python_callable=None, error=None, onerror=None): + """Return a callback object or a decorator making such a + callback object. 'cdecl' must name a C function pointer type. + The callback invokes the specified 'python_callable' (which may + be provided either directly or via a decorator). Important: the + callback object must be manually kept alive for as long as the + callback may be invoked from the C level. + """ + def callback_decorator_wrap(python_callable): + if not callable(python_callable): + raise TypeError("the 'python_callable' argument " + "is not callable") + return self._backend.callback(cdecl, python_callable, + error, onerror) + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl, consider_function_as_funcptr=True) + if python_callable is None: + return callback_decorator_wrap # decorator mode + else: + return callback_decorator_wrap(python_callable) # direct mode + + def getctype(self, cdecl, replace_with=''): + """Return a string giving the C type 'cdecl', which may be itself + a string or a object. If 'replace_with' is given, it gives + extra text to append (or insert for more complicated C types), like + a variable name, or '*' to get actually the C type 'pointer-to-cdecl'. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + replace_with = replace_with.strip() + if (replace_with.startswith('*') + and '&[' in self._backend.getcname(cdecl, '&')): + replace_with = '(%s)' % replace_with + elif replace_with and not replace_with[0] in '[(': + replace_with = ' ' + replace_with + return self._backend.getcname(cdecl, replace_with) + + def gc(self, cdata, destructor, size=0): + """Return a new cdata object that points to the same + data. Later, when this new cdata object is garbage-collected, + 'destructor(old_cdata_object)' will be called. + + The optional 'size' gives an estimate of the size, used to + trigger the garbage collection more eagerly. So far only used + on PyPy. It tells the GC that the returned object keeps alive + roughly 'size' bytes of external memory. + """ + return self._backend.gcp(cdata, destructor, size) + + def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! + try: + BType = self._cached_btypes[type] + except KeyError: + finishlist = [] + BType = type.get_cached_btype(self, finishlist) + for type in finishlist: + type.finish_backend_type(self, finishlist) + return BType + + def verify(self, source='', tmpdir=None, **kwargs): + """Verify that the current ffi signatures compile on this + machine, and return a dynamic library object. The dynamic + library can be used to call functions and access global + variables declared in this 'ffi'. The library is compiled + by the C compiler: it gives you C-level API compatibility + (including calling macros). This is unlike 'ffi.dlopen()', + which requires binary compatibility in the signatures. + """ + from .verifier import Verifier, _caller_dir_pycache + # + # If set_unicode(True) was called, insert the UNICODE and + # _UNICODE macro declarations + if self._windows_unicode: + self._apply_windows_unicode(kwargs) + # + # Set the tmpdir here, and not in Verifier.__init__: it picks + # up the caller's directory, which we want to be the caller of + # ffi.verify(), as opposed to the caller of Veritier(). + tmpdir = tmpdir or _caller_dir_pycache() + # + # Make a Verifier() and use it to load the library. + self.verifier = Verifier(self, source, tmpdir, **kwargs) + lib = self.verifier.load_library() + # + # Save the loaded library for keep-alive purposes, even + # if the caller doesn't keep it alive itself (it should). + self._libraries.append(lib) + return lib + + def _get_errno(self): + return self._backend.get_errno() + def _set_errno(self, errno): + self._backend.set_errno(errno) + errno = property(_get_errno, _set_errno, None, + "the value of 'errno' from/to the C calls") + + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + + def _pointer_to(self, ctype): + with self._lock: + return model.pointer_cache(self, ctype) + + def addressof(self, cdata, *fields_or_indexes): + """Return the address of a . + If 'fields_or_indexes' are given, returns the address of that + field or array item in the structure or array, recursively in + case of nested structures. + """ + try: + ctype = self._backend.typeof(cdata) + except TypeError: + if '__addressof__' in type(cdata).__dict__: + return type(cdata).__addressof__(cdata, *fields_or_indexes) + raise + if fields_or_indexes: + ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes) + else: + if ctype.kind == "pointer": + raise TypeError("addressof(pointer)") + offset = 0 + ctypeptr = self._pointer_to(ctype) + return self._backend.rawaddressof(ctypeptr, cdata, offset) + + def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes): + ctype, offset = self._backend.typeoffsetof(ctype, field_or_index) + for field1 in fields_or_indexes: + ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1) + offset += offset1 + return ctype, offset + + def include(self, ffi_to_include): + """Includes the typedefs, structs, unions and enums defined + in another FFI instance. Usage is similar to a #include in C, + where a part of the program might include types defined in + another part for its own usage. Note that the include() + method has no effect on functions, constants and global + variables, which must anyway be accessed directly from the + lib object returned by the original FFI instance. + """ + if not isinstance(ffi_to_include, FFI): + raise TypeError("ffi.include() expects an argument that is also of" + " type cffi.FFI, not %r" % ( + type(ffi_to_include).__name__,)) + if ffi_to_include is self: + raise ValueError("self.include(self)") + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') + self._included_ffis.append(ffi_to_include) + + def new_handle(self, x): + return self._backend.newp_handle(self.BVoidP, x) + + def from_handle(self, x): + return self._backend.from_handle(x) + + def release(self, x): + self._backend.release(x) + + def set_unicode(self, enabled_flag): + """Windows: if 'enabled_flag' is True, enable the UNICODE and + _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR + to be (pointers to) wchar_t. If 'enabled_flag' is False, + declare these types to be (pointers to) plain 8-bit characters. + This is mostly for backward compatibility; you usually want True. + """ + if self._windows_unicode is not None: + raise ValueError("set_unicode() can only be called once") + enabled_flag = bool(enabled_flag) + if enabled_flag: + self.cdef("typedef wchar_t TBYTE;" + "typedef wchar_t TCHAR;" + "typedef const wchar_t *LPCTSTR;" + "typedef const wchar_t *PCTSTR;" + "typedef wchar_t *LPTSTR;" + "typedef wchar_t *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + else: + self.cdef("typedef char TBYTE;" + "typedef char TCHAR;" + "typedef const char *LPCTSTR;" + "typedef const char *PCTSTR;" + "typedef char *LPTSTR;" + "typedef char *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + self._windows_unicode = enabled_flag + + def _apply_windows_unicode(self, kwds): + defmacros = kwds.get('define_macros', ()) + if not isinstance(defmacros, (list, tuple)): + raise TypeError("'define_macros' must be a list or tuple") + defmacros = list(defmacros) + [('UNICODE', '1'), + ('_UNICODE', '1')] + kwds['define_macros'] = defmacros + + def _apply_embedding_fix(self, kwds): + # must include an argument like "-lpython2.7" for the compiler + def ensure(key, value): + lst = kwds.setdefault(key, []) + if value not in lst: + lst.append(value) + # + if '__pypy__' in sys.builtin_module_names: + import os + if sys.platform == "win32": + # we need 'libpypy-c.lib'. Current distributions of + # pypy (>= 4.1) contain it as 'libs/python27.lib'. + pythonlib = "python{0[0]}{0[1]}".format(sys.version_info) + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'libs')) + else: + # we need 'libpypy-c.{so,dylib}', which should be by + # default located in 'sys.prefix/bin' for installed + # systems. + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'bin')) + # On uninstalled pypy's, the libpypy-c is typically found in + # .../pypy/goal/. + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'pypy', 'goal')) + else: + if sys.platform == "win32": + template = "python%d%d" + if hasattr(sys, 'gettotalrefcount'): + template += '_d' + else: + try: + import sysconfig + except ImportError: # 2.6 + from cffi._shimmed_dist_utils import sysconfig + template = "python%d.%d" + if sysconfig.get_config_var('DEBUG_EXT'): + template += sysconfig.get_config_var('DEBUG_EXT') + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + if hasattr(sys, 'abiflags'): + pythonlib += sys.abiflags + ensure('libraries', pythonlib) + if sys.platform == "win32": + ensure('extra_link_args', '/MANIFEST') + + def set_source(self, module_name, source, source_extension='.c', **kwds): + import os + if hasattr(self, '_assigned_source'): + raise ValueError("set_source() cannot be called several times " + "per ffi object") + if not isinstance(module_name, basestring): + raise TypeError("'module_name' must be a string") + if os.sep in module_name or (os.altsep and os.altsep in module_name): + raise ValueError("'module_name' must not contain '/': use a dotted " + "name to make a 'package.module' location") + self._assigned_source = (str(module_name), source, + source_extension, kwds) + + def set_source_pkgconfig(self, module_name, pkgconfig_libs, source, + source_extension='.c', **kwds): + from . import pkgconfig + if not isinstance(pkgconfig_libs, list): + raise TypeError("the pkgconfig_libs argument must be a list " + "of package names") + kwds2 = pkgconfig.flags_from_pkgconfig(pkgconfig_libs) + pkgconfig.merge_flags(kwds, kwds2) + self.set_source(module_name, source, source_extension, **kwds) + + def distutils_extension(self, tmpdir='build', verbose=True): + from cffi._shimmed_dist_utils import mkpath + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + if hasattr(self, 'verifier'): # fallback, 'tmpdir' ignored + return self.verifier.get_extension() + raise ValueError("set_source() must be called before" + " distutils_extension()") + module_name, source, source_extension, kwds = self._assigned_source + if source is None: + raise TypeError("distutils_extension() is only for C extension " + "modules, not for dlopen()-style pure Python " + "modules") + mkpath(tmpdir) + ext, updated = recompile(self, module_name, + source, tmpdir=tmpdir, extradir=tmpdir, + source_extension=source_extension, + call_c_compiler=False, **kwds) + if verbose: + if updated: + sys.stderr.write("regenerated: %r\n" % (ext.sources[0],)) + else: + sys.stderr.write("not modified: %r\n" % (ext.sources[0],)) + return ext + + def emit_c_code(self, filename): + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before emit_c_code()") + module_name, source, source_extension, kwds = self._assigned_source + if source is None: + raise TypeError("emit_c_code() is only for C extension modules, " + "not for dlopen()-style pure Python modules") + recompile(self, module_name, source, + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) + + def emit_python_code(self, filename): + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before emit_c_code()") + module_name, source, source_extension, kwds = self._assigned_source + if source is not None: + raise TypeError("emit_python_code() is only for dlopen()-style " + "pure Python modules, not for C extension modules") + recompile(self, module_name, source, + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) + + def compile(self, tmpdir='.', verbose=0, target=None, debug=None): + """The 'target' argument gives the final file name of the + compiled DLL. Use '*' to force distutils' choice, suitable for + regular CPython C API modules. Use a file name ending in '.*' + to ask for the system's default extension for dynamic libraries + (.so/.dll/.dylib). + + The default is '*' when building a non-embedded C API extension, + and (module_name + '.*') when building an embedded library. + """ + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before compile()") + module_name, source, source_extension, kwds = self._assigned_source + return recompile(self, module_name, source, tmpdir=tmpdir, + target=target, source_extension=source_extension, + compiler_verbose=verbose, debug=debug, **kwds) + + def init_once(self, func, tag): + # Read _init_once_cache[tag], which is either (False, lock) if + # we're calling the function now in some thread, or (True, result). + # Don't call setdefault() in most cases, to avoid allocating and + # immediately freeing a lock; but still use setdefaut() to avoid + # races. + try: + x = self._init_once_cache[tag] + except KeyError: + x = self._init_once_cache.setdefault(tag, (False, allocate_lock())) + # Common case: we got (True, result), so we return the result. + if x[0]: + return x[1] + # Else, it's a lock. Acquire it to serialize the following tests. + with x[1]: + # Read again from _init_once_cache the current status. + x = self._init_once_cache[tag] + if x[0]: + return x[1] + # Call the function and store the result back. + result = func() + self._init_once_cache[tag] = (True, result) + return result + + def embedding_init_code(self, pysource): + if self._embedding: + raise ValueError("embedding_init_code() can only be called once") + # fix 'pysource' before it gets dumped into the C file: + # - remove empty lines at the beginning, so it starts at "line 1" + # - dedent, if all non-empty lines are indented + # - check for SyntaxErrors + import re + match = re.match(r'\s*\n', pysource) + if match: + pysource = pysource[match.end():] + lines = pysource.splitlines() or [''] + prefix = re.match(r'\s*', lines[0]).group() + for i in range(1, len(lines)): + line = lines[i] + if line.rstrip(): + while not line.startswith(prefix): + prefix = prefix[:-1] + i = len(prefix) + lines = [line[i:]+'\n' for line in lines] + pysource = ''.join(lines) + # + compile(pysource, "cffi_init", "exec") + # + self._embedding = pysource + + def def_extern(self, *args, **kwds): + raise ValueError("ffi.def_extern() is only available on API-mode FFI " + "objects") + + def list_types(self): + """Returns the user type names known to this FFI instance. + This returns a tuple containing three lists of names: + (typedef_names, names_of_structs, names_of_unions) + """ + typedefs = [] + structs = [] + unions = [] + for key in self._parser._declarations: + if key.startswith('typedef '): + typedefs.append(key[8:]) + elif key.startswith('struct '): + structs.append(key[7:]) + elif key.startswith('union '): + unions.append(key[6:]) + typedefs.sort() + structs.sort() + unions.sort() + return (typedefs, structs, unions) + + +def _load_backend_lib(backend, name, flags): + import os + if not isinstance(name, basestring): + if sys.platform != "win32" or name is not None: + return backend.load_library(name, flags) + name = "c" # Windows: load_library(None) fails, but this works + # on Python 2 (backward compatibility hack only) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + if name == "c" and sys.platform == "win32" and sys.version_info >= (3,): + raise OSError("dlopen(None) cannot work on Windows for Python 3 " + "(see http://bugs.python.org/issue23606)") + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) + +def _make_ffi_library(ffi, libname, flags): + backend = ffi._backend + backendlib = _load_backend_lib(backend, libname, flags) + # + def accessor_function(name): + key = 'function ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + value = backendlib.load_function(BType, name) + library.__dict__[name] = value + # + def accessor_variable(name): + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + read_variable = backendlib.read_variable + write_variable = backendlib.write_variable + setattr(FFILibrary, name, property( + lambda self: read_variable(BType, name), + lambda self, value: write_variable(BType, name, value))) + # + def addressof_var(name): + try: + return addr_variables[name] + except KeyError: + with ffi._lock: + if name not in addr_variables: + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + if BType.kind != 'array': + BType = model.pointer_cache(ffi, BType) + p = backendlib.load_function(BType, name) + addr_variables[name] = p + return addr_variables[name] + # + def accessor_constant(name): + raise NotImplementedError("non-integer constant '%s' cannot be " + "accessed from a dlopen() library" % (name,)) + # + def accessor_int_constant(name): + library.__dict__[name] = ffi._parser._int_constants[name] + # + accessors = {} + accessors_version = [False] + addr_variables = {} + # + def update_accessors(): + if accessors_version[0] is ffi._cdef_version: + return + # + for key, (tp, _) in ffi._parser._declarations.items(): + if not isinstance(tp, model.EnumType): + tag, name = key.split(' ', 1) + if tag == 'function': + accessors[name] = accessor_function + elif tag == 'variable': + accessors[name] = accessor_variable + elif tag == 'constant': + accessors[name] = accessor_constant + else: + for i, enumname in enumerate(tp.enumerators): + def accessor_enum(name, tp=tp, i=i): + tp.check_not_partial() + library.__dict__[name] = tp.enumvalues[i] + accessors[enumname] = accessor_enum + for name in ffi._parser._int_constants: + accessors.setdefault(name, accessor_int_constant) + accessors_version[0] = ffi._cdef_version + # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + if name not in accessors: + update_accessors() + if name not in accessors: + raise AttributeError(name) + accessors[name](name) + # + class FFILibrary(object): + def __getattr__(self, name): + make_accessor(name) + return getattr(self, name) + def __setattr__(self, name, value): + try: + property = getattr(self.__class__, name) + except AttributeError: + make_accessor(name) + setattr(self, name, value) + else: + property.__set__(self, value) + def __dir__(self): + with ffi._lock: + update_accessors() + return accessors.keys() + def __addressof__(self, name): + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + make_accessor(name) + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + raise AttributeError("cffi library has no function or " + "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() + # + if isinstance(libname, basestring): + try: + if not isinstance(libname, str): # unicode, on Python 2 + libname = libname.encode('utf-8') + FFILibrary.__name__ = 'FFILibrary_%s' % libname + except UnicodeError: + pass + library = FFILibrary() + return library, library.__dict__ + +def _builtin_function_type(func): + # a hack to make at least ffi.typeof(builtin_function) work, + # if the builtin function was obtained by 'vengine_cpy'. + import sys + try: + module = sys.modules[func.__module__] + ffi = module._cffi_original_ffi + types_of_builtin_funcs = module._cffi_types_of_builtin_funcs + tp = types_of_builtin_funcs[func] + except (KeyError, AttributeError, TypeError): + return None + else: + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/lib/cffi/backend_ctypes.py b/lib/cffi/backend_ctypes.py new file mode 100644 index 0000000..e7956a7 --- /dev/null +++ b/lib/cffi/backend_ctypes.py @@ -0,0 +1,1121 @@ +import ctypes, ctypes.util, operator, sys +from . import model + +if sys.version_info < (3,): + bytechr = chr +else: + unicode = str + long = int + xrange = range + bytechr = lambda num: bytes([num]) + +class CTypesType(type): + pass + +class CTypesData(object): + __metaclass__ = CTypesType + __slots__ = ['__weakref__'] + __name__ = '' + + def __init__(self, *args): + raise TypeError("cannot instantiate %r" % (self.__class__,)) + + @classmethod + def _newp(cls, init): + raise TypeError("expected a pointer or array ctype, got '%s'" + % (cls._get_c_name(),)) + + @staticmethod + def _to_ctypes(value): + raise TypeError + + @classmethod + def _arg_to_ctypes(cls, *value): + try: + ctype = cls._ctype + except AttributeError: + raise TypeError("cannot create an instance of %r" % (cls,)) + if value: + res = cls._to_ctypes(*value) + if not isinstance(res, ctype): + res = cls._ctype(res) + else: + res = cls._ctype() + return res + + @classmethod + def _create_ctype_obj(cls, init): + if init is None: + return cls._arg_to_ctypes() + else: + return cls._arg_to_ctypes(init) + + @staticmethod + def _from_ctypes(ctypes_value): + raise TypeError + + @classmethod + def _get_c_name(cls, replace_with=''): + return cls._reftypename.replace(' &', replace_with) + + @classmethod + def _fix_class(cls): + cls.__name__ = 'CData<%s>' % (cls._get_c_name(),) + cls.__qualname__ = 'CData<%s>' % (cls._get_c_name(),) + cls.__module__ = 'ffi' + + def _get_own_repr(self): + raise NotImplementedError + + def _addr_repr(self, address): + if address == 0: + return 'NULL' + else: + if address < 0: + address += 1 << (8*ctypes.sizeof(ctypes.c_void_p)) + return '0x%x' % address + + def __repr__(self, c_name=None): + own = self._get_own_repr() + return '' % (c_name or self._get_c_name(), own) + + def _convert_to_address(self, BClass): + if BClass is None: + raise TypeError("cannot convert %r to an address" % ( + self._get_c_name(),)) + else: + raise TypeError("cannot convert %r to %r" % ( + self._get_c_name(), BClass._get_c_name())) + + @classmethod + def _get_size(cls): + return ctypes.sizeof(cls._ctype) + + def _get_size_of_instance(self): + return ctypes.sizeof(self._ctype) + + @classmethod + def _cast_from(cls, source): + raise TypeError("cannot cast to %r" % (cls._get_c_name(),)) + + def _cast_to_integer(self): + return self._convert_to_address(None) + + @classmethod + def _alignment(cls): + return ctypes.alignment(cls._ctype) + + def __iter__(self): + raise TypeError("cdata %r does not support iteration" % ( + self._get_c_name()),) + + def _make_cmp(name): + cmpfunc = getattr(operator, name) + def cmp(self, other): + v_is_ptr = not isinstance(self, CTypesGenericPrimitive) + w_is_ptr = (isinstance(other, CTypesData) and + not isinstance(other, CTypesGenericPrimitive)) + if v_is_ptr and w_is_ptr: + return cmpfunc(self._convert_to_address(None), + other._convert_to_address(None)) + elif v_is_ptr or w_is_ptr: + return NotImplemented + else: + if isinstance(self, CTypesGenericPrimitive): + self = self._value + if isinstance(other, CTypesGenericPrimitive): + other = other._value + return cmpfunc(self, other) + cmp.func_name = name + return cmp + + __eq__ = _make_cmp('__eq__') + __ne__ = _make_cmp('__ne__') + __lt__ = _make_cmp('__lt__') + __le__ = _make_cmp('__le__') + __gt__ = _make_cmp('__gt__') + __ge__ = _make_cmp('__ge__') + + def __hash__(self): + return hash(self._convert_to_address(None)) + + def _to_string(self, maxlen): + raise TypeError("string(): %r" % (self,)) + + +class CTypesGenericPrimitive(CTypesData): + __slots__ = [] + + def __hash__(self): + return hash(self._value) + + def _get_own_repr(self): + return repr(self._from_ctypes(self._value)) + + +class CTypesGenericArray(CTypesData): + __slots__ = [] + + @classmethod + def _newp(cls, init): + return cls(init) + + def __iter__(self): + for i in xrange(len(self)): + yield self[i] + + def _get_own_repr(self): + return self._addr_repr(ctypes.addressof(self._blob)) + + +class CTypesGenericPtr(CTypesData): + __slots__ = ['_address', '_as_ctype_ptr'] + _automatic_casts = False + kind = "pointer" + + @classmethod + def _newp(cls, init): + return cls(init) + + @classmethod + def _cast_from(cls, source): + if source is None: + address = 0 + elif isinstance(source, CTypesData): + address = source._cast_to_integer() + elif isinstance(source, (int, long)): + address = source + else: + raise TypeError("bad type for cast to %r: %r" % + (cls, type(source).__name__)) + return cls._new_pointer_at(address) + + @classmethod + def _new_pointer_at(cls, address): + self = cls.__new__(cls) + self._address = address + self._as_ctype_ptr = ctypes.cast(address, cls._ctype) + return self + + def _get_own_repr(self): + try: + return self._addr_repr(self._address) + except AttributeError: + return '???' + + def _cast_to_integer(self): + return self._address + + def __nonzero__(self): + return bool(self._address) + __bool__ = __nonzero__ + + @classmethod + def _to_ctypes(cls, value): + if not isinstance(value, CTypesData): + raise TypeError("unexpected %s object" % type(value).__name__) + address = value._convert_to_address(cls) + return ctypes.cast(address, cls._ctype) + + @classmethod + def _from_ctypes(cls, ctypes_ptr): + address = ctypes.cast(ctypes_ptr, ctypes.c_void_p).value or 0 + return cls._new_pointer_at(address) + + @classmethod + def _initialize(cls, ctypes_ptr, value): + if value: + ctypes_ptr.contents = cls._to_ctypes(value).contents + + def _convert_to_address(self, BClass): + if (BClass in (self.__class__, None) or BClass._automatic_casts + or self._automatic_casts): + return self._address + else: + return CTypesData._convert_to_address(self, BClass) + + +class CTypesBaseStructOrUnion(CTypesData): + __slots__ = ['_blob'] + + @classmethod + def _create_ctype_obj(cls, init): + # may be overridden + raise TypeError("cannot instantiate opaque type %s" % (cls,)) + + def _get_own_repr(self): + return self._addr_repr(ctypes.addressof(self._blob)) + + @classmethod + def _offsetof(cls, fieldname): + return getattr(cls._ctype, fieldname).offset + + def _convert_to_address(self, BClass): + if getattr(BClass, '_BItem', None) is self.__class__: + return ctypes.addressof(self._blob) + else: + return CTypesData._convert_to_address(self, BClass) + + @classmethod + def _from_ctypes(cls, ctypes_struct_or_union): + self = cls.__new__(cls) + self._blob = ctypes_struct_or_union + return self + + @classmethod + def _to_ctypes(cls, value): + return value._blob + + def __repr__(self, c_name=None): + return CTypesData.__repr__(self, c_name or self._get_c_name(' &')) + + +class CTypesBackend(object): + + PRIMITIVE_TYPES = { + 'char': ctypes.c_char, + 'short': ctypes.c_short, + 'int': ctypes.c_int, + 'long': ctypes.c_long, + 'long long': ctypes.c_longlong, + 'signed char': ctypes.c_byte, + 'unsigned char': ctypes.c_ubyte, + 'unsigned short': ctypes.c_ushort, + 'unsigned int': ctypes.c_uint, + 'unsigned long': ctypes.c_ulong, + 'unsigned long long': ctypes.c_ulonglong, + 'float': ctypes.c_float, + 'double': ctypes.c_double, + '_Bool': ctypes.c_bool, + } + + for _name in ['unsigned long long', 'unsigned long', + 'unsigned int', 'unsigned short', 'unsigned char']: + _size = ctypes.sizeof(PRIMITIVE_TYPES[_name]) + PRIMITIVE_TYPES['uint%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_void_p): + PRIMITIVE_TYPES['uintptr_t'] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_size_t): + PRIMITIVE_TYPES['size_t'] = PRIMITIVE_TYPES[_name] + + for _name in ['long long', 'long', 'int', 'short', 'signed char']: + _size = ctypes.sizeof(PRIMITIVE_TYPES[_name]) + PRIMITIVE_TYPES['int%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_void_p): + PRIMITIVE_TYPES['intptr_t'] = PRIMITIVE_TYPES[_name] + PRIMITIVE_TYPES['ptrdiff_t'] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_size_t): + PRIMITIVE_TYPES['ssize_t'] = PRIMITIVE_TYPES[_name] + + + def __init__(self): + self.RTLD_LAZY = 0 # not supported anyway by ctypes + self.RTLD_NOW = 0 + self.RTLD_GLOBAL = ctypes.RTLD_GLOBAL + self.RTLD_LOCAL = ctypes.RTLD_LOCAL + + def set_ffi(self, ffi): + self.ffi = ffi + + def _get_types(self): + return CTypesData, CTypesType + + def load_library(self, path, flags=0): + cdll = ctypes.CDLL(path, flags) + return CTypesLibrary(self, cdll) + + def new_void_type(self): + class CTypesVoid(CTypesData): + __slots__ = [] + _reftypename = 'void &' + @staticmethod + def _from_ctypes(novalue): + return None + @staticmethod + def _to_ctypes(novalue): + if novalue is not None: + raise TypeError("None expected, got %s object" % + (type(novalue).__name__,)) + return None + CTypesVoid._fix_class() + return CTypesVoid + + def new_primitive_type(self, name): + if name == 'wchar_t': + raise NotImplementedError(name) + ctype = self.PRIMITIVE_TYPES[name] + if name == 'char': + kind = 'char' + elif name in ('float', 'double'): + kind = 'float' + else: + if name in ('signed char', 'unsigned char'): + kind = 'byte' + elif name == '_Bool': + kind = 'bool' + else: + kind = 'int' + is_signed = (ctype(-1).value == -1) + # + def _cast_source_to_int(source): + if isinstance(source, (int, long, float)): + source = int(source) + elif isinstance(source, CTypesData): + source = source._cast_to_integer() + elif isinstance(source, bytes): + source = ord(source) + elif source is None: + source = 0 + else: + raise TypeError("bad type for cast to %r: %r" % + (CTypesPrimitive, type(source).__name__)) + return source + # + kind1 = kind + class CTypesPrimitive(CTypesGenericPrimitive): + __slots__ = ['_value'] + _ctype = ctype + _reftypename = '%s &' % name + kind = kind1 + + def __init__(self, value): + self._value = value + + @staticmethod + def _create_ctype_obj(init): + if init is None: + return ctype() + return ctype(CTypesPrimitive._to_ctypes(init)) + + if kind == 'int' or kind == 'byte': + @classmethod + def _cast_from(cls, source): + source = _cast_source_to_int(source) + source = ctype(source).value # cast within range + return cls(source) + def __int__(self): + return self._value + + if kind == 'bool': + @classmethod + def _cast_from(cls, source): + if not isinstance(source, (int, long, float)): + source = _cast_source_to_int(source) + return cls(bool(source)) + def __int__(self): + return int(self._value) + + if kind == 'char': + @classmethod + def _cast_from(cls, source): + source = _cast_source_to_int(source) + source = bytechr(source & 0xFF) + return cls(source) + def __int__(self): + return ord(self._value) + + if kind == 'float': + @classmethod + def _cast_from(cls, source): + if isinstance(source, float): + pass + elif isinstance(source, CTypesGenericPrimitive): + if hasattr(source, '__float__'): + source = float(source) + else: + source = int(source) + else: + source = _cast_source_to_int(source) + source = ctype(source).value # fix precision + return cls(source) + def __int__(self): + return int(self._value) + def __float__(self): + return self._value + + _cast_to_integer = __int__ + + if kind == 'int' or kind == 'byte' or kind == 'bool': + @staticmethod + def _to_ctypes(x): + if not isinstance(x, (int, long)): + if isinstance(x, CTypesData): + x = int(x) + else: + raise TypeError("integer expected, got %s" % + type(x).__name__) + if ctype(x).value != x: + if not is_signed and x < 0: + raise OverflowError("%s: negative integer" % name) + else: + raise OverflowError("%s: integer out of bounds" + % name) + return x + + if kind == 'char': + @staticmethod + def _to_ctypes(x): + if isinstance(x, bytes) and len(x) == 1: + return x + if isinstance(x, CTypesPrimitive): # > + return x._value + raise TypeError("character expected, got %s" % + type(x).__name__) + def __nonzero__(self): + return ord(self._value) != 0 + else: + def __nonzero__(self): + return self._value != 0 + __bool__ = __nonzero__ + + if kind == 'float': + @staticmethod + def _to_ctypes(x): + if not isinstance(x, (int, long, float, CTypesData)): + raise TypeError("float expected, got %s" % + type(x).__name__) + return ctype(x).value + + @staticmethod + def _from_ctypes(value): + return getattr(value, 'value', value) + + @staticmethod + def _initialize(blob, init): + blob.value = CTypesPrimitive._to_ctypes(init) + + if kind == 'char': + def _to_string(self, maxlen): + return self._value + if kind == 'byte': + def _to_string(self, maxlen): + return chr(self._value & 0xff) + # + CTypesPrimitive._fix_class() + return CTypesPrimitive + + def new_pointer_type(self, BItem): + getbtype = self.ffi._get_cached_btype + if BItem is getbtype(model.PrimitiveType('char')): + kind = 'charp' + elif BItem in (getbtype(model.PrimitiveType('signed char')), + getbtype(model.PrimitiveType('unsigned char'))): + kind = 'bytep' + elif BItem is getbtype(model.void_type): + kind = 'voidp' + else: + kind = 'generic' + # + class CTypesPtr(CTypesGenericPtr): + __slots__ = ['_own'] + if kind == 'charp': + __slots__ += ['__as_strbuf'] + _BItem = BItem + if hasattr(BItem, '_ctype'): + _ctype = ctypes.POINTER(BItem._ctype) + _bitem_size = ctypes.sizeof(BItem._ctype) + else: + _ctype = ctypes.c_void_p + if issubclass(BItem, CTypesGenericArray): + _reftypename = BItem._get_c_name('(* &)') + else: + _reftypename = BItem._get_c_name(' * &') + + def __init__(self, init): + ctypeobj = BItem._create_ctype_obj(init) + if kind == 'charp': + self.__as_strbuf = ctypes.create_string_buffer( + ctypeobj.value + b'\x00') + self._as_ctype_ptr = ctypes.cast( + self.__as_strbuf, self._ctype) + else: + self._as_ctype_ptr = ctypes.pointer(ctypeobj) + self._address = ctypes.cast(self._as_ctype_ptr, + ctypes.c_void_p).value + self._own = True + + def __add__(self, other): + if isinstance(other, (int, long)): + return self._new_pointer_at(self._address + + other * self._bitem_size) + else: + return NotImplemented + + def __sub__(self, other): + if isinstance(other, (int, long)): + return self._new_pointer_at(self._address - + other * self._bitem_size) + elif type(self) is type(other): + return (self._address - other._address) // self._bitem_size + else: + return NotImplemented + + def __getitem__(self, index): + if getattr(self, '_own', False) and index != 0: + raise IndexError + return BItem._from_ctypes(self._as_ctype_ptr[index]) + + def __setitem__(self, index, value): + self._as_ctype_ptr[index] = BItem._to_ctypes(value) + + if kind == 'charp' or kind == 'voidp': + @classmethod + def _arg_to_ctypes(cls, *value): + if value and isinstance(value[0], bytes): + return ctypes.c_char_p(value[0]) + else: + return super(CTypesPtr, cls)._arg_to_ctypes(*value) + + if kind == 'charp' or kind == 'bytep': + def _to_string(self, maxlen): + if maxlen < 0: + maxlen = sys.maxsize + p = ctypes.cast(self._as_ctype_ptr, + ctypes.POINTER(ctypes.c_char)) + n = 0 + while n < maxlen and p[n] != b'\x00': + n += 1 + return b''.join([p[i] for i in range(n)]) + + def _get_own_repr(self): + if getattr(self, '_own', False): + return 'owning %d bytes' % ( + ctypes.sizeof(self._as_ctype_ptr.contents),) + return super(CTypesPtr, self)._get_own_repr() + # + if (BItem is self.ffi._get_cached_btype(model.void_type) or + BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))): + CTypesPtr._automatic_casts = True + # + CTypesPtr._fix_class() + return CTypesPtr + + def new_array_type(self, CTypesPtr, length): + if length is None: + brackets = ' &[]' + else: + brackets = ' &[%d]' % length + BItem = CTypesPtr._BItem + getbtype = self.ffi._get_cached_btype + if BItem is getbtype(model.PrimitiveType('char')): + kind = 'char' + elif BItem in (getbtype(model.PrimitiveType('signed char')), + getbtype(model.PrimitiveType('unsigned char'))): + kind = 'byte' + else: + kind = 'generic' + # + class CTypesArray(CTypesGenericArray): + __slots__ = ['_blob', '_own'] + if length is not None: + _ctype = BItem._ctype * length + else: + __slots__.append('_ctype') + _reftypename = BItem._get_c_name(brackets) + _declared_length = length + _CTPtr = CTypesPtr + + def __init__(self, init): + if length is None: + if isinstance(init, (int, long)): + len1 = init + init = None + elif kind == 'char' and isinstance(init, bytes): + len1 = len(init) + 1 # extra null + else: + init = tuple(init) + len1 = len(init) + self._ctype = BItem._ctype * len1 + self._blob = self._ctype() + self._own = True + if init is not None: + self._initialize(self._blob, init) + + @staticmethod + def _initialize(blob, init): + if isinstance(init, bytes): + init = [init[i:i+1] for i in range(len(init))] + else: + if isinstance(init, CTypesGenericArray): + if (len(init) != len(blob) or + not isinstance(init, CTypesArray)): + raise TypeError("length/type mismatch: %s" % (init,)) + init = tuple(init) + if len(init) > len(blob): + raise IndexError("too many initializers") + addr = ctypes.cast(blob, ctypes.c_void_p).value + PTR = ctypes.POINTER(BItem._ctype) + itemsize = ctypes.sizeof(BItem._ctype) + for i, value in enumerate(init): + p = ctypes.cast(addr + i * itemsize, PTR) + BItem._initialize(p.contents, value) + + def __len__(self): + return len(self._blob) + + def __getitem__(self, index): + if not (0 <= index < len(self._blob)): + raise IndexError + return BItem._from_ctypes(self._blob[index]) + + def __setitem__(self, index, value): + if not (0 <= index < len(self._blob)): + raise IndexError + self._blob[index] = BItem._to_ctypes(value) + + if kind == 'char' or kind == 'byte': + def _to_string(self, maxlen): + if maxlen < 0: + maxlen = len(self._blob) + p = ctypes.cast(self._blob, + ctypes.POINTER(ctypes.c_char)) + n = 0 + while n < maxlen and p[n] != b'\x00': + n += 1 + return b''.join([p[i] for i in range(n)]) + + def _get_own_repr(self): + if getattr(self, '_own', False): + return 'owning %d bytes' % (ctypes.sizeof(self._blob),) + return super(CTypesArray, self)._get_own_repr() + + def _convert_to_address(self, BClass): + if BClass in (CTypesPtr, None) or BClass._automatic_casts: + return ctypes.addressof(self._blob) + else: + return CTypesData._convert_to_address(self, BClass) + + @staticmethod + def _from_ctypes(ctypes_array): + self = CTypesArray.__new__(CTypesArray) + self._blob = ctypes_array + return self + + @staticmethod + def _arg_to_ctypes(value): + return CTypesPtr._arg_to_ctypes(value) + + def __add__(self, other): + if isinstance(other, (int, long)): + return CTypesPtr._new_pointer_at( + ctypes.addressof(self._blob) + + other * ctypes.sizeof(BItem._ctype)) + else: + return NotImplemented + + @classmethod + def _cast_from(cls, source): + raise NotImplementedError("casting to %r" % ( + cls._get_c_name(),)) + # + CTypesArray._fix_class() + return CTypesArray + + def _new_struct_or_union(self, kind, name, base_ctypes_class): + # + class struct_or_union(base_ctypes_class): + pass + struct_or_union.__name__ = '%s_%s' % (kind, name) + kind1 = kind + # + class CTypesStructOrUnion(CTypesBaseStructOrUnion): + __slots__ = ['_blob'] + _ctype = struct_or_union + _reftypename = '%s &' % (name,) + _kind = kind = kind1 + # + CTypesStructOrUnion._fix_class() + return CTypesStructOrUnion + + def new_struct_type(self, name): + return self._new_struct_or_union('struct', name, ctypes.Structure) + + def new_union_type(self, name): + return self._new_struct_or_union('union', name, ctypes.Union) + + def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp, + totalsize=-1, totalalignment=-1, sflags=0, + pack=0): + if totalsize >= 0 or totalalignment >= 0: + raise NotImplementedError("the ctypes backend of CFFI does not support " + "structures completed by verify(); please " + "compile and install the _cffi_backend module.") + struct_or_union = CTypesStructOrUnion._ctype + fnames = [fname for (fname, BField, bitsize) in fields] + btypes = [BField for (fname, BField, bitsize) in fields] + bitfields = [bitsize for (fname, BField, bitsize) in fields] + # + bfield_types = {} + cfields = [] + for (fname, BField, bitsize) in fields: + if bitsize < 0: + cfields.append((fname, BField._ctype)) + bfield_types[fname] = BField + else: + cfields.append((fname, BField._ctype, bitsize)) + bfield_types[fname] = Ellipsis + if sflags & 8: + struct_or_union._pack_ = 1 + elif pack: + struct_or_union._pack_ = pack + struct_or_union._fields_ = cfields + CTypesStructOrUnion._bfield_types = bfield_types + # + @staticmethod + def _create_ctype_obj(init): + result = struct_or_union() + if init is not None: + initialize(result, init) + return result + CTypesStructOrUnion._create_ctype_obj = _create_ctype_obj + # + def initialize(blob, init): + if is_union: + if len(init) > 1: + raise ValueError("union initializer: %d items given, but " + "only one supported (use a dict if needed)" + % (len(init),)) + if not isinstance(init, dict): + if isinstance(init, (bytes, unicode)): + raise TypeError("union initializer: got a str") + init = tuple(init) + if len(init) > len(fnames): + raise ValueError("too many values for %s initializer" % + CTypesStructOrUnion._get_c_name()) + init = dict(zip(fnames, init)) + addr = ctypes.addressof(blob) + for fname, value in init.items(): + BField, bitsize = name2fieldtype[fname] + assert bitsize < 0, \ + "not implemented: initializer with bit fields" + offset = CTypesStructOrUnion._offsetof(fname) + PTR = ctypes.POINTER(BField._ctype) + p = ctypes.cast(addr + offset, PTR) + BField._initialize(p.contents, value) + is_union = CTypesStructOrUnion._kind == 'union' + name2fieldtype = dict(zip(fnames, zip(btypes, bitfields))) + # + for fname, BField, bitsize in fields: + if fname == '': + raise NotImplementedError("nested anonymous structs/unions") + if hasattr(CTypesStructOrUnion, fname): + raise ValueError("the field name %r conflicts in " + "the ctypes backend" % fname) + if bitsize < 0: + def getter(self, fname=fname, BField=BField, + offset=CTypesStructOrUnion._offsetof(fname), + PTR=ctypes.POINTER(BField._ctype)): + addr = ctypes.addressof(self._blob) + p = ctypes.cast(addr + offset, PTR) + return BField._from_ctypes(p.contents) + def setter(self, value, fname=fname, BField=BField): + setattr(self._blob, fname, BField._to_ctypes(value)) + # + if issubclass(BField, CTypesGenericArray): + setter = None + if BField._declared_length == 0: + def getter(self, fname=fname, BFieldPtr=BField._CTPtr, + offset=CTypesStructOrUnion._offsetof(fname), + PTR=ctypes.POINTER(BField._ctype)): + addr = ctypes.addressof(self._blob) + p = ctypes.cast(addr + offset, PTR) + return BFieldPtr._from_ctypes(p) + # + else: + def getter(self, fname=fname, BField=BField): + return BField._from_ctypes(getattr(self._blob, fname)) + def setter(self, value, fname=fname, BField=BField): + # xxx obscure workaround + value = BField._to_ctypes(value) + oldvalue = getattr(self._blob, fname) + setattr(self._blob, fname, value) + if value != getattr(self._blob, fname): + setattr(self._blob, fname, oldvalue) + raise OverflowError("value too large for bitfield") + setattr(CTypesStructOrUnion, fname, property(getter, setter)) + # + CTypesPtr = self.ffi._get_cached_btype(model.PointerType(tp)) + for fname in fnames: + if hasattr(CTypesPtr, fname): + raise ValueError("the field name %r conflicts in " + "the ctypes backend" % fname) + def getter(self, fname=fname): + return getattr(self[0], fname) + def setter(self, value, fname=fname): + setattr(self[0], fname, value) + setattr(CTypesPtr, fname, property(getter, setter)) + + def new_function_type(self, BArgs, BResult, has_varargs): + nameargs = [BArg._get_c_name() for BArg in BArgs] + if has_varargs: + nameargs.append('...') + nameargs = ', '.join(nameargs) + # + class CTypesFunctionPtr(CTypesGenericPtr): + __slots__ = ['_own_callback', '_name'] + _ctype = ctypes.CFUNCTYPE(getattr(BResult, '_ctype', None), + *[BArg._ctype for BArg in BArgs], + use_errno=True) + _reftypename = BResult._get_c_name('(* &)(%s)' % (nameargs,)) + + def __init__(self, init, error=None): + # create a callback to the Python callable init() + import traceback + assert not has_varargs, "varargs not supported for callbacks" + if getattr(BResult, '_ctype', None) is not None: + error = BResult._from_ctypes( + BResult._create_ctype_obj(error)) + else: + error = None + def callback(*args): + args2 = [] + for arg, BArg in zip(args, BArgs): + args2.append(BArg._from_ctypes(arg)) + try: + res2 = init(*args2) + res2 = BResult._to_ctypes(res2) + except: + traceback.print_exc() + res2 = error + if issubclass(BResult, CTypesGenericPtr): + if res2: + res2 = ctypes.cast(res2, ctypes.c_void_p).value + # .value: http://bugs.python.org/issue1574593 + else: + res2 = None + #print repr(res2) + return res2 + if issubclass(BResult, CTypesGenericPtr): + # The only pointers callbacks can return are void*s: + # http://bugs.python.org/issue5710 + callback_ctype = ctypes.CFUNCTYPE( + ctypes.c_void_p, + *[BArg._ctype for BArg in BArgs], + use_errno=True) + else: + callback_ctype = CTypesFunctionPtr._ctype + self._as_ctype_ptr = callback_ctype(callback) + self._address = ctypes.cast(self._as_ctype_ptr, + ctypes.c_void_p).value + self._own_callback = init + + @staticmethod + def _initialize(ctypes_ptr, value): + if value: + raise NotImplementedError("ctypes backend: not supported: " + "initializers for function pointers") + + def __repr__(self): + c_name = getattr(self, '_name', None) + if c_name: + i = self._reftypename.index('(* &)') + if self._reftypename[i-1] not in ' )*': + c_name = ' ' + c_name + c_name = self._reftypename.replace('(* &)', c_name) + return CTypesData.__repr__(self, c_name) + + def _get_own_repr(self): + if getattr(self, '_own_callback', None) is not None: + return 'calling %r' % (self._own_callback,) + return super(CTypesFunctionPtr, self)._get_own_repr() + + def __call__(self, *args): + if has_varargs: + assert len(args) >= len(BArgs) + extraargs = args[len(BArgs):] + args = args[:len(BArgs)] + else: + assert len(args) == len(BArgs) + ctypes_args = [] + for arg, BArg in zip(args, BArgs): + ctypes_args.append(BArg._arg_to_ctypes(arg)) + if has_varargs: + for i, arg in enumerate(extraargs): + if arg is None: + ctypes_args.append(ctypes.c_void_p(0)) # NULL + continue + if not isinstance(arg, CTypesData): + raise TypeError( + "argument %d passed in the variadic part " + "needs to be a cdata object (got %s)" % + (1 + len(BArgs) + i, type(arg).__name__)) + ctypes_args.append(arg._arg_to_ctypes(arg)) + result = self._as_ctype_ptr(*ctypes_args) + return BResult._from_ctypes(result) + # + CTypesFunctionPtr._fix_class() + return CTypesFunctionPtr + + def new_enum_type(self, name, enumerators, enumvalues, CTypesInt): + assert isinstance(name, str) + reverse_mapping = dict(zip(reversed(enumvalues), + reversed(enumerators))) + # + class CTypesEnum(CTypesInt): + __slots__ = [] + _reftypename = '%s &' % name + + def _get_own_repr(self): + value = self._value + try: + return '%d: %s' % (value, reverse_mapping[value]) + except KeyError: + return str(value) + + def _to_string(self, maxlen): + value = self._value + try: + return reverse_mapping[value] + except KeyError: + return str(value) + # + CTypesEnum._fix_class() + return CTypesEnum + + def get_errno(self): + return ctypes.get_errno() + + def set_errno(self, value): + ctypes.set_errno(value) + + def string(self, b, maxlen=-1): + return b._to_string(maxlen) + + def buffer(self, bptr, size=-1): + raise NotImplementedError("buffer() with ctypes backend") + + def sizeof(self, cdata_or_BType): + if isinstance(cdata_or_BType, CTypesData): + return cdata_or_BType._get_size_of_instance() + else: + assert issubclass(cdata_or_BType, CTypesData) + return cdata_or_BType._get_size() + + def alignof(self, BType): + assert issubclass(BType, CTypesData) + return BType._alignment() + + def newp(self, BType, source): + if not issubclass(BType, CTypesData): + raise TypeError + return BType._newp(source) + + def cast(self, BType, source): + return BType._cast_from(source) + + def callback(self, BType, source, error, onerror): + assert onerror is None # XXX not implemented + return BType(source, error) + + _weakref_cache_ref = None + + def gcp(self, cdata, destructor, size=0): + if self._weakref_cache_ref is None: + import weakref + class MyRef(weakref.ref): + def __eq__(self, other): + myref = self() + return self is other or ( + myref is not None and myref is other()) + def __ne__(self, other): + return not (self == other) + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self()) + return self._hash + self._weakref_cache_ref = {}, MyRef + weak_cache, MyRef = self._weakref_cache_ref + + if destructor is None: + try: + del weak_cache[MyRef(cdata)] + except KeyError: + raise TypeError("Can remove destructor only on a object " + "previously returned by ffi.gc()") + return None + + def remove(k): + cdata, destructor = weak_cache.pop(k, (None, None)) + if destructor is not None: + destructor(cdata) + + new_cdata = self.cast(self.typeof(cdata), cdata) + assert new_cdata is not cdata + weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor) + return new_cdata + + typeof = type + + def getcname(self, BType, replace_with): + return BType._get_c_name(replace_with) + + def typeoffsetof(self, BType, fieldname, num=0): + if isinstance(fieldname, str): + if num == 0 and issubclass(BType, CTypesGenericPtr): + BType = BType._BItem + if not issubclass(BType, CTypesBaseStructOrUnion): + raise TypeError("expected a struct or union ctype") + BField = BType._bfield_types[fieldname] + if BField is Ellipsis: + raise TypeError("not supported for bitfields") + return (BField, BType._offsetof(fieldname)) + elif isinstance(fieldname, (int, long)): + if issubclass(BType, CTypesGenericArray): + BType = BType._CTPtr + if not issubclass(BType, CTypesGenericPtr): + raise TypeError("expected an array or ptr ctype") + BItem = BType._BItem + offset = BItem._get_size() * fieldname + if offset > sys.maxsize: + raise OverflowError + return (BItem, offset) + else: + raise TypeError(type(fieldname)) + + def rawaddressof(self, BTypePtr, cdata, offset=None): + if isinstance(cdata, CTypesBaseStructOrUnion): + ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata)) + elif isinstance(cdata, CTypesGenericPtr): + if offset is None or not issubclass(type(cdata)._BItem, + CTypesBaseStructOrUnion): + raise TypeError("unexpected cdata type") + ptr = type(cdata)._to_ctypes(cdata) + elif isinstance(cdata, CTypesGenericArray): + ptr = type(cdata)._to_ctypes(cdata) + else: + raise TypeError("expected a ") + if offset: + ptr = ctypes.cast( + ctypes.c_void_p( + ctypes.cast(ptr, ctypes.c_void_p).value + offset), + type(ptr)) + return BTypePtr._from_ctypes(ptr) + + +class CTypesLibrary(object): + + def __init__(self, backend, cdll): + self.backend = backend + self.cdll = cdll + + def load_function(self, BType, name): + c_func = getattr(self.cdll, name) + funcobj = BType._from_ctypes(c_func) + funcobj._name = name + return funcobj + + def read_variable(self, BType, name): + try: + ctypes_obj = BType._ctype.in_dll(self.cdll, name) + except AttributeError as e: + raise NotImplementedError(e) + return BType._from_ctypes(ctypes_obj) + + def write_variable(self, BType, name, value): + new_ctypes_obj = BType._to_ctypes(value) + ctypes_obj = BType._ctype.in_dll(self.cdll, name) + ctypes.memmove(ctypes.addressof(ctypes_obj), + ctypes.addressof(new_ctypes_obj), + ctypes.sizeof(BType._ctype)) diff --git a/lib/cffi/cffi_opcode.py b/lib/cffi/cffi_opcode.py new file mode 100644 index 0000000..6421df6 --- /dev/null +++ b/lib/cffi/cffi_opcode.py @@ -0,0 +1,187 @@ +from .error import VerificationError + +class CffiOp(object): + def __init__(self, op, arg): + self.op = op + self.arg = arg + + def as_c_expr(self): + if self.op is None: + assert isinstance(self.arg, str) + return '(_cffi_opcode_t)(%s)' % (self.arg,) + classname = CLASS_NAME[self.op] + return '_CFFI_OP(_CFFI_OP_%s, %s)' % (classname, self.arg) + + def as_python_bytes(self): + if self.op is None and self.arg.isdigit(): + value = int(self.arg) # non-negative: '-' not in self.arg + if value >= 2**31: + raise OverflowError("cannot emit %r: limited to 2**31-1" + % (self.arg,)) + return format_four_bytes(value) + if isinstance(self.arg, str): + raise VerificationError("cannot emit to Python: %r" % (self.arg,)) + return format_four_bytes((self.arg << 8) | self.op) + + def __str__(self): + classname = CLASS_NAME.get(self.op, self.op) + return '(%s %s)' % (classname, self.arg) + +def format_four_bytes(num): + return '\\x%02X\\x%02X\\x%02X\\x%02X' % ( + (num >> 24) & 0xFF, + (num >> 16) & 0xFF, + (num >> 8) & 0xFF, + (num ) & 0xFF) + +OP_PRIMITIVE = 1 +OP_POINTER = 3 +OP_ARRAY = 5 +OP_OPEN_ARRAY = 7 +OP_STRUCT_UNION = 9 +OP_ENUM = 11 +OP_FUNCTION = 13 +OP_FUNCTION_END = 15 +OP_NOOP = 17 +OP_BITFIELD = 19 +OP_TYPENAME = 21 +OP_CPYTHON_BLTN_V = 23 # varargs +OP_CPYTHON_BLTN_N = 25 # noargs +OP_CPYTHON_BLTN_O = 27 # O (i.e. a single arg) +OP_CONSTANT = 29 +OP_CONSTANT_INT = 31 +OP_GLOBAL_VAR = 33 +OP_DLOPEN_FUNC = 35 +OP_DLOPEN_CONST = 37 +OP_GLOBAL_VAR_F = 39 +OP_EXTERN_PYTHON = 41 + +PRIM_VOID = 0 +PRIM_BOOL = 1 +PRIM_CHAR = 2 +PRIM_SCHAR = 3 +PRIM_UCHAR = 4 +PRIM_SHORT = 5 +PRIM_USHORT = 6 +PRIM_INT = 7 +PRIM_UINT = 8 +PRIM_LONG = 9 +PRIM_ULONG = 10 +PRIM_LONGLONG = 11 +PRIM_ULONGLONG = 12 +PRIM_FLOAT = 13 +PRIM_DOUBLE = 14 +PRIM_LONGDOUBLE = 15 + +PRIM_WCHAR = 16 +PRIM_INT8 = 17 +PRIM_UINT8 = 18 +PRIM_INT16 = 19 +PRIM_UINT16 = 20 +PRIM_INT32 = 21 +PRIM_UINT32 = 22 +PRIM_INT64 = 23 +PRIM_UINT64 = 24 +PRIM_INTPTR = 25 +PRIM_UINTPTR = 26 +PRIM_PTRDIFF = 27 +PRIM_SIZE = 28 +PRIM_SSIZE = 29 +PRIM_INT_LEAST8 = 30 +PRIM_UINT_LEAST8 = 31 +PRIM_INT_LEAST16 = 32 +PRIM_UINT_LEAST16 = 33 +PRIM_INT_LEAST32 = 34 +PRIM_UINT_LEAST32 = 35 +PRIM_INT_LEAST64 = 36 +PRIM_UINT_LEAST64 = 37 +PRIM_INT_FAST8 = 38 +PRIM_UINT_FAST8 = 39 +PRIM_INT_FAST16 = 40 +PRIM_UINT_FAST16 = 41 +PRIM_INT_FAST32 = 42 +PRIM_UINT_FAST32 = 43 +PRIM_INT_FAST64 = 44 +PRIM_UINT_FAST64 = 45 +PRIM_INTMAX = 46 +PRIM_UINTMAX = 47 +PRIM_FLOATCOMPLEX = 48 +PRIM_DOUBLECOMPLEX = 49 +PRIM_CHAR16 = 50 +PRIM_CHAR32 = 51 + +_NUM_PRIM = 52 +_UNKNOWN_PRIM = -1 +_UNKNOWN_FLOAT_PRIM = -2 +_UNKNOWN_LONG_DOUBLE = -3 + +_IO_FILE_STRUCT = -1 + +PRIMITIVE_TO_INDEX = { + 'char': PRIM_CHAR, + 'short': PRIM_SHORT, + 'int': PRIM_INT, + 'long': PRIM_LONG, + 'long long': PRIM_LONGLONG, + 'signed char': PRIM_SCHAR, + 'unsigned char': PRIM_UCHAR, + 'unsigned short': PRIM_USHORT, + 'unsigned int': PRIM_UINT, + 'unsigned long': PRIM_ULONG, + 'unsigned long long': PRIM_ULONGLONG, + 'float': PRIM_FLOAT, + 'double': PRIM_DOUBLE, + 'long double': PRIM_LONGDOUBLE, + '_cffi_float_complex_t': PRIM_FLOATCOMPLEX, + '_cffi_double_complex_t': PRIM_DOUBLECOMPLEX, + '_Bool': PRIM_BOOL, + 'wchar_t': PRIM_WCHAR, + 'char16_t': PRIM_CHAR16, + 'char32_t': PRIM_CHAR32, + 'int8_t': PRIM_INT8, + 'uint8_t': PRIM_UINT8, + 'int16_t': PRIM_INT16, + 'uint16_t': PRIM_UINT16, + 'int32_t': PRIM_INT32, + 'uint32_t': PRIM_UINT32, + 'int64_t': PRIM_INT64, + 'uint64_t': PRIM_UINT64, + 'intptr_t': PRIM_INTPTR, + 'uintptr_t': PRIM_UINTPTR, + 'ptrdiff_t': PRIM_PTRDIFF, + 'size_t': PRIM_SIZE, + 'ssize_t': PRIM_SSIZE, + 'int_least8_t': PRIM_INT_LEAST8, + 'uint_least8_t': PRIM_UINT_LEAST8, + 'int_least16_t': PRIM_INT_LEAST16, + 'uint_least16_t': PRIM_UINT_LEAST16, + 'int_least32_t': PRIM_INT_LEAST32, + 'uint_least32_t': PRIM_UINT_LEAST32, + 'int_least64_t': PRIM_INT_LEAST64, + 'uint_least64_t': PRIM_UINT_LEAST64, + 'int_fast8_t': PRIM_INT_FAST8, + 'uint_fast8_t': PRIM_UINT_FAST8, + 'int_fast16_t': PRIM_INT_FAST16, + 'uint_fast16_t': PRIM_UINT_FAST16, + 'int_fast32_t': PRIM_INT_FAST32, + 'uint_fast32_t': PRIM_UINT_FAST32, + 'int_fast64_t': PRIM_INT_FAST64, + 'uint_fast64_t': PRIM_UINT_FAST64, + 'intmax_t': PRIM_INTMAX, + 'uintmax_t': PRIM_UINTMAX, + } + +F_UNION = 0x01 +F_CHECK_FIELDS = 0x02 +F_PACKED = 0x04 +F_EXTERNAL = 0x08 +F_OPAQUE = 0x10 + +G_FLAGS = dict([('_CFFI_' + _key, globals()[_key]) + for _key in ['F_UNION', 'F_CHECK_FIELDS', 'F_PACKED', + 'F_EXTERNAL', 'F_OPAQUE']]) + +CLASS_NAME = {} +for _name, _value in list(globals().items()): + if _name.startswith('OP_') and isinstance(_value, int): + CLASS_NAME[_value] = _name[3:] diff --git a/lib/cffi/commontypes.py b/lib/cffi/commontypes.py new file mode 100644 index 0000000..d4dae35 --- /dev/null +++ b/lib/cffi/commontypes.py @@ -0,0 +1,82 @@ +import sys +from . import model +from .error import FFIError + + +COMMON_TYPES = {} + +try: + # fetch "bool" and all simple Windows types + from _cffi_backend import _get_common_types + _get_common_types(COMMON_TYPES) +except ImportError: + pass + +COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE') +COMMON_TYPES['bool'] = '_Bool' # in case we got ImportError above +COMMON_TYPES['float _Complex'] = '_cffi_float_complex_t' +COMMON_TYPES['double _Complex'] = '_cffi_double_complex_t' + +for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + if _type.endswith('_t'): + COMMON_TYPES[_type] = _type +del _type + +_CACHE = {} + +def resolve_common_type(parser, commontype): + try: + return _CACHE[commontype] + except KeyError: + cdecl = COMMON_TYPES.get(commontype, commontype) + if not isinstance(cdecl, str): + result, quals = cdecl, 0 # cdecl is already a BaseType + elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + result, quals = model.PrimitiveType(cdecl), 0 + elif cdecl == 'set-unicode-needed': + raise FFIError("The Windows type %r is only available after " + "you call ffi.set_unicode()" % (commontype,)) + else: + if commontype == cdecl: + raise FFIError( + "Unsupported type: %r. Please look at " + "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " + "and file an issue if you think this type should really " + "be supported." % (commontype,)) + result, quals = parser.parse_type_and_quals(cdecl) # recursive + + assert isinstance(result, model.BaseTypeByIdentity) + _CACHE[commontype] = result, quals + return result, quals + + +# ____________________________________________________________ +# extra types for Windows (most of them are in commontypes.c) + + +def win_common_types(): + return { + "UNICODE_STRING": model.StructType( + "_UNICODE_STRING", + ["Length", + "MaximumLength", + "Buffer"], + [model.PrimitiveType("unsigned short"), + model.PrimitiveType("unsigned short"), + model.PointerType(model.PrimitiveType("wchar_t"))], + [-1, -1, -1]), + "PUNICODE_STRING": "UNICODE_STRING *", + "PCUNICODE_STRING": "const UNICODE_STRING *", + + "TBYTE": "set-unicode-needed", + "TCHAR": "set-unicode-needed", + "LPCTSTR": "set-unicode-needed", + "PCTSTR": "set-unicode-needed", + "LPTSTR": "set-unicode-needed", + "PTSTR": "set-unicode-needed", + "PTBYTE": "set-unicode-needed", + "PTCHAR": "set-unicode-needed", + } + +if sys.platform == 'win32': + COMMON_TYPES.update(win_common_types()) diff --git a/lib/cffi/cparser.py b/lib/cffi/cparser.py new file mode 100644 index 0000000..dd590d8 --- /dev/null +++ b/lib/cffi/cparser.py @@ -0,0 +1,1015 @@ +from . import model +from .commontypes import COMMON_TYPES, resolve_common_type +from .error import FFIError, CDefError +try: + from . import _pycparser as pycparser +except ImportError: + import pycparser +import weakref, re, sys + +try: + if sys.version_info < (3,): + import thread as _thread + else: + import _thread + lock = _thread.allocate_lock() +except ImportError: + lock = None + +def _workaround_for_static_import_finders(): + # Issue #392: packaging tools like cx_Freeze can not find these + # because pycparser uses exec dynamic import. This is an obscure + # workaround. This function is never called. + import pycparser.yacctab + import pycparser.lextab + +CDEF_SOURCE_STRING = "" +_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", + re.DOTALL | re.MULTILINE) +_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" + r"\b((?:[^\n\\]|\\.)*?)$", + re.DOTALL | re.MULTILINE) +_r_line_directive = re.compile(r"^[ \t]*#[ \t]*(?:line|\d+)\b.*$", re.MULTILINE) +_r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}") +_r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$") +_r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") +_r_words = re.compile(r"\w+|\S") +_parser_cache = None +_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) +_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") +_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") +_r_cdecl = re.compile(r"\b__cdecl\b") +_r_extern_python = re.compile(r'\bextern\s*"' + r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') +_r_star_const_space = re.compile( # matches "* const " + r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") + +def _get_parser(): + global _parser_cache + if _parser_cache is None: + _parser_cache = pycparser.CParser() + return _parser_cache + +def _workaround_for_old_pycparser(csource): + # Workaround for a pycparser issue (fixed between pycparser 2.10 and + # 2.14): "char*const***" gives us a wrong syntax tree, the same as + # for "char***(*const)". This means we can't tell the difference + # afterwards. But "char(*const(***))" gives us the right syntax + # tree. The issue only occurs if there are several stars in + # sequence with no parenthesis in between, just possibly qualifiers. + # Attempt to fix it by adding some parentheses in the source: each + # time we see "* const" or "* const *", we add an opening + # parenthesis before each star---the hard part is figuring out where + # to close them. + parts = [] + while True: + match = _r_star_const_space.search(csource) + if not match: + break + #print repr(''.join(parts)+csource), '=>', + parts.append(csource[:match.start()]) + parts.append('('); closing = ')' + parts.append(match.group()) # e.g. "* const " + endpos = match.end() + if csource.startswith('*', endpos): + parts.append('('); closing += ')' + level = 0 + i = endpos + while i < len(csource): + c = csource[i] + if c == '(': + level += 1 + elif c == ')': + if level == 0: + break + level -= 1 + elif c in ',;=': + if level == 0: + break + i += 1 + csource = csource[endpos:i] + closing + csource[i:] + #print repr(''.join(parts)+csource) + parts.append(csource) + return ''.join(parts) + +def _preprocess_extern_python(csource): + # input: `extern "Python" int foo(int);` or + # `extern "Python" { int foo(int); }` + # output: + # void __cffi_extern_python_start; + # int foo(int); + # void __cffi_extern_python_stop; + # + # input: `extern "Python+C" int foo(int);` + # output: + # void __cffi_extern_python_plus_c_start; + # int foo(int); + # void __cffi_extern_python_stop; + parts = [] + while True: + match = _r_extern_python.search(csource) + if not match: + break + endpos = match.end() - 1 + #print + #print ''.join(parts)+csource + #print '=>' + parts.append(csource[:match.start()]) + if 'C' in match.group(1): + parts.append('void __cffi_extern_python_plus_c_start; ') + else: + parts.append('void __cffi_extern_python_start; ') + if csource[endpos] == '{': + # grouping variant + closing = csource.find('}', endpos) + if closing < 0: + raise CDefError("'extern \"Python\" {': no '}' found") + if csource.find('{', endpos + 1, closing) >= 0: + raise NotImplementedError("cannot use { } inside a block " + "'extern \"Python\" { ... }'") + parts.append(csource[endpos+1:closing]) + csource = csource[closing+1:] + else: + # non-grouping variant + semicolon = csource.find(';', endpos) + if semicolon < 0: + raise CDefError("'extern \"Python\": no ';' found") + parts.append(csource[endpos:semicolon+1]) + csource = csource[semicolon+1:] + parts.append(' void __cffi_extern_python_stop;') + #print ''.join(parts)+csource + #print + parts.append(csource) + return ''.join(parts) + +def _warn_for_string_literal(csource): + if '"' not in csource: + return + for line in csource.splitlines(): + if '"' in line and not line.lstrip().startswith('#'): + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + break + +def _warn_for_non_extern_non_static_global_variable(decl): + if not decl.storage: + import warnings + warnings.warn("Global variable '%s' in cdef(): for consistency " + "with C it should have a storage class specifier " + "(usually 'extern')" % (decl.name,)) + +def _remove_line_directives(csource): + # _r_line_directive matches whole lines, without the final \n, if they + # start with '#line' with some spacing allowed, or '#NUMBER'. This + # function stores them away and replaces them with exactly the string + # '#line@N', where N is the index in the list 'line_directives'. + line_directives = [] + def replace(m): + i = len(line_directives) + line_directives.append(m.group()) + return '#line@%d' % i + csource = _r_line_directive.sub(replace, csource) + return csource, line_directives + +def _put_back_line_directives(csource, line_directives): + def replace(m): + s = m.group() + if not s.startswith('#line@'): + raise AssertionError("unexpected #line directive " + "(should have been processed and removed") + return line_directives[int(s[6:])] + return _r_line_directive.sub(replace, csource) + +def _preprocess(csource): + # First, remove the lines of the form '#line N "filename"' because + # the "filename" part could confuse the rest + csource, line_directives = _remove_line_directives(csource) + # Remove comments. NOTE: this only work because the cdef() section + # should not contain any string literals (except in line directives)! + def replace_keeping_newlines(m): + return ' ' + m.group().count('\n') * '\n' + csource = _r_comment.sub(replace_keeping_newlines, csource) + # Remove the "#define FOO x" lines + macros = {} + for match in _r_define.finditer(csource): + macroname, macrovalue = match.groups() + macrovalue = macrovalue.replace('\\\n', '').strip() + macros[macroname] = macrovalue + csource = _r_define.sub('', csource) + # + if pycparser.__version__ < '2.14': + csource = _workaround_for_old_pycparser(csource) + # + # BIG HACK: replace WINAPI or __stdcall with "volatile const". + # It doesn't make sense for the return type of a function to be + # "volatile volatile const", so we abuse it to detect __stdcall... + # Hack number 2 is that "int(volatile *fptr)();" is not valid C + # syntax, so we place the "volatile" before the opening parenthesis. + csource = _r_stdcall2.sub(' volatile volatile const(', csource) + csource = _r_stdcall1.sub(' volatile volatile const ', csource) + csource = _r_cdecl.sub(' ', csource) + # + # Replace `extern "Python"` with start/end markers + csource = _preprocess_extern_python(csource) + # + # Now there should not be any string literal left; warn if we get one + _warn_for_string_literal(csource) + # + # Replace "[...]" with "[__dotdotdotarray__]" + csource = _r_partial_array.sub('[__dotdotdotarray__]', csource) + # + # Replace "...}" with "__dotdotdotNUM__}". This construction should + # occur only at the end of enums; at the end of structs we have "...;}" + # and at the end of vararg functions "...);". Also replace "=...[,}]" + # with ",__dotdotdotNUM__[,}]": this occurs in the enums too, when + # giving an unknown value. + matches = list(_r_partial_enum.finditer(csource)) + for number, match in enumerate(reversed(matches)): + p = match.start() + if csource[p] == '=': + p2 = csource.find('...', p, match.end()) + assert p2 > p + csource = '%s,__dotdotdot%d__ %s' % (csource[:p], number, + csource[p2+3:]) + else: + assert csource[p:p+3] == '...' + csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, + csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) + # Replace all remaining "..." with the same name, "__dotdotdot__", + # which is declared with a typedef for the purpose of C parsing. + csource = csource.replace('...', ' __dotdotdot__ ') + # Finally, put back the line directives + csource = _put_back_line_directives(csource, line_directives) + return csource, macros + +def _common_type_names(csource): + # Look in the source for what looks like usages of types from the + # list of common types. A "usage" is approximated here as the + # appearance of the word, minus a "definition" of the type, which + # is the last word in a "typedef" statement. Approximative only + # but should be fine for all the common types. + look_for_words = set(COMMON_TYPES) + look_for_words.add(';') + look_for_words.add(',') + look_for_words.add('(') + look_for_words.add(')') + look_for_words.add('typedef') + words_used = set() + is_typedef = False + paren = 0 + previous_word = '' + for word in _r_words.findall(csource): + if word in look_for_words: + if word == ';': + if is_typedef: + words_used.discard(previous_word) + look_for_words.discard(previous_word) + is_typedef = False + elif word == 'typedef': + is_typedef = True + paren = 0 + elif word == '(': + paren += 1 + elif word == ')': + paren -= 1 + elif word == ',': + if is_typedef and paren == 0: + words_used.discard(previous_word) + look_for_words.discard(previous_word) + else: # word in COMMON_TYPES + words_used.add(word) + previous_word = word + return words_used + + +class Parser(object): + + def __init__(self): + self._declarations = {} + self._included_declarations = set() + self._anonymous_counter = 0 + self._structnode2type = weakref.WeakKeyDictionary() + self._options = {} + self._int_constants = {} + self._recomplete = [] + self._uses_new_feature = None + + def _parse(self, csource): + csource, macros = _preprocess(csource) + # XXX: for more efficiency we would need to poke into the + # internals of CParser... the following registers the + # typedefs, because their presence or absence influences the + # parsing itself (but what they are typedef'ed to plays no role) + ctn = _common_type_names(csource) + typenames = [] + for name in sorted(self._declarations): + if name.startswith('typedef '): + name = name[8:] + typenames.append(name) + ctn.discard(name) + typenames += sorted(ctn) + # + csourcelines = [] + csourcelines.append('# 1 ""') + for typename in typenames: + csourcelines.append('typedef int %s;' % typename) + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') + # this forces pycparser to consider the following in the file + # called from line 1 + csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,)) + csourcelines.append(csource) + csourcelines.append('') # see test_missing_newline_bug + fullcsource = '\n'.join(csourcelines) + if lock is not None: + lock.acquire() # pycparser is not thread-safe... + try: + ast = _get_parser().parse(fullcsource) + except pycparser.c_parser.ParseError as e: + self.convert_pycparser_error(e, csource) + finally: + if lock is not None: + lock.release() + # csource will be used to find buggy source text + return ast, macros, csource + + def _convert_pycparser_error(self, e, csource): + # xxx look for ":NUM:" at the start of str(e) + # and interpret that as a line number. This will not work if + # the user gives explicit ``# NUM "FILE"`` directives. + line = None + msg = str(e) + match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg) + if match: + linenum = int(match.group(1), 10) + csourcelines = csource.splitlines() + if 1 <= linenum <= len(csourcelines): + line = csourcelines[linenum-1] + return line + + def convert_pycparser_error(self, e, csource): + line = self._convert_pycparser_error(e, csource) + + msg = str(e) + if line: + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) + else: + msg = 'parse error\n%s' % (msg,) + raise CDefError(msg) + + def parse(self, csource, override=False, packed=False, pack=None, + dllexport=False): + if packed: + if packed != True: + raise ValueError("'packed' should be False or True; use " + "'pack' to give another value") + if pack: + raise ValueError("cannot give both 'pack' and 'packed'") + pack = 1 + elif pack: + if pack & (pack - 1): + raise ValueError("'pack' must be a power of two, not %r" % + (pack,)) + else: + pack = 0 + prev_options = self._options + try: + self._options = {'override': override, + 'packed': pack, + 'dllexport': dllexport} + self._internal_parse(csource) + finally: + self._options = prev_options + + def _internal_parse(self, csource): + ast, macros, csource = self._parse(csource) + # add the macros + self._process_macros(macros) + # find the first "__dotdotdot__" and use that as a separator + # between the repeated typedefs and the real csource + iterator = iter(ast.ext) + for decl in iterator: + if decl.name == '__dotdotdot__': + break + else: + assert 0 + current_decl = None + # + try: + self._inside_extern_python = '__cffi_extern_python_stop' + for decl in iterator: + current_decl = decl + if isinstance(decl, pycparser.c_ast.Decl): + self._parse_decl(decl) + elif isinstance(decl, pycparser.c_ast.Typedef): + if not decl.name: + raise CDefError("typedef does not declare any name", + decl) + quals = 0 + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_type(decl) + elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and + isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and + isinstance(decl.type.type.type, + pycparser.c_ast.IdentifierType) and + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) + else: + realtype, quals = self._get_type_and_quals( + decl.type, name=decl.name, partial_length_ok=True, + typedef_example="*(%s *)0" % (decl.name,)) + self._declare('typedef ' + decl.name, realtype, quals=quals) + elif decl.__class__.__name__ == 'Pragma': + # skip pragma, only in pycparser 2.15 + import warnings + warnings.warn( + "#pragma in cdef() are entirely ignored. " + "They should be removed for now, otherwise your " + "code might behave differently in a future version " + "of CFFI if #pragma support gets added. Note that " + "'#pragma pack' needs to be replaced with the " + "'packed' keyword argument to cdef().") + else: + raise CDefError("unexpected <%s>: this construct is valid " + "C but not valid in cdef()" % + decl.__class__.__name__, decl) + except CDefError as e: + if len(e.args) == 1: + e.args = e.args + (current_decl,) + raise + except FFIError as e: + msg = self._convert_pycparser_error(e, csource) + if msg: + e.args = (e.args[0] + "\n *** Err: %s" % msg,) + raise + + def _add_constants(self, key, val): + if key in self._int_constants: + if self._int_constants[key] == val: + return # ignore identical double declarations + raise FFIError( + "multiple declarations of constant: %s" % (key,)) + self._int_constants[key] = val + + def _add_integer_constant(self, name, int_str): + int_str = int_str.lower().rstrip("ul") + neg = int_str.startswith('-') + if neg: + int_str = int_str[1:] + # "010" is not valid oct in py3 + if (int_str.startswith("0") and int_str != '0' + and not int_str.startswith("0x")): + int_str = "0o" + int_str[1:] + pyvalue = int(int_str, 0) + if neg: + pyvalue = -pyvalue + self._add_constants(name, pyvalue) + self._declare('macro ' + name, pyvalue) + + def _process_macros(self, macros): + for key, value in macros.items(): + value = value.strip() + if _r_int_literal.match(value): + self._add_integer_constant(key, value) + elif value == '...': + self._declare('macro ' + key, value) + else: + raise CDefError( + 'only supports one of the following syntax:\n' + ' #define %s ... (literally dot-dot-dot)\n' + ' #define %s NUMBER (with NUMBER an integer' + ' constant, decimal/hex/octal)\n' + 'got:\n' + ' #define %s %s' + % (key, key, key, value)) + + def _declare_function(self, tp, quals, decl): + tp = self._get_type_pointer(tp, quals) + if self._options.get('dllexport'): + tag = 'dllexport_python ' + elif self._inside_extern_python == '__cffi_extern_python_start': + tag = 'extern_python ' + elif self._inside_extern_python == '__cffi_extern_python_plus_c_start': + tag = 'extern_python_plus_c ' + else: + tag = 'function ' + self._declare(tag + decl.name, tp) + + def _parse_decl(self, decl): + node = decl.type + if isinstance(node, pycparser.c_ast.FuncDecl): + tp, quals = self._get_type_and_quals(node, name=decl.name) + assert isinstance(tp, model.RawFunctionType) + self._declare_function(tp, quals, decl) + else: + if isinstance(node, pycparser.c_ast.Struct): + self._get_struct_union_enum_type('struct', node) + elif isinstance(node, pycparser.c_ast.Union): + self._get_struct_union_enum_type('union', node) + elif isinstance(node, pycparser.c_ast.Enum): + self._get_struct_union_enum_type('enum', node) + elif not decl.name: + raise CDefError("construct does not declare any variable", + decl) + # + if decl.name: + tp, quals = self._get_type_and_quals(node, + partial_length_ok=True) + if tp.is_raw_function: + self._declare_function(tp, quals, decl) + elif (tp.is_integer_type() and + hasattr(decl, 'init') and + hasattr(decl.init, 'value') and + _r_int_literal.match(decl.init.value)): + self._add_integer_constant(decl.name, decl.init.value) + elif (tp.is_integer_type() and + isinstance(decl.init, pycparser.c_ast.UnaryOp) and + decl.init.op == '-' and + hasattr(decl.init.expr, 'value') and + _r_int_literal.match(decl.init.expr.value)): + self._add_integer_constant(decl.name, + '-' + decl.init.expr.value) + elif (tp is model.void_type and + decl.name.startswith('__cffi_extern_python_')): + # hack: `extern "Python"` in the C source is replaced + # with "void __cffi_extern_python_start;" and + # "void __cffi_extern_python_stop;" + self._inside_extern_python = decl.name + else: + if self._inside_extern_python !='__cffi_extern_python_stop': + raise CDefError( + "cannot declare constants or " + "variables with 'extern \"Python\"'") + if (quals & model.Q_CONST) and not tp.is_array_type: + self._declare('constant ' + decl.name, tp, quals=quals) + else: + _warn_for_non_extern_non_static_global_variable(decl) + self._declare('variable ' + decl.name, tp, quals=quals) + + def parse_type(self, cdecl): + return self.parse_type_and_quals(cdecl)[0] + + def parse_type_and_quals(self, cdecl): + ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] + assert not macros + exprnode = ast.ext[-1].type.args.params[0] + if isinstance(exprnode, pycparser.c_ast.ID): + raise CDefError("unknown identifier '%s'" % (exprnode.name,)) + return self._get_type_and_quals(exprnode.type) + + def _declare(self, name, obj, included=False, quals=0): + if name in self._declarations: + prevobj, prevquals = self._declarations[name] + if prevobj is obj and prevquals == quals: + return + if not self._options.get('override'): + raise FFIError( + "multiple declarations of %s (for interactive usage, " + "try cdef(xx, override=True))" % (name,)) + assert '__dotdotdot__' not in name.split() + self._declarations[name] = (obj, quals) + if included: + self._included_declarations.add(obj) + + def _extract_quals(self, type): + quals = 0 + if isinstance(type, (pycparser.c_ast.TypeDecl, + pycparser.c_ast.PtrDecl)): + if 'const' in type.quals: + quals |= model.Q_CONST + if 'volatile' in type.quals: + quals |= model.Q_VOLATILE + if 'restrict' in type.quals: + quals |= model.Q_RESTRICT + return quals + + def _get_type_pointer(self, type, quals, declname=None): + if isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + if (isinstance(type, model.StructOrUnionOrEnum) and + type.name.startswith('$') and type.name[1:].isdigit() and + type.forcename is None and declname is not None): + return model.NamedPointerType(type, declname, quals) + return model.PointerType(type, quals) + + def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False, + typedef_example=None): + # first, dereference typedefs, if we have it already parsed, we're good + if (isinstance(typenode, pycparser.c_ast.TypeDecl) and + isinstance(typenode.type, pycparser.c_ast.IdentifierType) and + len(typenode.type.names) == 1 and + ('typedef ' + typenode.type.names[0]) in self._declarations): + tp, quals = self._declarations['typedef ' + typenode.type.names[0]] + quals |= self._extract_quals(typenode) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.ArrayDecl): + # array type + if typenode.dim is None: + length = None + else: + length = self._parse_constant( + typenode.dim, partial_length_ok=partial_length_ok) + # a hack: in 'typedef int foo_t[...][...];', don't use '...' as + # the length but use directly the C expression that would be + # generated by recompiler.py. This lets the typedef be used in + # many more places within recompiler.py + if typedef_example is not None: + if length == '...': + length = '_cffi_array_len(%s)' % (typedef_example,) + typedef_example = "*" + typedef_example + # + tp, quals = self._get_type_and_quals(typenode.type, + partial_length_ok=partial_length_ok, + typedef_example=typedef_example) + return model.ArrayType(tp, length), quals + # + if isinstance(typenode, pycparser.c_ast.PtrDecl): + # pointer type + itemtype, itemquals = self._get_type_and_quals(typenode.type) + tp = self._get_type_pointer(itemtype, itemquals, declname=name) + quals = self._extract_quals(typenode) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.TypeDecl): + quals = self._extract_quals(typenode) + type = typenode.type + if isinstance(type, pycparser.c_ast.IdentifierType): + # assume a primitive type. get it from .names, but reduce + # synonyms to a single chosen combination + names = list(type.names) + if names != ['signed', 'char']: # keep this unmodified + prefixes = {} + while names: + name = names[0] + if name in ('short', 'long', 'signed', 'unsigned'): + prefixes[name] = prefixes.get(name, 0) + 1 + del names[0] + else: + break + # ignore the 'signed' prefix below, and reorder the others + newnames = [] + for prefix in ('unsigned', 'short', 'long'): + for i in range(prefixes.get(prefix, 0)): + newnames.append(prefix) + if not names: + names = ['int'] # implicitly + if names == ['int']: # but kill it if 'short' or 'long' + if 'short' in prefixes or 'long' in prefixes: + names = [] + names = newnames + names + ident = ' '.join(names) + if ident == 'void': + return model.void_type, quals + if ident == '__dotdotdot__': + raise FFIError(':%d: bad usage of "..."' % + typenode.coord.line) + tp0, quals0 = resolve_common_type(self, ident) + return tp0, (quals | quals0) + # + if isinstance(type, pycparser.c_ast.Struct): + # 'struct foobar' + tp = self._get_struct_union_enum_type('struct', type, name) + return tp, quals + # + if isinstance(type, pycparser.c_ast.Union): + # 'union foobar' + tp = self._get_struct_union_enum_type('union', type, name) + return tp, quals + # + if isinstance(type, pycparser.c_ast.Enum): + # 'enum foobar' + tp = self._get_struct_union_enum_type('enum', type, name) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.FuncDecl): + # a function type + return self._parse_function_type(typenode, name), 0 + # + # nested anonymous structs or unions end up here + if isinstance(typenode, pycparser.c_ast.Struct): + return self._get_struct_union_enum_type('struct', typenode, name, + nested=True), 0 + if isinstance(typenode, pycparser.c_ast.Union): + return self._get_struct_union_enum_type('union', typenode, name, + nested=True), 0 + # + raise FFIError(":%d: bad or unsupported type declaration" % + typenode.coord.line) + + def _parse_function_type(self, typenode, funcname=None): + params = list(getattr(typenode.args, 'params', [])) + for i, arg in enumerate(params): + if not hasattr(arg, 'type'): + raise CDefError("%s arg %d: unknown type '%s'" + " (if you meant to use the old C syntax of giving" + " untyped arguments, it is not supported)" + % (funcname or 'in expression', i + 1, + getattr(arg, 'name', '?'))) + ellipsis = ( + len(params) > 0 and + isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and + isinstance(params[-1].type.type, + pycparser.c_ast.IdentifierType) and + params[-1].type.type.names == ['__dotdotdot__']) + if ellipsis: + params.pop() + if not params: + raise CDefError( + "%s: a function with only '(...)' as argument" + " is not correct C" % (funcname or 'in expression')) + args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) + for argdeclnode in params] + if not ellipsis and args == [model.void_type]: + args = [] + result, quals = self._get_type_and_quals(typenode.type) + # the 'quals' on the result type are ignored. HACK: we absure them + # to detect __stdcall functions: we textually replace "__stdcall" + # with "volatile volatile const" above. + abi = None + if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway + if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']: + abi = '__stdcall' + return model.RawFunctionType(tuple(args), result, ellipsis, abi) + + def _as_func_arg(self, type, quals): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item, quals) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): + # First, a level of caching on the exact 'type' node of the AST. + # This is obscure, but needed because pycparser "unrolls" declarations + # such as "typedef struct { } foo_t, *foo_p" and we end up with + # an AST that is not a tree, but a DAG, with the "type" node of the + # two branches foo_t and foo_p of the trees being the same node. + # It's a bit silly but detecting "DAG-ness" in the AST tree seems + # to be the only way to distinguish this case from two independent + # structs. See test_struct_with_two_usages. + try: + return self._structnode2type[type] + except KeyError: + pass + # + # Note that this must handle parsing "struct foo" any number of + # times and always return the same StructType object. Additionally, + # one of these times (not necessarily the first), the fields of + # the struct can be specified with "struct foo { ...fields... }". + # If no name is given, then we have to create a new anonymous struct + # with no caching; in this case, the fields are either specified + # right now or never. + # + force_name = name + name = type.name + # + # get the type or create it if needed + if name is None: + # 'force_name' is used to guess a more readable name for + # anonymous structs, for the common case "typedef struct { } foo". + if force_name is not None: + explicit_name = '$%s' % force_name + else: + self._anonymous_counter += 1 + explicit_name = '$%d' % self._anonymous_counter + tp = None + else: + explicit_name = name + key = '%s %s' % (kind, name) + tp, _ = self._declarations.get(key, (None, None)) + # + if tp is None: + if kind == 'struct': + tp = model.StructType(explicit_name, None, None, None) + elif kind == 'union': + tp = model.UnionType(explicit_name, None, None, None) + elif kind == 'enum': + if explicit_name == '__dotdotdot__': + raise CDefError("Enums cannot be declared with ...") + tp = self._build_enum_type(explicit_name, type.values) + else: + raise AssertionError("kind = %r" % (kind,)) + if name is not None: + self._declare(key, tp) + else: + if kind == 'enum' and type.values is not None: + raise NotImplementedError( + "enum %s: the '{}' declaration should appear on the first " + "time the enum is mentioned, not later" % explicit_name) + if not tp.forcename: + tp.force_the_name(force_name) + if tp.forcename and '$' in tp.name: + self._declare('anonymous %s' % tp.forcename, tp) + # + self._structnode2type[type] = tp + # + # enums: done here + if kind == 'enum': + return tp + # + # is there a 'type.decls'? If yes, then this is the place in the + # C sources that declare the fields. If no, then just return the + # existing type, possibly still incomplete. + if type.decls is None: + return tp + # + if tp.fldnames is not None: + raise CDefError("duplicate declaration of struct %s" % name) + fldnames = [] + fldtypes = [] + fldbitsize = [] + fldquals = [] + for decl in type.decls: + if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and + ''.join(decl.type.names) == '__dotdotdot__'): + # XXX pycparser is inconsistent: 'names' should be a list + # of strings, but is sometimes just one string. Use + # str.join() as a way to cope with both. + self._make_partial(tp, nested) + continue + if decl.bitsize is None: + bitsize = -1 + else: + bitsize = self._parse_constant(decl.bitsize) + self._partial_length = False + type, fqual = self._get_type_and_quals(decl.type, + partial_length_ok=True) + if self._partial_length: + self._make_partial(tp, nested) + if isinstance(type, model.StructType) and type.partial: + self._make_partial(tp, nested) + fldnames.append(decl.name or '') + fldtypes.append(type) + fldbitsize.append(bitsize) + fldquals.append(fqual) + tp.fldnames = tuple(fldnames) + tp.fldtypes = tuple(fldtypes) + tp.fldbitsize = tuple(fldbitsize) + tp.fldquals = tuple(fldquals) + if fldbitsize != [-1] * len(fldbitsize): + if isinstance(tp, model.StructType) and tp.partial: + raise NotImplementedError("%s: using both bitfields and '...;'" + % (tp,)) + tp.packed = self._options.get('packed') + if tp.completed: # must be re-completed: it is not opaque any more + tp.completed = 0 + self._recomplete.append(tp) + return tp + + def _make_partial(self, tp, nested): + if not isinstance(tp, model.StructOrUnion): + raise CDefError("%s cannot be partial" % (tp,)) + if not tp.has_c_name() and not nested: + raise NotImplementedError("%s is partial but has no C name" %(tp,)) + tp.partial = True + + def _parse_constant(self, exprnode, partial_length_ok=False): + # for now, limited to expressions that are an immediate number + # or positive/negative number + if isinstance(exprnode, pycparser.c_ast.Constant): + s = exprnode.value + if '0' <= s[0] <= '9': + s = s.rstrip('uUlL') + try: + if s.startswith('0'): + return int(s, 8) + else: + return int(s, 10) + except ValueError: + if len(s) > 1: + if s.lower()[0:2] == '0x': + return int(s, 16) + elif s.lower()[0:2] == '0b': + return int(s, 2) + raise CDefError("invalid constant %r" % (s,)) + elif s[0] == "'" and s[-1] == "'" and ( + len(s) == 3 or (len(s) == 4 and s[1] == "\\")): + return ord(s[-2]) + else: + raise CDefError("invalid constant %r" % (s,)) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '+'): + return self._parse_constant(exprnode.expr) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '-'): + return -self._parse_constant(exprnode.expr) + # load previously defined int constant + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name in self._int_constants): + return self._int_constants[exprnode.name] + # + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name == '__dotdotdotarray__'): + if partial_length_ok: + self._partial_length = True + return '...' + raise FFIError(":%d: unsupported '[...]' here, cannot derive " + "the actual array length in this context" + % exprnode.coord.line) + # + if isinstance(exprnode, pycparser.c_ast.BinaryOp): + left = self._parse_constant(exprnode.left) + right = self._parse_constant(exprnode.right) + if exprnode.op == '+': + return left + right + elif exprnode.op == '-': + return left - right + elif exprnode.op == '*': + return left * right + elif exprnode.op == '/': + return self._c_div(left, right) + elif exprnode.op == '%': + return left - self._c_div(left, right) * right + elif exprnode.op == '<<': + return left << right + elif exprnode.op == '>>': + return left >> right + elif exprnode.op == '&': + return left & right + elif exprnode.op == '|': + return left | right + elif exprnode.op == '^': + return left ^ right + # + raise FFIError(":%d: unsupported expression: expected a " + "simple numeric constant" % exprnode.coord.line) + + def _c_div(self, a, b): + result = a // b + if ((a < 0) ^ (b < 0)) and (a % b) != 0: + result += 1 + return result + + def _build_enum_type(self, explicit_name, decls): + if decls is not None: + partial = False + enumerators = [] + enumvalues = [] + nextenumvalue = 0 + for enum in decls.enumerators: + if _r_enum_dotdotdot.match(enum.name): + partial = True + continue + if enum.value is not None: + nextenumvalue = self._parse_constant(enum.value) + enumerators.append(enum.name) + enumvalues.append(nextenumvalue) + self._add_constants(enum.name, nextenumvalue) + nextenumvalue += 1 + enumerators = tuple(enumerators) + enumvalues = tuple(enumvalues) + tp = model.EnumType(explicit_name, enumerators, enumvalues) + tp.partial = partial + else: # opaque enum + tp = model.EnumType(explicit_name, (), ()) + return tp + + def include(self, other): + for name, (tp, quals) in other._declarations.items(): + if name.startswith('anonymous $enum_$'): + continue # fix for test_anonymous_enum_include + kind = name.split(' ', 1)[0] + if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef'): + self._declare(name, tp, included=True, quals=quals) + for k, v in other._int_constants.items(): + self._add_constants(k, v) + + def _get_unknown_type(self, decl): + typenames = decl.type.type.names + if typenames == ['__dotdotdot__']: + return model.unknown_type(decl.name) + + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) + + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) + + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/lib/cffi/error.py b/lib/cffi/error.py new file mode 100644 index 0000000..0a27247 --- /dev/null +++ b/lib/cffi/error.py @@ -0,0 +1,31 @@ + +class FFIError(Exception): + __module__ = 'cffi' + +class CDefError(Exception): + __module__ = 'cffi' + def __str__(self): + try: + current_decl = self.args[1] + filename = current_decl.coord.file + linenum = current_decl.coord.line + prefix = '%s:%d: ' % (filename, linenum) + except (AttributeError, TypeError, IndexError): + prefix = '' + return '%s%s' % (prefix, self.args[0]) + +class VerificationError(Exception): + """ An error raised when verification fails + """ + __module__ = 'cffi' + +class VerificationMissing(Exception): + """ An error raised when incomplete structures are passed into + cdef, but no verification has been done + """ + __module__ = 'cffi' + +class PkgConfigError(Exception): + """ An error raised for missing modules in pkg-config + """ + __module__ = 'cffi' diff --git a/lib/cffi/ffiplatform.py b/lib/cffi/ffiplatform.py new file mode 100644 index 0000000..adca28f --- /dev/null +++ b/lib/cffi/ffiplatform.py @@ -0,0 +1,113 @@ +import sys, os +from .error import VerificationError + + +LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', + 'extra_objects', 'depends'] + +def get_extension(srcfilename, modname, sources=(), **kwds): + from cffi._shimmed_dist_utils import Extension + allsources = [srcfilename] + for src in sources: + allsources.append(os.path.normpath(src)) + return Extension(name=modname, sources=allsources, **kwds) + +def compile(tmpdir, ext, compiler_verbose=0, debug=None): + """Compile a C extension module using distutils.""" + + saved_environ = os.environ.copy() + try: + outputfilename = _build(tmpdir, ext, compiler_verbose, debug) + outputfilename = os.path.abspath(outputfilename) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(tmpdir, ext, compiler_verbose=0, debug=None): + # XXX compact but horrible :-( + from cffi._shimmed_dist_utils import Distribution, CompileError, LinkError, set_threshold, set_verbosity + + dist = Distribution({'ext_modules': [ext]}) + dist.parse_config_files() + options = dist.get_option_dict('build_ext') + if debug is None: + debug = sys.flags.debug + options['debug'] = ('ffiplatform', debug) + options['force'] = ('ffiplatform', True) + options['build_lib'] = ('ffiplatform', tmpdir) + options['build_temp'] = ('ffiplatform', tmpdir) + # + try: + old_level = set_threshold(0) or 0 + try: + set_verbosity(compiler_verbose) + dist.run_command('build_ext') + cmd_obj = dist.get_command_obj('build_ext') + [soname] = cmd_obj.get_outputs() + finally: + set_threshold(old_level) + except (CompileError, LinkError) as e: + raise VerificationError('%s: %s' % (e.__class__.__name__, e)) + # + return soname + +try: + from os.path import samefile +except ImportError: + def samefile(f1, f2): + return os.path.abspath(f1) == os.path.abspath(f2) + +def maybe_relative_path(path): + if not os.path.isabs(path): + return path # already relative + dir = path + names = [] + while True: + prevdir = dir + dir, name = os.path.split(prevdir) + if dir == prevdir or not dir: + return path # failed to make it relative + names.append(name) + try: + if samefile(dir, os.curdir): + names.reverse() + return os.path.join(*names) + except OSError: + pass + +# ____________________________________________________________ + +try: + int_or_long = (int, long) + import cStringIO +except NameError: + int_or_long = int # Python 3 + import io as cStringIO + +def _flatten(x, f): + if isinstance(x, str): + f.write('%ds%s' % (len(x), x)) + elif isinstance(x, dict): + keys = sorted(x.keys()) + f.write('%dd' % len(keys)) + for key in keys: + _flatten(key, f) + _flatten(x[key], f) + elif isinstance(x, (list, tuple)): + f.write('%dl' % len(x)) + for value in x: + _flatten(value, f) + elif isinstance(x, int_or_long): + f.write('%di' % (x,)) + else: + raise TypeError( + "the keywords to verify() contains unsupported object %r" % (x,)) + +def flatten(x): + f = cStringIO.StringIO() + _flatten(x, f) + return f.getvalue() diff --git a/lib/cffi/lock.py b/lib/cffi/lock.py new file mode 100644 index 0000000..db91b71 --- /dev/null +++ b/lib/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/lib/cffi/model.py b/lib/cffi/model.py new file mode 100644 index 0000000..e5f4cae --- /dev/null +++ b/lib/cffi/model.py @@ -0,0 +1,618 @@ +import types +import weakref + +from .lock import allocate_lock +from .error import CDefError, VerificationError, VerificationMissing + +# type qualifiers +Q_CONST = 0x01 +Q_RESTRICT = 0x02 +Q_VOLATILE = 0x04 + +def qualify(quals, replace_with): + if quals & Q_CONST: + replace_with = ' const ' + replace_with.lstrip() + if quals & Q_VOLATILE: + replace_with = ' volatile ' + replace_with.lstrip() + if quals & Q_RESTRICT: + # It seems that __restrict is supported by gcc and msvc. + # If you hit some different compiler, add a #define in + # _cffi_include.h for it (and in its copies, documented there) + replace_with = ' __restrict ' + replace_with.lstrip() + return replace_with + + +class BaseTypeByIdentity(object): + is_array_type = False + is_raw_function = False + + def get_c_name(self, replace_with='', context='a C file', quals=0): + result = self.c_name_with_marker + assert result.count('&') == 1 + # some logic duplication with ffi.getctype()... :-( + replace_with = replace_with.strip() + if replace_with: + if replace_with.startswith('*') and '&[' in result: + replace_with = '(%s)' % replace_with + elif not replace_with[0] in '[(': + replace_with = ' ' + replace_with + replace_with = qualify(quals, replace_with) + result = result.replace('&', replace_with) + if '$' in result: + raise VerificationError( + "cannot generate '%s' in %s: unknown type name" + % (self._get_c_name(), context)) + return result + + def _get_c_name(self): + return self.c_name_with_marker.replace('&', '') + + def has_c_name(self): + return '$' not in self._get_c_name() + + def is_integer_type(self): + return False + + def get_cached_btype(self, ffi, finishlist, can_delay=False): + try: + BType = ffi._cached_btypes[self] + except KeyError: + BType = self.build_backend_type(ffi, finishlist) + BType2 = ffi._cached_btypes.setdefault(self, BType) + assert BType2 is BType + return BType + + def __repr__(self): + return '<%s>' % (self._get_c_name(),) + + def _get_items(self): + return [(name, getattr(self, name)) for name in self._attrs_] + + +class BaseType(BaseTypeByIdentity): + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self._get_items() == other._get_items()) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.__class__, tuple(self._get_items()))) + + +class VoidType(BaseType): + _attrs_ = () + + def __init__(self): + self.c_name_with_marker = 'void&' + + def build_backend_type(self, ffi, finishlist): + return global_cache(self, ffi, 'new_void_type') + +void_type = VoidType() + + +class BasePrimitiveType(BaseType): + def is_complex_type(self): + return False + + +class PrimitiveType(BasePrimitiveType): + _attrs_ = ('name',) + + ALL_PRIMITIVE_TYPES = { + 'char': 'c', + 'short': 'i', + 'int': 'i', + 'long': 'i', + 'long long': 'i', + 'signed char': 'i', + 'unsigned char': 'i', + 'unsigned short': 'i', + 'unsigned int': 'i', + 'unsigned long': 'i', + 'unsigned long long': 'i', + 'float': 'f', + 'double': 'f', + 'long double': 'f', + '_cffi_float_complex_t': 'j', + '_cffi_double_complex_t': 'j', + '_Bool': 'i', + # the following types are not primitive in the C sense + 'wchar_t': 'c', + 'char16_t': 'c', + 'char32_t': 'c', + 'int8_t': 'i', + 'uint8_t': 'i', + 'int16_t': 'i', + 'uint16_t': 'i', + 'int32_t': 'i', + 'uint32_t': 'i', + 'int64_t': 'i', + 'uint64_t': 'i', + 'int_least8_t': 'i', + 'uint_least8_t': 'i', + 'int_least16_t': 'i', + 'uint_least16_t': 'i', + 'int_least32_t': 'i', + 'uint_least32_t': 'i', + 'int_least64_t': 'i', + 'uint_least64_t': 'i', + 'int_fast8_t': 'i', + 'uint_fast8_t': 'i', + 'int_fast16_t': 'i', + 'uint_fast16_t': 'i', + 'int_fast32_t': 'i', + 'uint_fast32_t': 'i', + 'int_fast64_t': 'i', + 'uint_fast64_t': 'i', + 'intptr_t': 'i', + 'uintptr_t': 'i', + 'intmax_t': 'i', + 'uintmax_t': 'i', + 'ptrdiff_t': 'i', + 'size_t': 'i', + 'ssize_t': 'i', + } + + def __init__(self, name): + assert name in self.ALL_PRIMITIVE_TYPES + self.name = name + self.c_name_with_marker = name + '&' + + def is_char_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'c' + def is_integer_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'i' + def is_float_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'f' + def is_complex_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'j' + + def build_backend_type(self, ffi, finishlist): + return global_cache(self, ffi, 'new_primitive_type', self.name) + + +class UnknownIntegerType(BasePrimitiveType): + _attrs_ = ('name',) + + def __init__(self, name): + self.name = name + self.c_name_with_marker = name + '&' + + def is_integer_type(self): + return True + + def build_backend_type(self, ffi, finishlist): + raise NotImplementedError("integer type '%s' can only be used after " + "compilation" % self.name) + +class UnknownFloatType(BasePrimitiveType): + _attrs_ = ('name', ) + + def __init__(self, name): + self.name = name + self.c_name_with_marker = name + '&' + + def build_backend_type(self, ffi, finishlist): + raise NotImplementedError("float type '%s' can only be used after " + "compilation" % self.name) + + +class BaseFunctionType(BaseType): + _attrs_ = ('args', 'result', 'ellipsis', 'abi') + + def __init__(self, args, result, ellipsis, abi=None): + self.args = args + self.result = result + self.ellipsis = ellipsis + self.abi = abi + # + reprargs = [arg._get_c_name() for arg in self.args] + if self.ellipsis: + reprargs.append('...') + reprargs = reprargs or ['void'] + replace_with = self._base_pattern % (', '.join(reprargs),) + if abi is not None: + replace_with = replace_with[:1] + abi + ' ' + replace_with[1:] + self.c_name_with_marker = ( + self.result.c_name_with_marker.replace('&', replace_with)) + + +class RawFunctionType(BaseFunctionType): + # Corresponds to a C type like 'int(int)', which is the C type of + # a function, but not a pointer-to-function. The backend has no + # notion of such a type; it's used temporarily by parsing. + _base_pattern = '(&)(%s)' + is_raw_function = True + + def build_backend_type(self, ffi, finishlist): + raise CDefError("cannot render the type %r: it is a function " + "type, not a pointer-to-function type" % (self,)) + + def as_function_pointer(self): + return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi) + + +class FunctionPtrType(BaseFunctionType): + _base_pattern = '(*&)(%s)' + + def build_backend_type(self, ffi, finishlist): + result = self.result.get_cached_btype(ffi, finishlist) + args = [] + for tp in self.args: + args.append(tp.get_cached_btype(ffi, finishlist)) + abi_args = () + if self.abi == "__stdcall": + if not self.ellipsis: # __stdcall ignored for variadic funcs + try: + abi_args = (ffi._backend.FFI_STDCALL,) + except AttributeError: + pass + return global_cache(self, ffi, 'new_function_type', + tuple(args), result, self.ellipsis, *abi_args) + + def as_raw_function(self): + return RawFunctionType(self.args, self.result, self.ellipsis, self.abi) + + +class PointerType(BaseType): + _attrs_ = ('totype', 'quals') + + def __init__(self, totype, quals=0): + self.totype = totype + self.quals = quals + extra = " *&" + if totype.is_array_type: + extra = "(%s)" % (extra.lstrip(),) + extra = qualify(quals, extra) + self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) + + def build_backend_type(self, ffi, finishlist): + BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True) + return global_cache(self, ffi, 'new_pointer_type', BItem) + +voidp_type = PointerType(void_type) + +def ConstPointerType(totype): + return PointerType(totype, Q_CONST) + +const_voidp_type = ConstPointerType(void_type) + + +class NamedPointerType(PointerType): + _attrs_ = ('totype', 'name') + + def __init__(self, totype, name, quals=0): + PointerType.__init__(self, totype, quals) + self.name = name + self.c_name_with_marker = name + '&' + + +class ArrayType(BaseType): + _attrs_ = ('item', 'length') + is_array_type = True + + def __init__(self, item, length): + self.item = item + self.length = length + # + if length is None: + brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' + else: + brackets = '&[%s]' % length + self.c_name_with_marker = ( + self.item.c_name_with_marker.replace('&', brackets)) + + def length_is_unknown(self): + return isinstance(self.length, str) + + def resolve_length(self, newlength): + return ArrayType(self.item, newlength) + + def build_backend_type(self, ffi, finishlist): + if self.length_is_unknown(): + raise CDefError("cannot render the type %r: unknown length" % + (self,)) + self.item.get_cached_btype(ffi, finishlist) # force the item BType + BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) + return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) + +char_array_type = ArrayType(PrimitiveType('char'), None) + + +class StructOrUnionOrEnum(BaseTypeByIdentity): + _attrs_ = ('name',) + forcename = None + + def build_c_name_with_marker(self): + name = self.forcename or '%s %s' % (self.kind, self.name) + self.c_name_with_marker = name + '&' + + def force_the_name(self, forcename): + self.forcename = forcename + self.build_c_name_with_marker() + + def get_official_name(self): + assert self.c_name_with_marker.endswith('&') + return self.c_name_with_marker[:-1] + + +class StructOrUnion(StructOrUnionOrEnum): + fixedlayout = None + completed = 0 + partial = False + packed = 0 + + def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): + self.name = name + self.fldnames = fldnames + self.fldtypes = fldtypes + self.fldbitsize = fldbitsize + self.fldquals = fldquals + self.build_c_name_with_marker() + + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type + + def enumfields(self, expand_anonymous_struct_union=True): + fldquals = self.fldquals + if fldquals is None: + fldquals = (0,) * len(self.fldnames) + for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, + self.fldbitsize, fldquals): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): + # nested anonymous struct/union + for result in type.enumfields(): + yield result + else: + yield (name, type, bitsize, quals) + + def force_flatten(self): + # force the struct or union to have a declaration that lists + # directly all fields returned by enumfields(), flattening + # nested anonymous structs/unions. + names = [] + types = [] + bitsizes = [] + fldquals = [] + for name, type, bitsize, quals in self.enumfields(): + names.append(name) + types.append(type) + bitsizes.append(bitsize) + fldquals.append(quals) + self.fldnames = tuple(names) + self.fldtypes = tuple(types) + self.fldbitsize = tuple(bitsizes) + self.fldquals = tuple(fldquals) + + def get_cached_btype(self, ffi, finishlist, can_delay=False): + BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist, + can_delay) + if not can_delay: + self.finish_backend_type(ffi, finishlist) + return BType + + def finish_backend_type(self, ffi, finishlist): + if self.completed: + if self.completed != 2: + raise NotImplementedError("recursive structure declaration " + "for '%s'" % (self.name,)) + return + BType = ffi._cached_btypes[self] + # + self.completed = 1 + # + if self.fldtypes is None: + pass # not completing it: it's an opaque struct + # + elif self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] + lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) + extra_flags = () + if self.packed: + if self.packed == 1: + extra_flags = (8,) # SF_PACKED + else: + extra_flags = (0, self.packed) + ffi._backend.complete_struct_or_union(BType, lst, self, + -1, -1, *extra_flags) + # + else: + fldtypes = [] + fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout + for i in range(len(self.fldnames)): + fsize = fieldsize[i] + ftype = self.fldtypes[i] + # + if isinstance(ftype, ArrayType) and ftype.length_is_unknown(): + # fix the length to match the total size + BItemType = ftype.item.get_cached_btype(ffi, finishlist) + nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) + if nrest != 0: + self._verification_error( + "field '%s.%s' has a bogus size?" % ( + self.name, self.fldnames[i] or '{}')) + ftype = ftype.resolve_length(nlen) + self.fldtypes = (self.fldtypes[:i] + (ftype,) + + self.fldtypes[i+1:]) + # + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # + lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) + ffi._backend.complete_struct_or_union(BType, lst, self, + totalsize, totalalignment) + self.completed = 2 + + def _verification_error(self, msg): + raise VerificationError(msg) + + def check_not_partial(self): + if self.partial and self.fixedlayout is None: + raise VerificationMissing(self._get_c_name()) + + def build_backend_type(self, ffi, finishlist): + self.check_not_partial() + finishlist.append(self) + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, + self.get_official_name(), key=self) + + +class StructType(StructOrUnion): + kind = 'struct' + + +class UnionType(StructOrUnion): + kind = 'union' + + +class EnumType(StructOrUnionOrEnum): + kind = 'enum' + partial = False + partial_resolved = False + + def __init__(self, name, enumerators, enumvalues, baseinttype=None): + self.name = name + self.enumerators = enumerators + self.enumvalues = enumvalues + self.baseinttype = baseinttype + self.build_c_name_with_marker() + + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + + def check_not_partial(self): + if self.partial and not self.partial_resolved: + raise VerificationMissing(self._get_c_name()) + + def build_backend_type(self, ffi, finishlist): + self.check_not_partial() + base_btype = self.build_baseinttype(ffi, finishlist) + return global_cache(self, ffi, 'new_enum_type', + self.get_official_name(), + self.enumerators, self.enumvalues, + base_btype, key=self) + + def build_baseinttype(self, ffi, finishlist): + if self.baseinttype is not None: + return self.baseinttype.get_cached_btype(ffi, finishlist) + # + if self.enumvalues: + smallest_value = min(self.enumvalues) + largest_value = max(self.enumvalues) + else: + import warnings + try: + # XXX! The goal is to ensure that the warnings.warn() + # will not suppress the warning. We want to get it + # several times if we reach this point several times. + __warningregistry__.clear() + except NameError: + pass + warnings.warn("%r has no values explicitly defined; " + "guessing that it is equivalent to 'unsigned int'" + % self._get_c_name()) + smallest_value = largest_value = 0 + if smallest_value < 0: # needs a signed type + sign = 1 + candidate1 = PrimitiveType("int") + candidate2 = PrimitiveType("long") + else: + sign = 0 + candidate1 = PrimitiveType("unsigned int") + candidate2 = PrimitiveType("unsigned long") + btype1 = candidate1.get_cached_btype(ffi, finishlist) + btype2 = candidate2.get_cached_btype(ffi, finishlist) + size1 = ffi.sizeof(btype1) + size2 = ffi.sizeof(btype2) + if (smallest_value >= ((-1) << (8*size1-1)) and + largest_value < (1 << (8*size1-sign))): + return btype1 + if (smallest_value >= ((-1) << (8*size2-1)) and + largest_value < (1 << (8*size2-sign))): + return btype2 + raise CDefError("%s values don't all fit into either 'long' " + "or 'unsigned long'" % self._get_c_name()) + +def unknown_type(name, structname=None): + if structname is None: + structname = '$%s' % name + tp = StructType(structname, None, None, None) + tp.force_the_name(name) + tp.origin = "unknown_type" + return tp + +def unknown_ptr_type(name, structname=None): + if structname is None: + structname = '$$%s' % name + tp = StructType(structname, None, None, None) + return NamedPointerType(tp, name) + + +global_lock = allocate_lock() +_typecache_cffi_backend = weakref.WeakValueDictionary() + +def get_typecache(backend): + # returns _typecache_cffi_backend if backend is the _cffi_backend + # module, or type(backend).__typecache if backend is an instance of + # CTypesBackend (or some FakeBackend class during tests) + if isinstance(backend, types.ModuleType): + return _typecache_cffi_backend + with global_lock: + if not hasattr(type(backend), '__typecache'): + type(backend).__typecache = weakref.WeakValueDictionary() + return type(backend).__typecache + +def global_cache(srctype, ffi, funcname, *args, **kwds): + key = kwds.pop('key', (funcname, args)) + assert not kwds + try: + return ffi._typecache[key] + except KeyError: + pass + try: + res = getattr(ffi._backend, funcname)(*args) + except NotImplementedError as e: + raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e)) + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 + +def pointer_cache(ffi, BType): + return global_cache('?', ffi, 'new_pointer_type', BType) + +def attach_exception_info(e, name): + if e.args and type(e.args[0]) is str: + e.args = ('%s: %s' % (name, e.args[0]),) + e.args[1:] diff --git a/lib/cffi/parse_c_type.h b/lib/cffi/parse_c_type.h new file mode 100644 index 0000000..84e4ef8 --- /dev/null +++ b/lib/cffi/parse_c_type.h @@ -0,0 +1,181 @@ + +/* This part is from file 'cffi/parse_c_type.h'. It is copied at the + beginning of C sources generated by CFFI's ffi.set_source(). */ + +typedef void *_cffi_opcode_t; + +#define _CFFI_OP(opcode, arg) (_cffi_opcode_t)(opcode | (((uintptr_t)(arg)) << 8)) +#define _CFFI_GETOP(cffi_opcode) ((unsigned char)(uintptr_t)cffi_opcode) +#define _CFFI_GETARG(cffi_opcode) (((intptr_t)cffi_opcode) >> 8) + +#define _CFFI_OP_PRIMITIVE 1 +#define _CFFI_OP_POINTER 3 +#define _CFFI_OP_ARRAY 5 +#define _CFFI_OP_OPEN_ARRAY 7 +#define _CFFI_OP_STRUCT_UNION 9 +#define _CFFI_OP_ENUM 11 +#define _CFFI_OP_FUNCTION 13 +#define _CFFI_OP_FUNCTION_END 15 +#define _CFFI_OP_NOOP 17 +#define _CFFI_OP_BITFIELD 19 +#define _CFFI_OP_TYPENAME 21 +#define _CFFI_OP_CPYTHON_BLTN_V 23 // varargs +#define _CFFI_OP_CPYTHON_BLTN_N 25 // noargs +#define _CFFI_OP_CPYTHON_BLTN_O 27 // O (i.e. a single arg) +#define _CFFI_OP_CONSTANT 29 +#define _CFFI_OP_CONSTANT_INT 31 +#define _CFFI_OP_GLOBAL_VAR 33 +#define _CFFI_OP_DLOPEN_FUNC 35 +#define _CFFI_OP_DLOPEN_CONST 37 +#define _CFFI_OP_GLOBAL_VAR_F 39 +#define _CFFI_OP_EXTERN_PYTHON 41 + +#define _CFFI_PRIM_VOID 0 +#define _CFFI_PRIM_BOOL 1 +#define _CFFI_PRIM_CHAR 2 +#define _CFFI_PRIM_SCHAR 3 +#define _CFFI_PRIM_UCHAR 4 +#define _CFFI_PRIM_SHORT 5 +#define _CFFI_PRIM_USHORT 6 +#define _CFFI_PRIM_INT 7 +#define _CFFI_PRIM_UINT 8 +#define _CFFI_PRIM_LONG 9 +#define _CFFI_PRIM_ULONG 10 +#define _CFFI_PRIM_LONGLONG 11 +#define _CFFI_PRIM_ULONGLONG 12 +#define _CFFI_PRIM_FLOAT 13 +#define _CFFI_PRIM_DOUBLE 14 +#define _CFFI_PRIM_LONGDOUBLE 15 + +#define _CFFI_PRIM_WCHAR 16 +#define _CFFI_PRIM_INT8 17 +#define _CFFI_PRIM_UINT8 18 +#define _CFFI_PRIM_INT16 19 +#define _CFFI_PRIM_UINT16 20 +#define _CFFI_PRIM_INT32 21 +#define _CFFI_PRIM_UINT32 22 +#define _CFFI_PRIM_INT64 23 +#define _CFFI_PRIM_UINT64 24 +#define _CFFI_PRIM_INTPTR 25 +#define _CFFI_PRIM_UINTPTR 26 +#define _CFFI_PRIM_PTRDIFF 27 +#define _CFFI_PRIM_SIZE 28 +#define _CFFI_PRIM_SSIZE 29 +#define _CFFI_PRIM_INT_LEAST8 30 +#define _CFFI_PRIM_UINT_LEAST8 31 +#define _CFFI_PRIM_INT_LEAST16 32 +#define _CFFI_PRIM_UINT_LEAST16 33 +#define _CFFI_PRIM_INT_LEAST32 34 +#define _CFFI_PRIM_UINT_LEAST32 35 +#define _CFFI_PRIM_INT_LEAST64 36 +#define _CFFI_PRIM_UINT_LEAST64 37 +#define _CFFI_PRIM_INT_FAST8 38 +#define _CFFI_PRIM_UINT_FAST8 39 +#define _CFFI_PRIM_INT_FAST16 40 +#define _CFFI_PRIM_UINT_FAST16 41 +#define _CFFI_PRIM_INT_FAST32 42 +#define _CFFI_PRIM_UINT_FAST32 43 +#define _CFFI_PRIM_INT_FAST64 44 +#define _CFFI_PRIM_UINT_FAST64 45 +#define _CFFI_PRIM_INTMAX 46 +#define _CFFI_PRIM_UINTMAX 47 +#define _CFFI_PRIM_FLOATCOMPLEX 48 +#define _CFFI_PRIM_DOUBLECOMPLEX 49 +#define _CFFI_PRIM_CHAR16 50 +#define _CFFI_PRIM_CHAR32 51 + +#define _CFFI__NUM_PRIM 52 +#define _CFFI__UNKNOWN_PRIM (-1) +#define _CFFI__UNKNOWN_FLOAT_PRIM (-2) +#define _CFFI__UNKNOWN_LONG_DOUBLE (-3) + +#define _CFFI__IO_FILE_STRUCT (-1) + + +struct _cffi_global_s { + const char *name; + void *address; + _cffi_opcode_t type_op; + void *size_or_direct_fn; // OP_GLOBAL_VAR: size, or 0 if unknown + // OP_CPYTHON_BLTN_*: addr of direct function +}; + +struct _cffi_getconst_s { + unsigned long long value; + const struct _cffi_type_context_s *ctx; + int gindex; +}; + +struct _cffi_struct_union_s { + const char *name; + int type_index; // -> _cffi_types, on a OP_STRUCT_UNION + int flags; // _CFFI_F_* flags below + size_t size; + int alignment; + int first_field_index; // -> _cffi_fields array + int num_fields; +}; +#define _CFFI_F_UNION 0x01 // is a union, not a struct +#define _CFFI_F_CHECK_FIELDS 0x02 // complain if fields are not in the + // "standard layout" or if some are missing +#define _CFFI_F_PACKED 0x04 // for CHECK_FIELDS, assume a packed struct +#define _CFFI_F_EXTERNAL 0x08 // in some other ffi.include() +#define _CFFI_F_OPAQUE 0x10 // opaque + +struct _cffi_field_s { + const char *name; + size_t field_offset; + size_t field_size; + _cffi_opcode_t field_type_op; +}; + +struct _cffi_enum_s { + const char *name; + int type_index; // -> _cffi_types, on a OP_ENUM + int type_prim; // _CFFI_PRIM_xxx + const char *enumerators; // comma-delimited string +}; + +struct _cffi_typename_s { + const char *name; + int type_index; /* if opaque, points to a possibly artificial + OP_STRUCT which is itself opaque */ +}; + +struct _cffi_type_context_s { + _cffi_opcode_t *types; + const struct _cffi_global_s *globals; + const struct _cffi_field_s *fields; + const struct _cffi_struct_union_s *struct_unions; + const struct _cffi_enum_s *enums; + const struct _cffi_typename_s *typenames; + int num_globals; + int num_struct_unions; + int num_enums; + int num_typenames; + const char *const *includes; + int num_types; + int flags; /* future extension */ +}; + +struct _cffi_parse_info_s { + const struct _cffi_type_context_s *ctx; + _cffi_opcode_t *output; + unsigned int output_size; + size_t error_location; + const char *error_message; +}; + +struct _cffi_externpy_s { + const char *name; + size_t size_of_result; + void *reserved1, *reserved2; +}; + +#ifdef _CFFI_INTERNAL +static int parse_c_type(struct _cffi_parse_info_s *info, const char *input); +static int search_in_globals(const struct _cffi_type_context_s *ctx, + const char *search, size_t search_len); +static int search_in_struct_unions(const struct _cffi_type_context_s *ctx, + const char *search, size_t search_len); +#endif diff --git a/lib/cffi/pkgconfig.py b/lib/cffi/pkgconfig.py new file mode 100644 index 0000000..5c93f15 --- /dev/null +++ b/lib/cffi/pkgconfig.py @@ -0,0 +1,121 @@ +# pkg-config, https://www.freedesktop.org/wiki/Software/pkg-config/ integration for cffi +import sys, os, subprocess + +from .error import PkgConfigError + + +def merge_flags(cfg1, cfg2): + """Merge values from cffi config flags cfg2 to cf1 + + Example: + merge_flags({"libraries": ["one"]}, {"libraries": ["two"]}) + {"libraries": ["one", "two"]} + """ + for key, value in cfg2.items(): + if key not in cfg1: + cfg1[key] = value + else: + if not isinstance(cfg1[key], list): + raise TypeError("cfg1[%r] should be a list of strings" % (key,)) + if not isinstance(value, list): + raise TypeError("cfg2[%r] should be a list of strings" % (key,)) + cfg1[key].extend(value) + return cfg1 + + +def call(libname, flag, encoding=sys.getfilesystemencoding()): + """Calls pkg-config and returns the output if found + """ + a = ["pkg-config", "--print-errors"] + a.append(flag) + a.append(libname) + try: + pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except EnvironmentError as e: + raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) + + bout, berr = pc.communicate() + if pc.returncode != 0: + try: + berr = berr.decode(encoding) + except Exception: + pass + raise PkgConfigError(berr.strip()) + + if sys.version_info >= (3,) and not isinstance(bout, str): # Python 3.x + try: + bout = bout.decode(encoding) + except UnicodeDecodeError: + raise PkgConfigError("pkg-config %s %s returned bytes that cannot " + "be decoded with encoding %r:\n%r" % + (flag, libname, encoding, bout)) + + if os.altsep != '\\' and '\\' in bout: + raise PkgConfigError("pkg-config %s %s returned an unsupported " + "backslash-escaped output:\n%r" % + (flag, libname, bout)) + return bout + + +def flags_from_pkgconfig(libs): + r"""Return compiler line flags for FFI.set_source based on pkg-config output + + Usage + ... + ffibuilder.set_source("_foo", pkgconfig = ["libfoo", "libbar >= 1.8.3"]) + + If pkg-config is installed on build machine, then arguments include_dirs, + library_dirs, libraries, define_macros, extra_compile_args and + extra_link_args are extended with an output of pkg-config for libfoo and + libbar. + + Raises PkgConfigError in case the pkg-config call fails. + """ + + def get_include_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-I")] + + def get_library_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-L")] + + def get_libraries(string): + return [x[2:] for x in string.split() if x.startswith("-l")] + + # convert -Dfoo=bar to list of tuples [("foo", "bar")] expected by distutils + def get_macros(string): + def _macro(x): + x = x[2:] # drop "-D" + if '=' in x: + return tuple(x.split("=", 1)) # "-Dfoo=bar" => ("foo", "bar") + else: + return (x, None) # "-Dfoo" => ("foo", None) + return [_macro(x) for x in string.split() if x.startswith("-D")] + + def get_other_cflags(string): + return [x for x in string.split() if not x.startswith("-I") and + not x.startswith("-D")] + + def get_other_libs(string): + return [x for x in string.split() if not x.startswith("-L") and + not x.startswith("-l")] + + # return kwargs for given libname + def kwargs(libname): + fse = sys.getfilesystemencoding() + all_cflags = call(libname, "--cflags") + all_libs = call(libname, "--libs") + return { + "include_dirs": get_include_dirs(all_cflags), + "library_dirs": get_library_dirs(all_libs), + "libraries": get_libraries(all_libs), + "define_macros": get_macros(all_cflags), + "extra_compile_args": get_other_cflags(all_cflags), + "extra_link_args": get_other_libs(all_libs), + } + + # merge all arguments together + ret = {} + for libname in libs: + lib_flags = kwargs(libname) + merge_flags(ret, lib_flags) + return ret diff --git a/lib/cffi/recompiler.py b/lib/cffi/recompiler.py new file mode 100644 index 0000000..7734a34 --- /dev/null +++ b/lib/cffi/recompiler.py @@ -0,0 +1,1598 @@ +import io, os, sys, sysconfig +from . import ffiplatform, model +from .error import VerificationError +from .cffi_opcode import * + +VERSION_BASE = 0x2601 +VERSION_EMBEDDED = 0x2701 +VERSION_CHAR16CHAR32 = 0x2801 + +USE_LIMITED_API = ((sys.platform != 'win32' or sys.version_info < (3, 0) or + sys.version_info >= (3, 5)) and + not sysconfig.get_config_var("Py_GIL_DISABLED")) # free-threaded doesn't yet support limited API + +class GlobalExpr: + def __init__(self, name, address, type_op, size=0, check_value=0): + self.name = name + self.address = address + self.type_op = type_op + self.size = size + self.check_value = check_value + + def as_c_expr(self): + return ' { "%s", (void *)%s, %s, (void *)%s },' % ( + self.name, self.address, self.type_op.as_c_expr(), self.size) + + def as_python_expr(self): + return "b'%s%s',%d" % (self.type_op.as_python_bytes(), self.name, + self.check_value) + +class FieldExpr: + def __init__(self, name, field_offset, field_size, fbitsize, field_type_op): + self.name = name + self.field_offset = field_offset + self.field_size = field_size + self.fbitsize = fbitsize + self.field_type_op = field_type_op + + def as_c_expr(self): + spaces = " " * len(self.name) + return (' { "%s", %s,\n' % (self.name, self.field_offset) + + ' %s %s,\n' % (spaces, self.field_size) + + ' %s %s },' % (spaces, self.field_type_op.as_c_expr())) + + def as_python_expr(self): + raise NotImplementedError + + def as_field_python_expr(self): + if self.field_type_op.op == OP_NOOP: + size_expr = '' + elif self.field_type_op.op == OP_BITFIELD: + size_expr = format_four_bytes(self.fbitsize) + else: + raise NotImplementedError + return "b'%s%s%s'" % (self.field_type_op.as_python_bytes(), + size_expr, + self.name) + +class StructUnionExpr: + def __init__(self, name, type_index, flags, size, alignment, comment, + first_field_index, c_fields): + self.name = name + self.type_index = type_index + self.flags = flags + self.size = size + self.alignment = alignment + self.comment = comment + self.first_field_index = first_field_index + self.c_fields = c_fields + + def as_c_expr(self): + return (' { "%s", %d, %s,' % (self.name, self.type_index, self.flags) + + '\n %s, %s, ' % (self.size, self.alignment) + + '%d, %d ' % (self.first_field_index, len(self.c_fields)) + + ('/* %s */ ' % self.comment if self.comment else '') + + '},') + + def as_python_expr(self): + flags = eval(self.flags, G_FLAGS) + fields_expr = [c_field.as_field_python_expr() + for c_field in self.c_fields] + return "(b'%s%s%s',%s)" % ( + format_four_bytes(self.type_index), + format_four_bytes(flags), + self.name, + ','.join(fields_expr)) + +class EnumExpr: + def __init__(self, name, type_index, size, signed, allenums): + self.name = name + self.type_index = type_index + self.size = size + self.signed = signed + self.allenums = allenums + + def as_c_expr(self): + return (' { "%s", %d, _cffi_prim_int(%s, %s),\n' + ' "%s" },' % (self.name, self.type_index, + self.size, self.signed, self.allenums)) + + def as_python_expr(self): + prim_index = { + (1, 0): PRIM_UINT8, (1, 1): PRIM_INT8, + (2, 0): PRIM_UINT16, (2, 1): PRIM_INT16, + (4, 0): PRIM_UINT32, (4, 1): PRIM_INT32, + (8, 0): PRIM_UINT64, (8, 1): PRIM_INT64, + }[self.size, self.signed] + return "b'%s%s%s\\x00%s'" % (format_four_bytes(self.type_index), + format_four_bytes(prim_index), + self.name, self.allenums) + +class TypenameExpr: + def __init__(self, name, type_index): + self.name = name + self.type_index = type_index + + def as_c_expr(self): + return ' { "%s", %d },' % (self.name, self.type_index) + + def as_python_expr(self): + return "b'%s%s'" % (format_four_bytes(self.type_index), self.name) + + +# ____________________________________________________________ + + +class Recompiler: + _num_externpy = 0 + + def __init__(self, ffi, module_name, target_is_python=False): + self.ffi = ffi + self.module_name = module_name + self.target_is_python = target_is_python + self._version = VERSION_BASE + + def needs_version(self, ver): + self._version = max(self._version, ver) + + def collect_type_table(self): + self._typesdict = {} + self._generate("collecttype") + # + all_decls = sorted(self._typesdict, key=str) + # + # prepare all FUNCTION bytecode sequences first + self.cffi_types = [] + for tp in all_decls: + if tp.is_raw_function: + assert self._typesdict[tp] is None + self._typesdict[tp] = len(self.cffi_types) + self.cffi_types.append(tp) # placeholder + for tp1 in tp.args: + assert isinstance(tp1, (model.VoidType, + model.BasePrimitiveType, + model.PointerType, + model.StructOrUnionOrEnum, + model.FunctionPtrType)) + if self._typesdict[tp1] is None: + self._typesdict[tp1] = len(self.cffi_types) + self.cffi_types.append(tp1) # placeholder + self.cffi_types.append('END') # placeholder + # + # prepare all OTHER bytecode sequences + for tp in all_decls: + if not tp.is_raw_function and self._typesdict[tp] is None: + self._typesdict[tp] = len(self.cffi_types) + self.cffi_types.append(tp) # placeholder + if tp.is_array_type and tp.length is not None: + self.cffi_types.append('LEN') # placeholder + assert None not in self._typesdict.values() + # + # collect all structs and unions and enums + self._struct_unions = {} + self._enums = {} + for tp in all_decls: + if isinstance(tp, model.StructOrUnion): + self._struct_unions[tp] = None + elif isinstance(tp, model.EnumType): + self._enums[tp] = None + for i, tp in enumerate(sorted(self._struct_unions, + key=lambda tp: tp.name)): + self._struct_unions[tp] = i + for i, tp in enumerate(sorted(self._enums, + key=lambda tp: tp.name)): + self._enums[tp] = i + # + # emit all bytecode sequences now + for tp in all_decls: + method = getattr(self, '_emit_bytecode_' + tp.__class__.__name__) + method(tp, self._typesdict[tp]) + # + # consistency check + for op in self.cffi_types: + assert isinstance(op, CffiOp) + self.cffi_types = tuple(self.cffi_types) # don't change any more + + def _enum_fields(self, tp): + # When producing C, expand all anonymous struct/union fields. + # That's necessary to have C code checking the offsets of the + # individual fields contained in them. When producing Python, + # don't do it and instead write it like it is, with the + # corresponding fields having an empty name. Empty names are + # recognized at runtime when we import the generated Python + # file. + expand_anonymous_struct_union = not self.target_is_python + return tp.enumfields(expand_anonymous_struct_union) + + def _do_collect_type(self, tp): + if not isinstance(tp, model.BaseTypeByIdentity): + if isinstance(tp, tuple): + for x in tp: + self._do_collect_type(x) + return + if tp not in self._typesdict: + self._typesdict[tp] = None + if isinstance(tp, model.FunctionPtrType): + self._do_collect_type(tp.as_raw_function()) + elif isinstance(tp, model.StructOrUnion): + if tp.fldtypes is not None and ( + tp not in self.ffi._parser._included_declarations): + for name1, tp1, _, _ in self._enum_fields(tp): + self._do_collect_type(self._field_type(tp, name1, tp1)) + else: + for _, x in tp._get_items(): + self._do_collect_type(x) + + def _generate(self, step_name): + lst = self.ffi._parser._declarations.items() + for name, (tp, quals) in sorted(lst): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_cpy_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in recompile(): %r" % name) + try: + self._current_quals = quals + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + # ---------- + + ALL_STEPS = ["global", "field", "struct_union", "enum", "typename"] + + def collect_step_tables(self): + # collect the declarations for '_cffi_globals', '_cffi_typenames', etc. + self._lsts = {} + for step_name in self.ALL_STEPS: + self._lsts[step_name] = [] + self._seen_struct_unions = set() + self._generate("ctx") + self._add_missing_struct_unions() + # + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + if step_name != "field": + lst.sort(key=lambda entry: entry.name) + self._lsts[step_name] = tuple(lst) # don't change any more + # + # check for a possible internal inconsistency: _cffi_struct_unions + # should have been generated with exactly self._struct_unions + lst = self._lsts["struct_union"] + for tp, i in self._struct_unions.items(): + assert i < len(lst) + assert lst[i].name == tp.name + assert len(lst) == len(self._struct_unions) + # same with enums + lst = self._lsts["enum"] + for tp, i in self._enums.items(): + assert i < len(lst) + assert lst[i].name == tp.name + assert len(lst) == len(self._enums) + + # ---------- + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def write_source_to_f(self, f, preamble): + if self.target_is_python: + assert preamble is None + self.write_py_source_to_f(f) + else: + assert preamble is not None + self.write_c_source_to_f(f, preamble) + + def _rel_readlines(self, filename): + g = open(os.path.join(os.path.dirname(__file__), filename), 'r') + lines = g.readlines() + g.close() + return lines + + def write_c_source_to_f(self, f, preamble): + self._f = f + prnt = self._prnt + if self.ffi._embedding is not None: + prnt('#define _CFFI_USE_EMBEDDING') + if not USE_LIMITED_API: + prnt('#define _CFFI_NO_LIMITED_API') + # + # first the '#include' (actually done by inlining the file's content) + lines = self._rel_readlines('_cffi_include.h') + i = lines.index('#include "parse_c_type.h"\n') + lines[i:i+1] = self._rel_readlines('parse_c_type.h') + prnt(''.join(lines)) + # + # if we have ffi._embedding != None, we give it here as a macro + # and include an extra file + base_module_name = self.module_name.split('.')[-1] + if self.ffi._embedding is not None: + prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') + prnt('#ifdef PYPY_VERSION') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( + base_module_name,)) + prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( + base_module_name,)) + prnt('#else') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + base_module_name,)) + prnt('#endif') + lines = self._rel_readlines('_embedding.h') + i = lines.index('#include "_cffi_errors.h"\n') + lines[i:i+1] = self._rel_readlines('_cffi_errors.h') + prnt(''.join(lines)) + self.needs_version(VERSION_EMBEDDED) + # + # then paste the C source given by the user, verbatim. + prnt('/************************************************************/') + prnt() + prnt(preamble) + prnt() + prnt('/************************************************************/') + prnt() + # + # the declaration of '_cffi_types' + prnt('static void *_cffi_types[] = {') + typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()]) + for i, op in enumerate(self.cffi_types): + comment = '' + if i in typeindex2type: + comment = ' // ' + typeindex2type[i]._get_c_name() + prnt('/* %2d */ %s,%s' % (i, op.as_c_expr(), comment)) + if not self.cffi_types: + prnt(' 0') + prnt('};') + prnt() + # + # call generate_cpy_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._seen_constants = set() + self._generate("decl") + # + # the declaration of '_cffi_globals' and '_cffi_typenames' + nums = {} + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + nums[step_name] = len(lst) + if nums[step_name] > 0: + prnt('static const struct _cffi_%s_s _cffi_%ss[] = {' % ( + step_name, step_name)) + for entry in lst: + prnt(entry.as_c_expr()) + prnt('};') + prnt() + # + # the declaration of '_cffi_includes' + if self.ffi._included_ffis: + prnt('static const char * const _cffi_includes[] = {') + for ffi_to_include in self.ffi._included_ffis: + try: + included_module_name, included_source = ( + ffi_to_include._assigned_source[:2]) + except AttributeError: + raise VerificationError( + "ffi object %r includes %r, but the latter has not " + "been prepared with set_source()" % ( + self.ffi, ffi_to_include,)) + if included_source is None: + raise VerificationError( + "not implemented yet: ffi.include() of a Python-based " + "ffi inside a C-based ffi") + prnt(' "%s",' % (included_module_name,)) + prnt(' NULL') + prnt('};') + prnt() + # + # the declaration of '_cffi_type_context' + prnt('static const struct _cffi_type_context_s _cffi_type_context = {') + prnt(' _cffi_types,') + for step_name in self.ALL_STEPS: + if nums[step_name] > 0: + prnt(' _cffi_%ss,' % step_name) + else: + prnt(' NULL, /* no %ss */' % step_name) + for step_name in self.ALL_STEPS: + if step_name != "field": + prnt(' %d, /* num_%ss */' % (nums[step_name], step_name)) + if self.ffi._included_ffis: + prnt(' _cffi_includes,') + else: + prnt(' NULL, /* no includes */') + prnt(' %d, /* num_types */' % (len(self.cffi_types),)) + flags = 0 + if self._num_externpy > 0 or self.ffi._embedding is not None: + flags |= 1 # set to mean that we use extern "Python" + prnt(' %d, /* flags */' % flags) + prnt('};') + prnt() + # + # the init function + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility push(default) /* for -fvisibility= */') + prnt('#endif') + prnt() + prnt('#ifdef PYPY_VERSION') + prnt('PyMODINIT_FUNC') + prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,)) + prnt('{') + if flags & 1: + prnt(' if (((intptr_t)p[0]) >= 0x0A03) {') + prnt(' _cffi_call_python_org = ' + '(void(*)(struct _cffi_externpy_s *, char *))p[1];') + prnt(' }') + prnt(' p[0] = (const void *)0x%x;' % self._version) + prnt(' p[1] = &_cffi_type_context;') + prnt('#if PY_MAJOR_VERSION >= 3') + prnt(' return NULL;') + prnt('#endif') + prnt('}') + # on Windows, distutils insists on putting init_cffi_xyz in + # 'export_symbols', so instead of fighting it, just give up and + # give it one + prnt('# ifdef _MSC_VER') + prnt(' PyMODINIT_FUNC') + prnt('# if PY_MAJOR_VERSION >= 3') + prnt(' PyInit_%s(void) { return NULL; }' % (base_module_name,)) + prnt('# else') + prnt(' init%s(void) { }' % (base_module_name,)) + prnt('# endif') + prnt('# endif') + prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % (base_module_name,)) + prnt('{') + prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( + self.module_name, self._version)) + prnt('}') + prnt('#else') + prnt('PyMODINIT_FUNC') + prnt('init%s(void)' % (base_module_name,)) + prnt('{') + prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( + self.module_name, self._version)) + prnt('}') + prnt('#endif') + prnt() + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility pop') + prnt('#endif') + self._version = None + + def _to_py(self, x): + if isinstance(x, str): + return "b'%s'" % (x,) + if isinstance(x, (list, tuple)): + rep = [self._to_py(item) for item in x] + if len(rep) == 1: + rep.append('') + return "(%s)" % (','.join(rep),) + return x.as_python_expr() # Py2: unicode unexpected; Py3: bytes unexp. + + def write_py_source_to_f(self, f): + self._f = f + prnt = self._prnt + # + # header + prnt("# auto-generated file") + prnt("import _cffi_backend") + # + # the 'import' of the included ffis + num_includes = len(self.ffi._included_ffis or ()) + for i in range(num_includes): + ffi_to_include = self.ffi._included_ffis[i] + try: + included_module_name, included_source = ( + ffi_to_include._assigned_source[:2]) + except AttributeError: + raise VerificationError( + "ffi object %r includes %r, but the latter has not " + "been prepared with set_source()" % ( + self.ffi, ffi_to_include,)) + if included_source is not None: + raise VerificationError( + "not implemented yet: ffi.include() of a C-based " + "ffi inside a Python-based ffi") + prnt('from %s import ffi as _ffi%d' % (included_module_name, i)) + prnt() + prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,)) + prnt(" _version = 0x%x," % (self._version,)) + self._version = None + # + # the '_types' keyword argument + self.cffi_types = tuple(self.cffi_types) # don't change any more + types_lst = [op.as_python_bytes() for op in self.cffi_types] + prnt(' _types = %s,' % (self._to_py(''.join(types_lst)),)) + typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()]) + # + # the keyword arguments from ALL_STEPS + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + if len(lst) > 0 and step_name != "field": + prnt(' _%ss = %s,' % (step_name, self._to_py(lst))) + # + # the '_includes' keyword argument + if num_includes > 0: + prnt(' _includes = (%s,),' % ( + ', '.join(['_ffi%d' % i for i in range(num_includes)]),)) + # + # the footer + prnt(')') + + # ---------- + + def _gettypenum(self, type): + # a KeyError here is a bug. please report it! :-) + return self._typesdict[type] + + def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode): + extraarg = '' + if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type(): + if tp.is_integer_type() and tp.name != '_Bool': + converter = '_cffi_to_c_int' + extraarg = ', %s' % tp.name + elif isinstance(tp, model.UnknownFloatType): + # don't check with is_float_type(): it may be a 'long + # double' here, and _cffi_to_c_double would loose precision + converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),) + else: + cname = tp.get_c_name('') + converter = '(%s)_cffi_to_c_%s' % (cname, + tp.name.replace(' ', '_')) + if cname in ('char16_t', 'char32_t'): + self.needs_version(VERSION_CHAR16CHAR32) + errvalue = '-1' + # + elif isinstance(tp, model.PointerType): + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return + # + elif (isinstance(tp, model.StructOrUnionOrEnum) or + isinstance(tp, model.BasePrimitiveType)): + # a struct (not a struct pointer) as a function argument; + # or, a complex (the same code works) + self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' + % (tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + return + # + elif isinstance(tp, model.FunctionPtrType): + converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') + extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) + errvalue = 'NULL' + # + else: + raise NotImplementedError(tp) + # + self._prnt(' %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg)) + self._prnt(' if (%s == (%s)%s && PyErr_Occurred())' % ( + tovar, tp.get_c_name(''), errvalue)) + self._prnt(' %s;' % errcode) + + def _extra_local_variables(self, tp, localvars, freelines): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + localvars.add('struct _cffi_freeme_s *large_args_free = NULL') + freelines.add('if (large_args_free != NULL)' + ' _cffi_free_array_arguments(large_args_free);') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' %s = ((size_t)datasize) <= 640 ? ' + '(%s)alloca((size_t)datasize) : NULL;' % ( + tovar, tp.get_c_name(''))) + self._prnt(' if (_cffi_convert_array_argument(_cffi_type(%d), %s, ' + '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar)) + self._prnt(' datasize, &large_args_free) < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' }') + + def _convert_expr_from_c(self, tp, var, context): + if isinstance(tp, model.BasePrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + return '_cffi_from_c_int(%s, %s)' % (var, tp.name) + elif isinstance(tp, model.UnknownFloatType): + return '_cffi_from_c_double(%s)' % (var,) + elif tp.name != 'long double' and not tp.is_complex_type(): + cname = tp.name.replace(' ', '_') + if cname in ('char16_t', 'char32_t'): + self.needs_version(VERSION_CHAR16CHAR32) + return '_cffi_from_c_%s(%s)' % (cname, var) + else: + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, (model.PointerType, model.FunctionPtrType)): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.ArrayType): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) + elif isinstance(tp, model.StructOrUnion): + if tp.fldnames is None: + raise TypeError("'%s' is used as %s, but is opaque" % ( + tp._get_c_name(), context)) + return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.EnumType): + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + else: + raise NotImplementedError(tp) + + # ---------- + # typedefs + + def _typedef_type(self, tp, name): + return self._global_type(tp, "(*(%s *)0)" % (name,)) + + def _generate_cpy_typedef_collecttype(self, tp, name): + self._do_collect_type(self._typedef_type(tp, name)) + + def _generate_cpy_typedef_decl(self, tp, name): + pass + + def _typedef_ctx(self, tp, name): + type_index = self._typesdict[tp] + self._lsts["typename"].append(TypenameExpr(name, type_index)) + + def _generate_cpy_typedef_ctx(self, tp, name): + tp = self._typedef_type(tp, name) + self._typedef_ctx(tp, name) + if getattr(tp, "origin", None) == "unknown_type": + self._struct_ctx(tp, tp.name, approxname=None) + elif isinstance(tp, model.NamedPointerType): + self._struct_ctx(tp.totype, tp.totype.name, approxname=tp.name, + named_ptr=tp) + + # ---------- + # function declarations + + def _generate_cpy_function_collecttype(self, tp, name): + self._do_collect_type(tp.as_raw_function()) + if tp.ellipsis and not self.target_is_python: + self._do_collect_type(tp) + + def _generate_cpy_function_decl(self, tp, name): + assert not self.target_is_python + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no CPython wrapper) + self._generate_cpy_constant_decl(tp, name) + return + prnt = self._prnt + numargs = len(tp.args) + if numargs == 0: + argname = 'noarg' + elif numargs == 1: + argname = 'arg0' + else: + argname = 'args' + # + # ------------------------------ + # the 'd' version of the function, only for addressof(lib, 'func') + arguments = [] + call_arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arguments.append(type.get_c_name(' x%d' % i, context)) + call_arguments.append('x%d' % i) + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + name_and_arguments = '%s_cffi_d_%s(%s)' % (abi, name, repr_arguments) + prnt('static %s' % (tp.result.get_c_name(name_and_arguments),)) + prnt('{') + call_arguments = ', '.join(call_arguments) + result_code = 'return ' + if isinstance(tp.result, model.VoidType): + result_code = '' + prnt(' %s%s(%s);' % (result_code, name, call_arguments)) + prnt('}') + # + prnt('#ifndef PYPY_VERSION') # ------------------------------ + # + prnt('static PyObject *') + prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname)) + prnt('{') + # + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arg = type.get_c_name(' x%d' % i, context) + prnt(' %s;' % arg) + # + localvars = set() + freelines = set() + for type in tp.args: + self._extra_local_variables(type, localvars, freelines) + for decl in sorted(localvars): + prnt(' %s;' % (decl,)) + # + if not isinstance(tp.result, model.VoidType): + result_code = 'result = ' + context = 'result of %s' % name + result_decl = ' %s;' % tp.result.get_c_name(' result', context) + prnt(result_decl) + prnt(' PyObject *pyresult;') + else: + result_decl = None + result_code = '' + # + if len(tp.args) > 1: + rng = range(len(tp.args)) + for i in rng: + prnt(' PyObject *arg%d;' % i) + prnt() + prnt(' if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % ( + name, len(rng), len(rng), + ', '.join(['&arg%d' % i for i in rng]))) + prnt(' return NULL;') + prnt() + # + for i, type in enumerate(tp.args): + self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i, + 'return NULL') + prnt() + # + prnt(' Py_BEGIN_ALLOW_THREADS') + prnt(' _cffi_restore_errno();') + call_arguments = ['x%d' % i for i in range(len(tp.args))] + call_arguments = ', '.join(call_arguments) + prnt(' { %s%s(%s); }' % (result_code, name, call_arguments)) + prnt(' _cffi_save_errno();') + prnt(' Py_END_ALLOW_THREADS') + prnt() + # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') + if result_code: + prnt(' pyresult = %s;' % + self._convert_expr_from_c(tp.result, 'result', 'result type')) + for freeline in freelines: + prnt(' ' + freeline) + prnt(' return pyresult;') + else: + for freeline in freelines: + prnt(' ' + freeline) + prnt(' Py_INCREF(Py_None);') + prnt(' return Py_None;') + prnt('}') + # + prnt('#else') # ------------------------------ + # + # the PyPy version: need to replace struct/union arguments with + # pointers, and if the result is a struct/union, insert a first + # arg that is a pointer to the result. We also do that for + # complex args and return type. + def need_indirection(type): + return (isinstance(type, model.StructOrUnion) or + (isinstance(type, model.PrimitiveType) and + type.is_complex_type())) + difference = False + arguments = [] + call_arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + indirection = '' + if need_indirection(type): + indirection = '*' + difference = True + arg = type.get_c_name(' %sx%d' % (indirection, i), context) + arguments.append(arg) + call_arguments.append('%sx%d' % (indirection, i)) + tp_result = tp.result + if need_indirection(tp_result): + context = 'result of %s' % name + arg = tp_result.get_c_name(' *result', context) + arguments.insert(0, arg) + tp_result = model.void_type + result_decl = None + result_code = '*result = ' + difference = True + if difference: + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + name_and_arguments = '%s_cffi_f_%s(%s)' % (abi, name, + repr_arguments) + prnt('static %s' % (tp_result.get_c_name(name_and_arguments),)) + prnt('{') + if result_decl: + prnt(result_decl) + call_arguments = ', '.join(call_arguments) + prnt(' { %s%s(%s); }' % (result_code, name, call_arguments)) + if result_decl: + prnt(' return result;') + prnt('}') + else: + prnt('# define _cffi_f_%s _cffi_d_%s' % (name, name)) + # + prnt('#endif') # ------------------------------ + prnt() + + def _generate_cpy_function_ctx(self, tp, name): + if tp.ellipsis and not self.target_is_python: + self._generate_cpy_constant_ctx(tp, name) + return + type_index = self._typesdict[tp.as_raw_function()] + numargs = len(tp.args) + if self.target_is_python: + meth_kind = OP_DLOPEN_FUNC + elif numargs == 0: + meth_kind = OP_CPYTHON_BLTN_N # 'METH_NOARGS' + elif numargs == 1: + meth_kind = OP_CPYTHON_BLTN_O # 'METH_O' + else: + meth_kind = OP_CPYTHON_BLTN_V # 'METH_VARARGS' + self._lsts["global"].append( + GlobalExpr(name, '_cffi_f_%s' % name, + CffiOp(meth_kind, type_index), + size='_cffi_d_%s' % name)) + + # ---------- + # named structs or unions + + def _field_type(self, tp_struct, field_name, tp_field): + if isinstance(tp_field, model.ArrayType): + actual_length = tp_field.length + if actual_length == '...': + ptr_struct_name = tp_struct.get_c_name('*') + actual_length = '_cffi_array_len(((%s)0)->%s)' % ( + ptr_struct_name, field_name) + tp_item = self._field_type(tp_struct, '%s[0]' % field_name, + tp_field.item) + tp_field = model.ArrayType(tp_item, actual_length) + return tp_field + + def _struct_collecttype(self, tp): + self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) + + def _struct_decl(self, tp, cname, approxname): + if tp.fldtypes is None: + return + prnt = self._prnt + checkfuncname = '_cffi_checkfld_%s' % (approxname,) + prnt('_CFFI_UNUSED_FN') + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in self._enum_fields(tp): + try: + if ftype.is_integer_type() or fbitsize >= 0: + # accept all integers, but complain on float or double + if fname != '': + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " + "an integer */" % (fname, cname, fname)) + continue + # only accept exactly the type declared, except that '[]' + # is interpreted as a '*' and so will match any array length. + # (It would also match '*', but that's harder to detect...) + while (isinstance(ftype, model.ArrayType) + and (ftype.length is None or ftype.length == '...')): + ftype = ftype.item + fname = fname + '[0]' + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname)) + prnt() + + def _struct_ctx(self, tp, cname, approxname, named_ptr=None): + type_index = self._typesdict[tp] + reason_for_not_expanding = None + flags = [] + if isinstance(tp, model.UnionType): + flags.append("_CFFI_F_UNION") + if tp.fldtypes is None: + flags.append("_CFFI_F_OPAQUE") + reason_for_not_expanding = "opaque" + if (tp not in self.ffi._parser._included_declarations and + (named_ptr is None or + named_ptr not in self.ffi._parser._included_declarations)): + if tp.fldtypes is None: + pass # opaque + elif tp.partial or any(tp.anonymous_struct_fields()): + pass # field layout obtained silently from the C compiler + else: + flags.append("_CFFI_F_CHECK_FIELDS") + if tp.packed: + if tp.packed > 1: + raise NotImplementedError( + "%r is declared with 'pack=%r'; only 0 or 1 are " + "supported in API mode (try to use \"...;\", which " + "does not require a 'pack' declaration)" % + (tp, tp.packed)) + flags.append("_CFFI_F_PACKED") + else: + flags.append("_CFFI_F_EXTERNAL") + reason_for_not_expanding = "external" + flags = '|'.join(flags) or '0' + c_fields = [] + if reason_for_not_expanding is None: + enumfields = list(self._enum_fields(tp)) + for fldname, fldtype, fbitsize, fqual in enumfields: + fldtype = self._field_type(tp, fldname, fldtype) + self._check_not_opaque(fldtype, + "field '%s.%s'" % (tp.name, fldname)) + # cname is None for _add_missing_struct_unions() only + op = OP_NOOP + if fbitsize >= 0: + op = OP_BITFIELD + size = '%d /* bits */' % fbitsize + elif cname is None or ( + isinstance(fldtype, model.ArrayType) and + fldtype.length is None): + size = '(size_t)-1' + else: + size = 'sizeof(((%s)0)->%s)' % ( + tp.get_c_name('*') if named_ptr is None + else named_ptr.name, + fldname) + if cname is None or fbitsize >= 0: + offset = '(size_t)-1' + elif named_ptr is not None: + offset = '(size_t)(((char *)&((%s)4096)->%s) - (char *)4096)' % ( + named_ptr.name, fldname) + else: + offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname) + c_fields.append( + FieldExpr(fldname, offset, size, fbitsize, + CffiOp(op, self._typesdict[fldtype]))) + first_field_index = len(self._lsts["field"]) + self._lsts["field"].extend(c_fields) + # + if cname is None: # unknown name, for _add_missing_struct_unions + size = '(size_t)-2' + align = -2 + comment = "unnamed" + else: + if named_ptr is not None: + size = 'sizeof(*(%s)0)' % (named_ptr.name,) + align = '-1 /* unknown alignment */' + else: + size = 'sizeof(%s)' % (cname,) + align = 'offsetof(struct _cffi_align_%s, y)' % (approxname,) + comment = None + else: + size = '(size_t)-1' + align = -1 + first_field_index = -1 + comment = reason_for_not_expanding + self._lsts["struct_union"].append( + StructUnionExpr(tp.name, type_index, flags, size, align, comment, + first_field_index, c_fields)) + self._seen_struct_unions.add(tp) + + def _check_not_opaque(self, tp, location): + while isinstance(tp, model.ArrayType): + tp = tp.item + if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None: + raise TypeError( + "%s is of an opaque type (not declared in cdef())" % location) + + def _add_missing_struct_unions(self): + # not very nice, but some struct declarations might be missing + # because they don't have any known C name. Check that they are + # not partial (we can't complete or verify them!) and emit them + # anonymously. + lst = list(self._struct_unions.items()) + lst.sort(key=lambda tp_order: tp_order[1]) + for tp, order in lst: + if tp not in self._seen_struct_unions: + if tp.partial: + raise NotImplementedError("internal inconsistency: %r is " + "partial but was not seen at " + "this point" % (tp,)) + if tp.name.startswith('$') and tp.name[1:].isdigit(): + approxname = tp.name[1:] + elif tp.name == '_IO_FILE' and tp.forcename == 'FILE': + approxname = 'FILE' + self._typedef_ctx(tp, 'FILE') + else: + raise NotImplementedError("internal inconsistency: %r" % + (tp,)) + self._struct_ctx(tp, None, approxname) + + def _generate_cpy_struct_collecttype(self, tp, name): + self._struct_collecttype(tp) + _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype + + def _struct_names(self, tp): + cname = tp.get_c_name('') + if ' ' in cname: + return cname, cname.replace(' ', '_') + else: + return cname, '_' + cname + + def _generate_cpy_struct_decl(self, tp, name): + self._struct_decl(tp, *self._struct_names(tp)) + _generate_cpy_union_decl = _generate_cpy_struct_decl + + def _generate_cpy_struct_ctx(self, tp, name): + self._struct_ctx(tp, *self._struct_names(tp)) + _generate_cpy_union_ctx = _generate_cpy_struct_ctx + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_cpy_anonymous_collecttype(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_collecttype(tp, name) + else: + self._struct_collecttype(tp) + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp) + else: + self._struct_decl(tp, name, 'typedef_' + name) + + def _generate_cpy_anonymous_ctx(self, tp, name): + if isinstance(tp, model.EnumType): + self._enum_ctx(tp, name) + else: + self._struct_ctx(tp, name, 'typedef_' + name) + + # ---------- + # constants, declared with "static const ..." + + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + check_value=None): + if (category, name) in self._seen_constants: + raise VerificationError( + "duplicate declaration of %s '%s'" % (category, name)) + self._seen_constants.add((category, name)) + # + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + if is_int: + prnt('static int %s(unsigned long long *o)' % funcname) + prnt('{') + prnt(' int n = (%s) <= 0;' % (name,)) + prnt(' *o = (unsigned long long)((%s) | 0);' + ' /* check that %s is an integer */' % (name, name)) + if check_value is not None: + if check_value > 0: + check_value = '%dU' % (check_value,) + prnt(' if (!_cffi_check_int(*o, n, %s))' % (check_value,)) + prnt(' n |= 2;') + prnt(' return n;') + prnt('}') + else: + assert check_value is None + prnt('static void %s(char *o)' % funcname) + prnt('{') + prnt(' *(%s)o = %s;' % (tp.get_c_name('*'), name)) + prnt('}') + prnt() + + def _generate_cpy_constant_collecttype(self, tp, name): + is_int = tp.is_integer_type() + if not is_int or self.target_is_python: + self._do_collect_type(tp) + + def _generate_cpy_constant_decl(self, tp, name): + is_int = tp.is_integer_type() + self._generate_cpy_const(is_int, name, tp) + + def _generate_cpy_constant_ctx(self, tp, name): + if not self.target_is_python and tp.is_integer_type(): + type_op = CffiOp(OP_CONSTANT_INT, -1) + else: + if self.target_is_python: + const_kind = OP_DLOPEN_CONST + else: + const_kind = OP_CONSTANT + type_index = self._typesdict[tp] + type_op = CffiOp(const_kind, type_index) + self._lsts["global"].append( + GlobalExpr(name, '_cffi_const_%s' % name, type_op)) + + # ---------- + # enums + + def _generate_cpy_enum_collecttype(self, tp, name): + self._do_collect_type(tp) + + def _generate_cpy_enum_decl(self, tp, name=None): + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator) + + def _enum_ctx(self, tp, cname): + type_index = self._typesdict[tp] + type_op = CffiOp(OP_ENUM, -1) + if self.target_is_python: + tp.check_not_partial() + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._lsts["global"].append( + GlobalExpr(enumerator, '_cffi_const_%s' % enumerator, type_op, + check_value=enumvalue)) + # + if cname is not None and '$' not in cname and not self.target_is_python: + size = "sizeof(%s)" % cname + signed = "((%s)-1) <= 0" % cname + else: + basetp = tp.build_baseinttype(self.ffi, []) + size = self.ffi.sizeof(basetp) + signed = int(int(self.ffi.cast(basetp, -1)) < 0) + allenums = ",".join(tp.enumerators) + self._lsts["enum"].append( + EnumExpr(tp.name, type_index, size, signed, allenums)) + + def _generate_cpy_enum_ctx(self, tp, name): + self._enum_ctx(tp, tp._get_c_name()) + + # ---------- + # macros: for now only for integers + + def _generate_cpy_macro_collecttype(self, tp, name): + pass + + def _generate_cpy_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) + + def _generate_cpy_macro_ctx(self, tp, name): + if tp == '...': + if self.target_is_python: + raise VerificationError( + "cannot use the syntax '...' in '#define %s ...' when " + "using the ABI mode" % (name,)) + check_value = None + else: + check_value = tp # an integer + type_op = CffiOp(OP_CONSTANT_INT, -1) + self._lsts["global"].append( + GlobalExpr(name, '_cffi_const_%s' % name, type_op, + check_value=check_value)) + + # ---------- + # global variables + + def _global_type(self, tp, global_name): + if isinstance(tp, model.ArrayType): + actual_length = tp.length + if actual_length == '...': + actual_length = '_cffi_array_len(%s)' % (global_name,) + tp_item = self._global_type(tp.item, '%s[0]' % global_name) + tp = model.ArrayType(tp_item, actual_length) + return tp + + def _generate_cpy_variable_collecttype(self, tp, name): + self._do_collect_type(self._global_type(tp, name)) + + def _generate_cpy_variable_decl(self, tp, name): + prnt = self._prnt + tp = self._global_type(tp, name) + if isinstance(tp, model.ArrayType) and tp.length is None: + tp = tp.item + ampersand = '' + else: + ampersand = '&' + # This code assumes that casts from "tp *" to "void *" is a + # no-op, i.e. a function that returns a "tp *" can be called + # as if it returned a "void *". This should be generally true + # on any modern machine. The only exception to that rule (on + # uncommon architectures, and as far as I can tell) might be + # if 'tp' were a function type, but that is not possible here. + # (If 'tp' is a function _pointer_ type, then casts from "fn_t + # **" to "void *" are again no-ops, as far as I can tell.) + decl = '*_cffi_var_%s(void)' % (name,) + prnt('static ' + tp.get_c_name(decl, quals=self._current_quals)) + prnt('{') + prnt(' return %s(%s);' % (ampersand, name)) + prnt('}') + prnt() + + def _generate_cpy_variable_ctx(self, tp, name): + tp = self._global_type(tp, name) + type_index = self._typesdict[tp] + if self.target_is_python: + op = OP_GLOBAL_VAR + else: + op = OP_GLOBAL_VAR_F + self._lsts["global"].append( + GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index))) + + # ---------- + # extern "Python" + + def _generate_cpy_extern_python_collecttype(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + self._do_collect_type(tp) + _generate_cpy_dllexport_python_collecttype = \ + _generate_cpy_extern_python_plus_c_collecttype = \ + _generate_cpy_extern_python_collecttype + + def _extern_python_decl(self, tp, name, tag_and_space): + prnt = self._prnt + if isinstance(tp.result, model.VoidType): + size_of_result = '0' + else: + context = 'result of %s' % name + size_of_result = '(int)sizeof(%s)' % ( + tp.result.get_c_name('', context),) + prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name) + prnt(' { "%s.%s", %s, 0, 0 };' % ( + self.module_name, name, size_of_result)) + prnt() + # + arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arg = type.get_c_name(' a%d' % i, context) + arguments.append(arg) + # + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + name_and_arguments = '%s(%s)' % (name, repr_arguments) + if tp.abi == "__stdcall": + name_and_arguments = '_cffi_stdcall ' + name_and_arguments + # + def may_need_128_bits(tp): + return (isinstance(tp, model.PrimitiveType) and + tp.name == 'long double') + # + size_of_a = max(len(tp.args)*8, 8) + if may_need_128_bits(tp.result): + size_of_a = max(size_of_a, 16) + if isinstance(tp.result, model.StructOrUnion): + size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % ( + tp.result.get_c_name(''), size_of_a, + tp.result.get_c_name(''), size_of_a) + prnt('%s%s' % (tag_and_space, tp.result.get_c_name(name_and_arguments))) + prnt('{') + prnt(' char a[%s];' % size_of_a) + prnt(' char *p = a;') + for i, type in enumerate(tp.args): + arg = 'a%d' % i + if (isinstance(type, model.StructOrUnion) or + may_need_128_bits(type)): + arg = '&' + arg + type = model.PointerType(type) + prnt(' *(%s)(p + %d) = %s;' % (type.get_c_name('*'), i*8, arg)) + prnt(' _cffi_call_python(&_cffi_externpy__%s, p);' % name) + if not isinstance(tp.result, model.VoidType): + prnt(' return *(%s)p;' % (tp.result.get_c_name('*'),)) + prnt('}') + prnt() + self._num_externpy += 1 + + def _generate_cpy_extern_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'static ') + + def _generate_cpy_dllexport_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'CFFI_DLLEXPORT ') + + def _generate_cpy_extern_python_plus_c_decl(self, tp, name): + self._extern_python_decl(tp, name, '') + + def _generate_cpy_extern_python_ctx(self, tp, name): + if self.target_is_python: + raise VerificationError( + "cannot use 'extern \"Python\"' in the ABI mode") + if tp.ellipsis: + raise NotImplementedError("a vararg function is extern \"Python\"") + type_index = self._typesdict[tp] + type_op = CffiOp(OP_EXTERN_PYTHON, type_index) + self._lsts["global"].append( + GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name)) + + _generate_cpy_dllexport_python_ctx = \ + _generate_cpy_extern_python_plus_c_ctx = \ + _generate_cpy_extern_python_ctx + + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + if not isinstance(s, bytes): # unicode + s = s.encode('utf-8') # -> bytes + else: + s.decode('utf-8') # got bytes, check for valid utf-8 + try: + s.decode('ascii') + except UnicodeDecodeError: + s = b'# -*- encoding: utf8 -*-\n' + s + for line in s.splitlines(True): + comment = line + if type('//') is bytes: # python2 + line = map(ord, line) # make a list of integers + else: # python3 + # type(line) is bytes, which enumerates like a list of integers + comment = ascii(comment)[1:-1] + prnt(('// ' + comment).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (c,) + prnt(printed_line) + + # ---------- + # emitting the opcodes for individual types + + def _emit_bytecode_VoidType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, PRIM_VOID) + + def _emit_bytecode_PrimitiveType(self, tp, index): + prim_index = PRIMITIVE_TO_INDEX[tp.name] + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index) + + def _emit_bytecode_UnknownIntegerType(self, tp, index): + s = ('_cffi_prim_int(sizeof(%s), (\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' + ' ) <= 0)' % (tp.name, tp.name, tp.name)) + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) + + def _emit_bytecode_UnknownFloatType(self, tp, index): + s = ('_cffi_prim_float(sizeof(%s) *\n' + ' (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n' + ' )' % (tp.name, tp.name)) + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) + + def _emit_bytecode_RawFunctionType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result]) + index += 1 + for tp1 in tp.args: + realindex = self._typesdict[tp1] + if index != realindex: + if isinstance(tp1, model.PrimitiveType): + self._emit_bytecode_PrimitiveType(tp1, index) + else: + self.cffi_types[index] = CffiOp(OP_NOOP, realindex) + index += 1 + flags = int(tp.ellipsis) + if tp.abi is not None: + if tp.abi == '__stdcall': + flags |= 2 + else: + raise NotImplementedError("abi=%r" % (tp.abi,)) + self.cffi_types[index] = CffiOp(OP_FUNCTION_END, flags) + + def _emit_bytecode_PointerType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype]) + + _emit_bytecode_ConstPointerType = _emit_bytecode_PointerType + _emit_bytecode_NamedPointerType = _emit_bytecode_PointerType + + def _emit_bytecode_FunctionPtrType(self, tp, index): + raw = tp.as_raw_function() + self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw]) + + def _emit_bytecode_ArrayType(self, tp, index): + item_index = self._typesdict[tp.item] + if tp.length is None: + self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index) + elif tp.length == '...': + raise VerificationError( + "type %s badly placed: the '...' array length can only be " + "used on global arrays or on fields of structures" % ( + str(tp).replace('/*...*/', '...'),)) + else: + assert self.cffi_types[index + 1] == 'LEN' + self.cffi_types[index] = CffiOp(OP_ARRAY, item_index) + self.cffi_types[index + 1] = CffiOp(None, str(tp.length)) + + def _emit_bytecode_StructType(self, tp, index): + struct_index = self._struct_unions[tp] + self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index) + _emit_bytecode_UnionType = _emit_bytecode_StructType + + def _emit_bytecode_EnumType(self, tp, index): + enum_index = self._enums[tp] + self.cffi_types[index] = CffiOp(OP_ENUM, enum_index) + + +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + +def _is_file_like(maybefile): + # compare to xml.etree.ElementTree._get_writer + return hasattr(maybefile, 'write') + +def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): + if verbose: + print("generating %s" % (target_file,)) + recompiler = Recompiler(ffi, module_name, + target_is_python=(preamble is None)) + recompiler.collect_type_table() + recompiler.collect_step_tables() + if _is_file_like(target_file): + recompiler.write_source_to_f(target_file, preamble) + return True + f = NativeIO() + recompiler.write_source_to_f(f, preamble) + output = f.getvalue() + try: + with open(target_file, 'r') as f1: + if f1.read(len(output) + 1) != output: + raise IOError + if verbose: + print("(already up-to-date)") + return False # already up-to-date + except IOError: + tmp_file = '%s.~%d' % (target_file, os.getpid()) + with open(tmp_file, 'w') as f1: + f1.write(output) + try: + os.rename(tmp_file, target_file) + except OSError: + os.unlink(target_file) + os.rename(tmp_file, target_file) + return True + +def make_c_source(ffi, module_name, preamble, target_c_file, verbose=False): + assert preamble is not None + return _make_c_or_py_source(ffi, module_name, preamble, target_c_file, + verbose) + +def make_py_source(ffi, module_name, target_py_file, verbose=False): + return _make_c_or_py_source(ffi, module_name, None, target_py_file, + verbose) + +def _modname_to_file(outputdir, modname, extension): + parts = modname.split('.') + try: + os.makedirs(os.path.join(outputdir, *parts[:-1])) + except OSError: + pass + parts[-1] += extension + return os.path.join(outputdir, *parts), parts + + +# Aaargh. Distutils is not tested at all for the purpose of compiling +# DLLs that are not extension modules. Here are some hacks to work +# around that, in the _patch_for_*() functions... + +def _patch_meth(patchlist, cls, name, new_meth): + old = getattr(cls, name) + patchlist.append((cls, name, old)) + setattr(cls, name, new_meth) + return old + +def _unpatch_meths(patchlist): + for cls, name, old_meth in reversed(patchlist): + setattr(cls, name, old_meth) + +def _patch_for_embedding(patchlist): + if sys.platform == 'win32': + # we must not remove the manifest when building for embedding! + # FUTURE: this module was removed in setuptools 74; this is likely dead code and should be removed, + # since the toolchain it supports (VS2005-2008) is also long dead. + from cffi._shimmed_dist_utils import MSVCCompiler + if MSVCCompiler is not None: + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) + + if sys.platform == 'darwin': + # we must not make a '-bundle', but a '-dynamiclib' instead + from cffi._shimmed_dist_utils import CCompiler + def my_link_shared_object(self, *args, **kwds): + if '-bundle' in self.linker_so: + self.linker_so = list(self.linker_so) + i = self.linker_so.index('-bundle') + self.linker_so[i] = '-dynamiclib' + return old_link_shared_object(self, *args, **kwds) + old_link_shared_object = _patch_meth(patchlist, CCompiler, + 'link_shared_object', + my_link_shared_object) + +def _patch_for_target(patchlist, target): + from cffi._shimmed_dist_utils import build_ext + # if 'target' is different from '*', we need to patch some internal + # method to just return this 'target' value, instead of having it + # built from module_name + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + elif sys.platform == 'darwin': + target += '.dylib' + else: + target += '.so' + _patch_meth(patchlist, build_ext, 'get_ext_filename', + lambda self, ext_name: target) + + +def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, + c_file=None, source_extension='.c', extradir=None, + compiler_verbose=1, target=None, debug=None, + uses_ffiplatform=True, **kwds): + if not isinstance(module_name, str): + module_name = module_name.encode('ascii') + if ffi._windows_unicode: + ffi._apply_windows_unicode(kwds) + if preamble is not None: + if call_c_compiler and _is_file_like(c_file): + raise TypeError("Writing to file-like objects is not supported " + "with call_c_compiler=True") + embedding = (ffi._embedding is not None) + if embedding: + ffi._apply_embedding_fix(kwds) + if c_file is None: + c_file, parts = _modname_to_file(tmpdir, module_name, + source_extension) + if extradir: + parts = [extradir] + parts + ext_c_file = os.path.join(*parts) + else: + ext_c_file = c_file + # + if target is None: + if embedding: + target = '%s.*' % module_name + else: + target = '*' + # + if uses_ffiplatform: + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + else: + ext = None + updated = make_c_source(ffi, module_name, preamble, c_file, + verbose=compiler_verbose) + if call_c_compiler: + patchlist = [] + cwd = os.getcwd() + try: + if embedding: + _patch_for_embedding(patchlist) + if target != '*': + _patch_for_target(patchlist, target) + if compiler_verbose: + if tmpdir == '.': + msg = 'the current directory is' + else: + msg = 'setting the current directory to' + print('%s %r' % (msg, os.path.abspath(tmpdir))) + os.chdir(tmpdir) + outputfilename = ffiplatform.compile('.', ext, + compiler_verbose, debug) + finally: + os.chdir(cwd) + _unpatch_meths(patchlist) + return outputfilename + else: + return ext, updated + else: + if c_file is None: + c_file, _ = _modname_to_file(tmpdir, module_name, '.py') + updated = make_py_source(ffi, module_name, c_file, + verbose=compiler_verbose) + if call_c_compiler: + return c_file + else: + return None, updated + diff --git a/lib/cffi/setuptools_ext.py b/lib/cffi/setuptools_ext.py new file mode 100644 index 0000000..5cdd246 --- /dev/null +++ b/lib/cffi/setuptools_ext.py @@ -0,0 +1,229 @@ +import os +import sys +import sysconfig + +try: + basestring +except NameError: + # Python 3.x + basestring = str + +def error(msg): + from cffi._shimmed_dist_utils import DistutilsSetupError + raise DistutilsSetupError(msg) + + +def execfile(filename, glob): + # We use execfile() (here rewritten for Python 3) instead of + # __import__() to load the build script. The problem with + # a normal import is that in some packages, the intermediate + # __init__.py files may already try to import the file that + # we are generating. + with open(filename) as f: + src = f.read() + src += '\n' # Python 2.6 compatibility + code = compile(src, filename, 'exec') + exec(code, glob, glob) + + +def add_cffi_module(dist, mod_spec): + from cffi.api import FFI + + if not isinstance(mod_spec, basestring): + error("argument to 'cffi_modules=...' must be a str or a list of str," + " not %r" % (type(mod_spec).__name__,)) + mod_spec = str(mod_spec) + try: + build_file_name, ffi_var_name = mod_spec.split(':') + except ValueError: + error("%r must be of the form 'path/build.py:ffi_variable'" % + (mod_spec,)) + if not os.path.exists(build_file_name): + ext = '' + rewritten = build_file_name.replace('.', '/') + '.py' + if os.path.exists(rewritten): + ext = ' (rewrite cffi_modules to [%r])' % ( + rewritten + ':' + ffi_var_name,) + error("%r does not name an existing file%s" % (build_file_name, ext)) + + mod_vars = {'__name__': '__cffi__', '__file__': build_file_name} + execfile(build_file_name, mod_vars) + + try: + ffi = mod_vars[ffi_var_name] + except KeyError: + error("%r: object %r not found in module" % (mod_spec, + ffi_var_name)) + if not isinstance(ffi, FFI): + ffi = ffi() # maybe it's a function instead of directly an ffi + if not isinstance(ffi, FFI): + error("%r is not an FFI instance (got %r)" % (mod_spec, + type(ffi).__name__)) + if not hasattr(ffi, '_assigned_source'): + error("%r: the set_source() method was not called" % (mod_spec,)) + module_name, source, source_extension, kwds = ffi._assigned_source + if ffi._windows_unicode: + kwds = kwds.copy() + ffi._apply_windows_unicode(kwds) + + if source is None: + _add_py_module(dist, ffi, module_name) + else: + _add_c_module(dist, ffi, module_name, source, source_extension, kwds) + +def _set_py_limited_api(Extension, kwds): + """ + Add py_limited_api to kwds if setuptools >= 26 is in use. + Do not alter the setting if it already exists. + Setuptools takes care of ignoring the flag on Python 2 and PyPy. + + CPython itself should ignore the flag in a debugging version + (by not listing .abi3.so in the extensions it supports), but + it doesn't so far, creating troubles. That's why we check + for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent + of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, with CPython <= 3.4, it's better not to use py_limited_api + because virtualenv *still* doesn't copy PYTHON3.DLL on these versions. + Recently (2020) we started shipping only >= 3.5 wheels, though. So + we'll give it another try and set py_limited_api on Windows >= 3.5. + """ + from cffi._shimmed_dist_utils import log + from cffi import recompiler + + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and recompiler.USE_LIMITED_API): + import setuptools + try: + setuptools_major_version = int(setuptools.__version__.partition('.')[0]) + if setuptools_major_version >= 26: + kwds['py_limited_api'] = True + except ValueError: # certain development versions of setuptools + # If we don't know the version number of setuptools, we + # try to set 'py_limited_api' anyway. At worst, we get a + # warning. + kwds['py_limited_api'] = True + + if sysconfig.get_config_var("Py_GIL_DISABLED"): + if kwds.get('py_limited_api'): + log.info("Ignoring py_limited_api=True for free-threaded build.") + + kwds['py_limited_api'] = False + + if kwds.get('py_limited_api') is False: + # avoid setting Py_LIMITED_API if py_limited_api=False + # which _cffi_include.h does unless _CFFI_NO_LIMITED_API is defined + kwds.setdefault("define_macros", []).append(("_CFFI_NO_LIMITED_API", None)) + return kwds + +def _add_c_module(dist, ffi, module_name, source, source_extension, kwds): + # We are a setuptools extension. Need this build_ext for py_limited_api. + from setuptools.command.build_ext import build_ext + from cffi._shimmed_dist_utils import Extension, log, mkpath + from cffi import recompiler + + allsources = ['$PLACEHOLDER'] + allsources.extend(kwds.pop('sources', [])) + kwds = _set_py_limited_api(Extension, kwds) + ext = Extension(name=module_name, sources=allsources, **kwds) + + def make_mod(tmpdir, pre_run=None): + c_file = os.path.join(tmpdir, module_name + source_extension) + log.info("generating cffi module %r" % c_file) + mkpath(tmpdir) + # a setuptools-only, API-only hook: called with the "ext" and "ffi" + # arguments just before we turn the ffi into C code. To use it, + # subclass the 'distutils.command.build_ext.build_ext' class and + # add a method 'def pre_run(self, ext, ffi)'. + if pre_run is not None: + pre_run(ext, ffi) + updated = recompiler.make_c_source(ffi, module_name, source, c_file) + if not updated: + log.info("already up-to-date") + return c_file + + if dist.ext_modules is None: + dist.ext_modules = [] + dist.ext_modules.append(ext) + + base_class = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class): + def run(self): + if ext.sources[0] == '$PLACEHOLDER': + pre_run = getattr(self, 'pre_run', None) + ext.sources[0] = make_mod(self.build_temp, pre_run) + base_class.run(self) + dist.cmdclass['build_ext'] = build_ext_make_mod + # NB. multiple runs here will create multiple 'build_ext_make_mod' + # classes. Even in this case the 'build_ext' command should be + # run once; but just in case, the logic above does nothing if + # called again. + + +def _add_py_module(dist, ffi, module_name): + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext + from cffi._shimmed_dist_utils import log, mkpath + from cffi import recompiler + + def generate_mod(py_file): + log.info("generating cffi module %r" % py_file) + mkpath(os.path.dirname(py_file)) + updated = recompiler.make_py_source(ffi, module_name, py_file) + if not updated: + log.info("already up-to-date") + + base_class = dist.cmdclass.get('build_py', build_py) + class build_py_make_mod(base_class): + def run(self): + base_class.run(self) + module_path = module_name.split('.') + module_path[-1] += '.py' + generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules + dist.cmdclass['build_py'] = build_py_make_mod + + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + # Then we need to hack more in get_source_files(); see above. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + + # the following is only for "build_ext -i" + base_class_2 = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class_2): + def run(self): + base_class_2.run(self) + if self.inplace: + # from get_ext_fullpath() in distutils/command/build_ext.py + module_path = module_name.split('.') + package = '.'.join(module_path[:-1]) + build_py = self.get_finalized_command('build_py') + package_dir = build_py.get_package_dir(package) + file_name = module_path[-1] + '.py' + generate_mod(os.path.join(package_dir, file_name)) + dist.cmdclass['build_ext'] = build_ext_make_mod + +def cffi_modules(dist, attr, value): + assert attr == 'cffi_modules' + if isinstance(value, basestring): + value = [value] + + for cffi_module in value: + add_cffi_module(dist, cffi_module) diff --git a/lib/cffi/vengine_cpy.py b/lib/cffi/vengine_cpy.py new file mode 100644 index 0000000..02e6a47 --- /dev/null +++ b/lib/cffi/vengine_cpy.py @@ -0,0 +1,1087 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys +from . import model +from .error import VerificationError +from . import _imp_emulation as imp + + +class VCPythonEngine(object): + _class_key = 'x' + _gen_python_module = True + + def __init__(self, verifier): + self.verifier = verifier + self.ffi = verifier.ffi + self._struct_pending_verification = {} + self._types_of_builtin_functions = {} + + def patch_extension_kwds(self, kwds): + pass + + def find_module(self, module_name, path, so_suffixes): + try: + f, filename, descr = imp.find_module(module_name, path) + except ImportError: + return None + if f is not None: + f.close() + # Note that after a setuptools installation, there are both .py + # and .so files with the same basename. The code here relies on + # imp.find_module() locating the .so in priority. + if descr[0] not in so_suffixes: + return None + return filename + + def collect_types(self): + self._typesdict = {} + self._generate("collecttype") + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def _gettypenum(self, type): + # a KeyError here is a bug. please report it! :-) + return self._typesdict[type] + + def _do_collect_type(self, tp): + if ((not isinstance(tp, model.PrimitiveType) + or tp.name == 'long double') + and tp not in self._typesdict): + num = len(self._typesdict) + self._typesdict[tp] = num + + def write_source_to_f(self): + self.collect_types() + # + # The new module will have a _cffi_setup() function that receives + # objects from the ffi world, and that calls some setup code in + # the module. This setup code is split in several independent + # functions, e.g. one per constant. The functions are "chained" + # by ending in a tail call to each other. + # + # This is further split in two chained lists, depending on if we + # can do it at import-time or if we must wait for _cffi_setup() to + # provide us with the objects. This is needed because we + # need the values of the enum constants in order to build the + # that we may have to pass to _cffi_setup(). + # + # The following two 'chained_list_constants' items contains + # the head of these two chained lists, as a string that gives the + # call to do, if any. + self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)'] + # + prnt = self._prnt + # first paste some standard set of lines that are mostly '#define' + prnt(cffimod_header) + prnt() + # then paste the C source given by the user, verbatim. + prnt(self.verifier.preamble) + prnt() + # + # call generate_cpy_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._generate("decl") + # + # implement the function _cffi_setup_custom() as calling the + # head of the chained list. + self._generate_setup_custom() + prnt() + # + # produce the method table, including the entries for the + # generated Python->C function wrappers, which are done + # by generate_cpy_function_method(). + prnt('static PyMethodDef _cffi_methods[] = {') + self._generate("method") + prnt(' {"_cffi_setup", _cffi_setup, METH_VARARGS, NULL},') + prnt(' {NULL, NULL, 0, NULL} /* Sentinel */') + prnt('};') + prnt() + # + # standard init. + modname = self.verifier.get_module_name() + constants = self._chained_list_constants[False] + prnt('#if PY_MAJOR_VERSION >= 3') + prnt() + prnt('static struct PyModuleDef _cffi_module_def = {') + prnt(' PyModuleDef_HEAD_INIT,') + prnt(' "%s",' % modname) + prnt(' NULL,') + prnt(' -1,') + prnt(' _cffi_methods,') + prnt(' NULL, NULL, NULL, NULL') + prnt('};') + prnt() + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % modname) + prnt('{') + prnt(' PyObject *lib;') + prnt(' lib = PyModule_Create(&_cffi_module_def);') + prnt(' if (lib == NULL)') + prnt(' return NULL;') + prnt(' if (%s < 0 || _cffi_init() < 0) {' % (constants,)) + prnt(' Py_DECREF(lib);') + prnt(' return NULL;') + prnt(' }') + prnt('#if Py_GIL_DISABLED') + prnt(' PyUnstable_Module_SetGIL(lib, Py_MOD_GIL_NOT_USED);') + prnt('#endif') + prnt(' return lib;') + prnt('}') + prnt() + prnt('#else') + prnt() + prnt('PyMODINIT_FUNC') + prnt('init%s(void)' % modname) + prnt('{') + prnt(' PyObject *lib;') + prnt(' lib = Py_InitModule("%s", _cffi_methods);' % modname) + prnt(' if (lib == NULL)') + prnt(' return;') + prnt(' if (%s < 0 || _cffi_init() < 0)' % (constants,)) + prnt(' return;') + prnt(' return;') + prnt('}') + prnt() + prnt('#endif') + + def load_library(self, flags=None): + # XXX review all usages of 'self' here! + # import it as a new extension module + imp.acquire_lock() + try: + if hasattr(sys, "getdlopenflags"): + previous_flags = sys.getdlopenflags() + try: + if hasattr(sys, "setdlopenflags") and flags is not None: + sys.setdlopenflags(flags) + module = imp.load_dynamic(self.verifier.get_module_name(), + self.verifier.modulefilename) + except ImportError as e: + error = "importing %r: %s" % (self.verifier.modulefilename, e) + raise VerificationError(error) + finally: + if hasattr(sys, "setdlopenflags"): + sys.setdlopenflags(previous_flags) + finally: + imp.release_lock() + # + # call loading_cpy_struct() to get the struct layout inferred by + # the C compiler + self._load(module, 'loading') + # + # the C code will need the objects. Collect them in + # order in a list. + revmapping = dict([(value, key) + for (key, value) in self._typesdict.items()]) + lst = [revmapping[i] for i in range(len(revmapping))] + lst = list(map(self.ffi._get_cached_btype, lst)) + # + # build the FFILibrary class and instance and call _cffi_setup(). + # this will set up some fields like '_cffi_types', and only then + # it will invoke the chained list of functions that will really + # build (notably) the constant objects, as if they are + # pointers, and store them as attributes on the 'library' object. + class FFILibrary(object): + _cffi_python_module = module + _cffi_ffi = self.ffi + _cffi_dir = [] + def __dir__(self): + return FFILibrary._cffi_dir + list(self.__dict__) + library = FFILibrary() + if module._cffi_setup(lst, VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) + # + # finally, call the loaded_cpy_xxx() functions. This will perform + # the final adjustments, like copying the Python->C wrapper + # functions from the module to the 'library' object, and setting + # up the FFILibrary class with properties for the global C variables. + self._load(module, 'loaded', library=library) + module._cffi_original_ffi = self.ffi + module._cffi_types_of_builtin_funcs = self._types_of_builtin_functions + return library + + def _get_declarations(self): + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst + + def _generate(self, step_name): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_cpy_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in verify(): %r" % name) + try: + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _load(self, module, step_name, **kwds): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + method = getattr(self, '_%s_cpy_%s' % (step_name, kind)) + try: + method(tp, realname, module, **kwds) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _generate_nothing(self, tp, name): + pass + + def _loaded_noop(self, tp, name, module, **kwds): + pass + + # ---------- + + def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode): + extraarg = '' + if isinstance(tp, model.PrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + converter = '_cffi_to_c_int' + extraarg = ', %s' % tp.name + elif tp.is_complex_type(): + raise VerificationError( + "not implemented in verify(): complex types") + else: + converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''), + tp.name.replace(' ', '_')) + errvalue = '-1' + # + elif isinstance(tp, model.PointerType): + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return + # + elif isinstance(tp, (model.StructOrUnion, model.EnumType)): + # a struct (not a struct pointer) as a function argument + self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' + % (tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + return + # + elif isinstance(tp, model.FunctionPtrType): + converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') + extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) + errvalue = 'NULL' + # + else: + raise NotImplementedError(tp) + # + self._prnt(' %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg)) + self._prnt(' if (%s == (%s)%s && PyErr_Occurred())' % ( + tovar, tp.get_c_name(''), errvalue)) + self._prnt(' %s;' % errcode) + + def _extra_local_variables(self, tp, localvars, freelines): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + localvars.add('struct _cffi_freeme_s *large_args_free = NULL') + freelines.add('if (large_args_free != NULL)' + ' _cffi_free_array_arguments(large_args_free);') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' %s = ((size_t)datasize) <= 640 ? ' + 'alloca((size_t)datasize) : NULL;' % (tovar,)) + self._prnt(' if (_cffi_convert_array_argument(_cffi_type(%d), %s, ' + '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar)) + self._prnt(' datasize, &large_args_free) < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' }') + + def _convert_expr_from_c(self, tp, var, context): + if isinstance(tp, model.PrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + return '_cffi_from_c_int(%s, %s)' % (var, tp.name) + elif tp.name != 'long double': + return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var) + else: + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, (model.PointerType, model.FunctionPtrType)): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.ArrayType): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) + elif isinstance(tp, model.StructOrUnion): + if tp.fldnames is None: + raise TypeError("'%s' is used as %s, but is opaque" % ( + tp._get_c_name(), context)) + return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.EnumType): + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + else: + raise NotImplementedError(tp) + + # ---------- + # typedefs: generates no code so far + + _generate_cpy_typedef_collecttype = _generate_nothing + _generate_cpy_typedef_decl = _generate_nothing + _generate_cpy_typedef_method = _generate_nothing + _loading_cpy_typedef = _loaded_noop + _loaded_cpy_typedef = _loaded_noop + + # ---------- + # function declarations + + def _generate_cpy_function_collecttype(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + self._do_collect_type(tp) + else: + # don't call _do_collect_type(tp) in this common case, + # otherwise test_autofilled_struct_as_argument fails + for type in tp.args: + self._do_collect_type(type) + self._do_collect_type(tp.result) + + def _generate_cpy_function_decl(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no CPython wrapper) + self._generate_cpy_const(False, name, tp) + return + prnt = self._prnt + numargs = len(tp.args) + if numargs == 0: + argname = 'noarg' + elif numargs == 1: + argname = 'arg0' + else: + argname = 'args' + prnt('static PyObject *') + prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname)) + prnt('{') + # + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + prnt(' %s;' % type.get_c_name(' x%d' % i, context)) + # + localvars = set() + freelines = set() + for type in tp.args: + self._extra_local_variables(type, localvars, freelines) + for decl in sorted(localvars): + prnt(' %s;' % (decl,)) + # + if not isinstance(tp.result, model.VoidType): + result_code = 'result = ' + context = 'result of %s' % name + prnt(' %s;' % tp.result.get_c_name(' result', context)) + prnt(' PyObject *pyresult;') + else: + result_code = '' + # + if len(tp.args) > 1: + rng = range(len(tp.args)) + for i in rng: + prnt(' PyObject *arg%d;' % i) + prnt() + prnt(' if (!PyArg_ParseTuple(args, "%s:%s", %s))' % ( + 'O' * numargs, name, ', '.join(['&arg%d' % i for i in rng]))) + prnt(' return NULL;') + prnt() + # + for i, type in enumerate(tp.args): + self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i, + 'return NULL') + prnt() + # + prnt(' Py_BEGIN_ALLOW_THREADS') + prnt(' _cffi_restore_errno();') + prnt(' { %s%s(%s); }' % ( + result_code, name, + ', '.join(['x%d' % i for i in range(len(tp.args))]))) + prnt(' _cffi_save_errno();') + prnt(' Py_END_ALLOW_THREADS') + prnt() + # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') + if result_code: + prnt(' pyresult = %s;' % + self._convert_expr_from_c(tp.result, 'result', 'result type')) + for freeline in freelines: + prnt(' ' + freeline) + prnt(' return pyresult;') + else: + for freeline in freelines: + prnt(' ' + freeline) + prnt(' Py_INCREF(Py_None);') + prnt(' return Py_None;') + prnt('}') + prnt() + + def _generate_cpy_function_method(self, tp, name): + if tp.ellipsis: + return + numargs = len(tp.args) + if numargs == 0: + meth = 'METH_NOARGS' + elif numargs == 1: + meth = 'METH_O' + else: + meth = 'METH_VARARGS' + self._prnt(' {"%s", _cffi_f_%s, %s, NULL},' % (name, name, meth)) + + _loading_cpy_function = _loaded_noop + + def _loaded_cpy_function(self, tp, name, module, library): + if tp.ellipsis: + return + func = getattr(module, name) + setattr(library, name, func) + self._types_of_builtin_functions[func] = tp + + # ---------- + # named structs + + _generate_cpy_struct_collecttype = _generate_nothing + def _generate_cpy_struct_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'struct', name) + def _generate_cpy_struct_method(self, tp, name): + self._generate_struct_or_union_method(tp, 'struct', name) + def _loading_cpy_struct(self, tp, name, module): + self._loading_struct_or_union(tp, 'struct', name, module) + def _loaded_cpy_struct(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + _generate_cpy_union_collecttype = _generate_nothing + def _generate_cpy_union_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'union', name) + def _generate_cpy_union_method(self, tp, name): + self._generate_struct_or_union_method(tp, 'union', name) + def _loading_cpy_union(self, tp, name, module): + self._loading_struct_or_union(tp, 'union', name, module) + def _loaded_cpy_union(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_struct_or_union_decl(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + checkfuncname = '_cffi_check_%s_%s' % (prefix, name) + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + cname = ('%s %s' % (prefix, name)).strip() + # + prnt = self._prnt + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if (isinstance(ftype, model.PrimitiveType) + and ftype.is_integer_type()) or fbitsize >= 0: + # accept all integers, but complain on float or double + prnt(' (void)((p->%s) << 1);' % fname) + else: + # only accept exactly the type declared. + try: + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + prnt('static PyObject *') + prnt('%s(PyObject *self, PyObject *noarg)' % (layoutfuncname,)) + prnt('{') + prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) + prnt(' static Py_ssize_t nums[] = {') + prnt(' sizeof(%s),' % cname) + prnt(' offsetof(struct _cffi_aligncheck, y),') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + prnt(' offsetof(%s, %s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + prnt(' -1') + prnt(' };') + prnt(' (void)self; /* unused */') + prnt(' (void)noarg; /* unused */') + prnt(' return _cffi_get_struct_layout(nums);') + prnt(' /* the next line is not executed, but compiled */') + prnt(' %s(0);' % (checkfuncname,)) + prnt('}') + prnt() + + def _generate_struct_or_union_method(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + self._prnt(' {"%s", %s, METH_NOARGS, NULL},' % (layoutfuncname, + layoutfuncname)) + + def _loading_struct_or_union(self, tp, prefix, name, module): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + # + function = getattr(module, layoutfuncname) + layout = function() + if isinstance(tp, model.StructOrUnion) and tp.partial: + # use the function()'s sizes and offsets to guide the + # layout of the struct + totalsize = layout[0] + totalalignment = layout[1] + fieldofs = layout[2::2] + fieldsize = layout[3::2] + tp.force_flatten() + assert len(fieldofs) == len(fieldsize) == len(tp.fldnames) + tp.fixedlayout = fieldofs, fieldsize, totalsize, totalalignment + else: + cname = ('%s %s' % (prefix, name)).strip() + self._struct_pending_verification[tp] = layout, cname + + def _loaded_struct_or_union(self, tp): + if tp.fldnames is None: + return # nothing to do with opaque structs + self.ffi._get_cached_btype(tp) # force 'fixedlayout' to be considered + + if tp in self._struct_pending_verification: + # check that the layout sizes and offsets match the real ones + def check(realvalue, expectedvalue, msg): + if realvalue != expectedvalue: + raise VerificationError( + "%s (we have %d, but C compiler says %d)" + % (msg, expectedvalue, realvalue)) + ffi = self.ffi + BStruct = ffi._get_cached_btype(tp) + layout, cname = self._struct_pending_verification.pop(tp) + check(layout[0], ffi.sizeof(BStruct), "wrong total size") + check(layout[1], ffi.alignof(BStruct), "wrong total alignment") + i = 2 + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + check(layout[i], ffi.offsetof(BStruct, fname), + "wrong offset for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) + i += 2 + assert i == len(layout) + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + _generate_cpy_anonymous_collecttype = _generate_nothing + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp, name, '') + else: + self._generate_struct_or_union_decl(tp, '', name) + + def _generate_cpy_anonymous_method(self, tp, name): + if not isinstance(tp, model.EnumType): + self._generate_struct_or_union_method(tp, '', name) + + def _loading_cpy_anonymous(self, tp, name, module): + if isinstance(tp, model.EnumType): + self._loading_cpy_enum(tp, name, module) + else: + self._loading_struct_or_union(tp, '', name, module) + + def _loaded_cpy_anonymous(self, tp, name, module, **kwds): + if isinstance(tp, model.EnumType): + self._loaded_cpy_enum(tp, name, module, **kwds) + else: + self._loaded_struct_or_union(tp) + + # ---------- + # constants, likely declared with '#define' + + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + vartp=None, delayed=True, size_too=False, + check_value=None): + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + prnt('static int %s(PyObject *lib)' % funcname) + prnt('{') + prnt(' PyObject *o;') + prnt(' int res;') + if not is_int: + prnt(' %s;' % (vartp or tp).get_c_name(' i', name)) + else: + assert category == 'const' + # + if check_value is not None: + self._check_int_constant_value(name, check_value) + # + if not is_int: + if category == 'var': + realexpr = '&' + name + else: + realexpr = name + prnt(' i = (%s);' % (realexpr,)) + prnt(' o = %s;' % (self._convert_expr_from_c(tp, 'i', + 'variable type'),)) + assert delayed + else: + prnt(' o = _cffi_from_c_int_const(%s);' % name) + prnt(' if (o == NULL)') + prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') + prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) + prnt(' Py_DECREF(o);') + prnt(' if (res < 0)') + prnt(' return -1;') + prnt(' return %s;' % self._chained_list_constants[delayed]) + self._chained_list_constants[delayed] = funcname + '(lib)' + prnt('}') + prnt() + + def _generate_cpy_constant_collecttype(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + if not is_int: + self._do_collect_type(tp) + + def _generate_cpy_constant_decl(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + self._generate_cpy_const(is_int, name, tp) + + _generate_cpy_constant_method = _generate_nothing + _loading_cpy_constant = _loaded_noop + _loaded_cpy_constant = _loaded_noop + + # ---------- + # enums + + def _check_int_constant_value(self, name, value, err_prefix=''): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + name) + prnt(' PyErr_Format(_cffi_VerificationError,') + prnt(' "%s%s has the real value %s, not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + err_prefix, name, value)) + prnt(' return -1;') + prnt(' }') + + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + + def _generate_cpy_enum_decl(self, tp, name, prefix='enum'): + if tp.partial: + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator, delayed=False) + return + # + funcname = self._enum_funcname(prefix, name) + prnt = self._prnt + prnt('static int %s(PyObject *lib)' % funcname) + prnt('{') + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._check_int_constant_value(enumerator, enumvalue, + "enum %s: " % name) + prnt(' return %s;' % self._chained_list_constants[True]) + self._chained_list_constants[True] = funcname + '(lib)' + prnt('}') + prnt() + + _generate_cpy_enum_collecttype = _generate_nothing + _generate_cpy_enum_method = _generate_nothing + + def _loading_cpy_enum(self, tp, name, module): + if tp.partial: + enumvalues = [getattr(module, enumerator) + for enumerator in tp.enumerators] + tp.enumvalues = tuple(enumvalues) + tp.partial_resolved = True + + def _loaded_cpy_enum(self, tp, name, module, library): + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + setattr(library, enumerator, enumvalue) + + # ---------- + # macros: for now only for integers + + def _generate_cpy_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) + + _generate_cpy_macro_collecttype = _generate_nothing + _generate_cpy_macro_method = _generate_nothing + _loading_cpy_macro = _loaded_noop + _loaded_cpy_macro = _loaded_noop + + # ---------- + # global variables + + def _generate_cpy_variable_collecttype(self, tp, name): + if isinstance(tp, model.ArrayType): + tp_ptr = model.PointerType(tp.item) + else: + tp_ptr = model.PointerType(tp) + self._do_collect_type(tp_ptr) + + def _generate_cpy_variable_decl(self, tp, name): + if isinstance(tp, model.ArrayType): + tp_ptr = model.PointerType(tp.item) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = tp.length_is_unknown()) + else: + tp_ptr = model.PointerType(tp) + self._generate_cpy_const(False, name, tp_ptr, category='var') + + _generate_cpy_variable_method = _generate_nothing + _loading_cpy_variable = _loaded_noop + + def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) + if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the + # sense that "a=..." is forbidden + if tp.length_is_unknown(): + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return + # remove ptr= from the library instance, and replace + # it by a property on the class, which reads/writes into ptr[0]. + ptr = value + delattr(library, name) + def getter(library): + return ptr[0] + def setter(library, value): + ptr[0] = value + setattr(type(library), name, property(getter, setter)) + type(library)._cffi_dir.append(name) + + # ---------- + + def _generate_setup_custom(self): + prnt = self._prnt + prnt('static int _cffi_setup_custom(PyObject *lib)') + prnt('{') + prnt(' return %s;' % self._chained_list_constants[True]) + prnt('}') + +cffimod_header = r''' +#include +#include + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +# define _cffi_float_complex_t _Fcomplex /* include for it */ +# define _cffi_double_complex_t _Dcomplex /* include for it */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +# define _cffi_float_complex_t float _Complex +# define _cffi_double_complex_t double _Complex +#endif + +#if PY_MAJOR_VERSION < 3 +# undef PyCapsule_CheckExact +# undef PyCapsule_GetPointer +# define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) +# define PyCapsule_GetPointer(capsule, name) \ + (PyCObject_AsVoidPtr(capsule)) +#endif + +#if PY_MAJOR_VERSION >= 3 +# define PyInt_FromLong PyLong_FromLong +#endif + +#define _cffi_from_c_double PyFloat_FromDouble +#define _cffi_from_c_float PyFloat_FromDouble +#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_ulong PyLong_FromUnsignedLong +#define _cffi_from_c_longlong PyLong_FromLongLong +#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong +#define _cffi_from_c__Bool PyBool_FromLong + +#define _cffi_to_c_double PyFloat_AsDouble +#define _cffi_to_c_float PyFloat_AsDouble + +#define _cffi_from_c_int_const(x) \ + (((x) > 0) ? \ + ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ + PyInt_FromLong((long)(x)) : \ + PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ + ((long long)(x) >= (long long)LONG_MIN) ? \ + PyInt_FromLong((long)(x)) : \ + PyLong_FromLongLong((long long)(x))) + +#define _cffi_from_c_int(x, type) \ + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) + +#define _cffi_to_c_int(o, type) \ + ((type)( \ + sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ + (Py_FatalError("unsupported size for type " #type), (type)0))) + +#define _cffi_to_c_i8 \ + ((int(*)(PyObject *))_cffi_exports[1]) +#define _cffi_to_c_u8 \ + ((int(*)(PyObject *))_cffi_exports[2]) +#define _cffi_to_c_i16 \ + ((int(*)(PyObject *))_cffi_exports[3]) +#define _cffi_to_c_u16 \ + ((int(*)(PyObject *))_cffi_exports[4]) +#define _cffi_to_c_i32 \ + ((int(*)(PyObject *))_cffi_exports[5]) +#define _cffi_to_c_u32 \ + ((unsigned int(*)(PyObject *))_cffi_exports[6]) +#define _cffi_to_c_i64 \ + ((long long(*)(PyObject *))_cffi_exports[7]) +#define _cffi_to_c_u64 \ + ((unsigned long long(*)(PyObject *))_cffi_exports[8]) +#define _cffi_to_c_char \ + ((int(*)(PyObject *))_cffi_exports[9]) +#define _cffi_from_c_pointer \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10]) +#define _cffi_to_c_pointer \ + ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11]) +#define _cffi_get_struct_layout \ + ((PyObject *(*)(Py_ssize_t[]))_cffi_exports[12]) +#define _cffi_restore_errno \ + ((void(*)(void))_cffi_exports[13]) +#define _cffi_save_errno \ + ((void(*)(void))_cffi_exports[14]) +#define _cffi_from_c_char \ + ((PyObject *(*)(char))_cffi_exports[15]) +#define _cffi_from_c_deref \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16]) +#define _cffi_to_c \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17]) +#define _cffi_from_c_struct \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18]) +#define _cffi_to_c_wchar_t \ + ((wchar_t(*)(PyObject *))_cffi_exports[19]) +#define _cffi_from_c_wchar_t \ + ((PyObject *(*)(wchar_t))_cffi_exports[20]) +#define _cffi_to_c_long_double \ + ((long double(*)(PyObject *))_cffi_exports[21]) +#define _cffi_to_c__Bool \ + ((_Bool(*)(PyObject *))_cffi_exports[22]) +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) +#define _CFFI_NUM_EXPORTS 25 + +typedef struct _ctypedescr CTypeDescrObject; + +static void *_cffi_exports[_CFFI_NUM_EXPORTS]; +static PyObject *_cffi_types, *_cffi_VerificationError; + +static int _cffi_setup_custom(PyObject *lib); /* forward */ + +static PyObject *_cffi_setup(PyObject *self, PyObject *args) +{ + PyObject *library; + int was_alive = (_cffi_types != NULL); + (void)self; /* unused */ + if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, + &library)) + return NULL; + Py_INCREF(_cffi_types); + Py_INCREF(_cffi_VerificationError); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); +} + +union _cffi_union_alignment_u { + unsigned char m_char; + unsigned short m_short; + unsigned int m_int; + unsigned long m_long; + unsigned long long m_longlong; + float m_float; + double m_double; + long double m_longdouble; +}; + +struct _cffi_freeme_s { + struct _cffi_freeme_s *next; + union _cffi_union_alignment_u alignment; +}; + +#ifdef __GNUC__ + __attribute__((unused)) +#endif +static int _cffi_convert_array_argument(CTypeDescrObject *ctptr, PyObject *arg, + char **output_data, Py_ssize_t datasize, + struct _cffi_freeme_s **freeme) +{ + char *p; + if (datasize < 0) + return -1; + + p = *output_data; + if (p == NULL) { + struct _cffi_freeme_s *fp = (struct _cffi_freeme_s *)PyObject_Malloc( + offsetof(struct _cffi_freeme_s, alignment) + (size_t)datasize); + if (fp == NULL) + return -1; + fp->next = *freeme; + *freeme = fp; + p = *output_data = (char *)&fp->alignment; + } + memset((void *)p, 0, (size_t)datasize); + return _cffi_convert_array_from_object(p, ctptr, arg); +} + +#ifdef __GNUC__ + __attribute__((unused)) +#endif +static void _cffi_free_array_arguments(struct _cffi_freeme_s *freeme) +{ + do { + void *p = (void *)freeme; + freeme = freeme->next; + PyObject_Free(p); + } while (freeme != NULL); +} + +static int _cffi_init(void) +{ + PyObject *module, *c_api_object = NULL; + + module = PyImport_ImportModule("_cffi_backend"); + if (module == NULL) + goto failure; + + c_api_object = PyObject_GetAttrString(module, "_C_API"); + if (c_api_object == NULL) + goto failure; + if (!PyCapsule_CheckExact(c_api_object)) { + PyErr_SetNone(PyExc_ImportError); + goto failure; + } + memcpy(_cffi_exports, PyCapsule_GetPointer(c_api_object, "cffi"), + _CFFI_NUM_EXPORTS * sizeof(void *)); + + Py_DECREF(module); + Py_DECREF(c_api_object); + return 0; + + failure: + Py_XDECREF(module); + Py_XDECREF(c_api_object); + return -1; +} + +#define _cffi_type(num) ((CTypeDescrObject *)PyList_GET_ITEM(_cffi_types, num)) + +/**********/ +''' diff --git a/lib/cffi/vengine_gen.py b/lib/cffi/vengine_gen.py new file mode 100644 index 0000000..bffc821 --- /dev/null +++ b/lib/cffi/vengine_gen.py @@ -0,0 +1,679 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, os +import types + +from . import model +from .error import VerificationError + + +class VGenericEngine(object): + _class_key = 'g' + _gen_python_module = False + + def __init__(self, verifier): + self.verifier = verifier + self.ffi = verifier.ffi + self.export_symbols = [] + self._struct_pending_verification = {} + + def patch_extension_kwds(self, kwds): + # add 'export_symbols' to the dictionary. Note that we add the + # list before filling it. When we fill it, it will thus also show + # up in kwds['export_symbols']. + kwds.setdefault('export_symbols', self.export_symbols) + + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename + + def collect_types(self): + pass # not needed in the generic engine + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def write_source_to_f(self): + prnt = self._prnt + # first paste some standard set of lines that are mostly '#include' + prnt(cffimod_header) + # then paste the C source given by the user, verbatim. + prnt(self.verifier.preamble) + # + # call generate_gen_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._generate('decl') + # + # on Windows, distutils insists on putting init_cffi_xyz in + # 'export_symbols', so instead of fighting it, just give up and + # give it one + if sys.platform == 'win32': + if sys.version_info >= (3,): + prefix = 'PyInit_' + else: + prefix = 'init' + modname = self.verifier.get_module_name() + prnt("void %s%s(void) { }\n" % (prefix, modname)) + + def load_library(self, flags=0): + # import it with the CFFI backend + backend = self.ffi._backend + # needs to make a path that contains '/', on Posix + filename = os.path.join(os.curdir, self.verifier.modulefilename) + module = backend.load_library(filename, flags) + # + # call loading_gen_struct() to get the struct layout inferred by + # the C compiler + self._load(module, 'loading') + + # build the FFILibrary class and instance, this is a module subclass + # because modules are expected to have usually-constant-attributes and + # in PyPy this means the JIT is able to treat attributes as constant, + # which we want. + class FFILibrary(types.ModuleType): + _cffi_generic_module = module + _cffi_ffi = self.ffi + _cffi_dir = [] + def __dir__(self): + return FFILibrary._cffi_dir + library = FFILibrary("") + # + # finally, call the loaded_gen_xxx() functions. This will set + # up the 'library' object. + self._load(module, 'loaded', library=library) + return library + + def _get_declarations(self): + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst + + def _generate(self, step_name): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_gen_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in verify(): %r" % name) + try: + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _load(self, module, step_name, **kwds): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + method = getattr(self, '_%s_gen_%s' % (step_name, kind)) + try: + method(tp, realname, module, **kwds) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _generate_nothing(self, tp, name): + pass + + def _loaded_noop(self, tp, name, module, **kwds): + pass + + # ---------- + # typedefs: generates no code so far + + _generate_gen_typedef_decl = _generate_nothing + _loading_gen_typedef = _loaded_noop + _loaded_gen_typedef = _loaded_noop + + # ---------- + # function declarations + + def _generate_gen_function_decl(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no _cffi_f_%s wrapper) + self._generate_gen_const(False, name, tp) + return + prnt = self._prnt + numargs = len(tp.args) + argnames = [] + for i, type in enumerate(tp.args): + indirection = '' + if isinstance(type, model.StructOrUnion): + indirection = '*' + argnames.append('%sx%d' % (indirection, i)) + context = 'argument of %s' % name + arglist = [type.get_c_name(' %s' % arg, context) + for type, arg in zip(tp.args, argnames)] + tpresult = tp.result + if isinstance(tpresult, model.StructOrUnion): + arglist.insert(0, tpresult.get_c_name(' *r', context)) + tpresult = model.void_type + arglist = ', '.join(arglist) or 'void' + wrappername = '_cffi_f_%s' % name + self.export_symbols.append(wrappername) + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + funcdecl = ' %s%s(%s)' % (abi, wrappername, arglist) + context = 'result of %s' % name + prnt(tpresult.get_c_name(funcdecl, context)) + prnt('{') + # + if isinstance(tp.result, model.StructOrUnion): + result_code = '*r = ' + elif not isinstance(tp.result, model.VoidType): + result_code = 'return ' + else: + result_code = '' + prnt(' %s%s(%s);' % (result_code, name, ', '.join(argnames))) + prnt('}') + prnt() + + _loading_gen_function = _loaded_noop + + def _loaded_gen_function(self, tp, name, module, library): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + newfunction = self._load_constant(False, tp, name, module) + else: + indirections = [] + base_tp = tp + if (any(isinstance(typ, model.StructOrUnion) for typ in tp.args) + or isinstance(tp.result, model.StructOrUnion)): + indirect_args = [] + for i, typ in enumerate(tp.args): + if isinstance(typ, model.StructOrUnion): + typ = model.PointerType(typ) + indirections.append((i, typ)) + indirect_args.append(typ) + indirect_result = tp.result + if isinstance(indirect_result, model.StructOrUnion): + if indirect_result.fldtypes is None: + raise TypeError("'%s' is used as result type, " + "but is opaque" % ( + indirect_result._get_c_name(),)) + indirect_result = model.PointerType(indirect_result) + indirect_args.insert(0, indirect_result) + indirections.insert(0, ("result", indirect_result)) + indirect_result = model.void_type + tp = model.FunctionPtrType(tuple(indirect_args), + indirect_result, tp.ellipsis) + BFunc = self.ffi._get_cached_btype(tp) + wrappername = '_cffi_f_%s' % name + newfunction = module.load_function(BFunc, wrappername) + for i, typ in indirections: + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) + setattr(library, name, newfunction) + type(library)._cffi_dir.append(name) + + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): + backend = self.ffi._backend + BType = self.ffi._get_cached_btype(tp) + if i == "result": + ffi = self.ffi + def newfunc(*args): + res = ffi.new(BType) + oldfunc(res, *args) + return res[0] + else: + def newfunc(*args): + args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] + return oldfunc(*args) + newfunc._cffi_base_type = base_tp + return newfunc + + # ---------- + # named structs + + def _generate_gen_struct_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'struct', name) + + def _loading_gen_struct(self, tp, name, module): + self._loading_struct_or_union(tp, 'struct', name, module) + + def _loaded_gen_struct(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_gen_union_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'union', name) + + def _loading_gen_union(self, tp, name, module): + self._loading_struct_or_union(tp, 'union', name, module) + + def _loaded_gen_union(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_struct_or_union_decl(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + checkfuncname = '_cffi_check_%s_%s' % (prefix, name) + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + cname = ('%s %s' % (prefix, name)).strip() + # + prnt = self._prnt + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if (isinstance(ftype, model.PrimitiveType) + and ftype.is_integer_type()) or fbitsize >= 0: + # accept all integers, but complain on float or double + prnt(' (void)((p->%s) << 1);' % fname) + else: + # only accept exactly the type declared. + try: + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + self.export_symbols.append(layoutfuncname) + prnt('intptr_t %s(intptr_t i)' % (layoutfuncname,)) + prnt('{') + prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) + prnt(' static intptr_t nums[] = {') + prnt(' sizeof(%s),' % cname) + prnt(' offsetof(struct _cffi_aligncheck, y),') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + prnt(' offsetof(%s, %s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + prnt(' -1') + prnt(' };') + prnt(' return nums[i];') + prnt(' /* the next line is not executed, but compiled */') + prnt(' %s(0);' % (checkfuncname,)) + prnt('}') + prnt() + + def _loading_struct_or_union(self, tp, prefix, name, module): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + # + BFunc = self.ffi._typeof_locked("intptr_t(*)(intptr_t)")[0] + function = module.load_function(BFunc, layoutfuncname) + layout = [] + num = 0 + while True: + x = function(num) + if x < 0: break + layout.append(x) + num += 1 + if isinstance(tp, model.StructOrUnion) and tp.partial: + # use the function()'s sizes and offsets to guide the + # layout of the struct + totalsize = layout[0] + totalalignment = layout[1] + fieldofs = layout[2::2] + fieldsize = layout[3::2] + tp.force_flatten() + assert len(fieldofs) == len(fieldsize) == len(tp.fldnames) + tp.fixedlayout = fieldofs, fieldsize, totalsize, totalalignment + else: + cname = ('%s %s' % (prefix, name)).strip() + self._struct_pending_verification[tp] = layout, cname + + def _loaded_struct_or_union(self, tp): + if tp.fldnames is None: + return # nothing to do with opaque structs + self.ffi._get_cached_btype(tp) # force 'fixedlayout' to be considered + + if tp in self._struct_pending_verification: + # check that the layout sizes and offsets match the real ones + def check(realvalue, expectedvalue, msg): + if realvalue != expectedvalue: + raise VerificationError( + "%s (we have %d, but C compiler says %d)" + % (msg, expectedvalue, realvalue)) + ffi = self.ffi + BStruct = ffi._get_cached_btype(tp) + layout, cname = self._struct_pending_verification.pop(tp) + check(layout[0], ffi.sizeof(BStruct), "wrong total size") + check(layout[1], ffi.alignof(BStruct), "wrong total alignment") + i = 2 + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + check(layout[i], ffi.offsetof(BStruct, fname), + "wrong offset for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) + i += 2 + assert i == len(layout) + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_gen_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_gen_enum_decl(tp, name, '') + else: + self._generate_struct_or_union_decl(tp, '', name) + + def _loading_gen_anonymous(self, tp, name, module): + if isinstance(tp, model.EnumType): + self._loading_gen_enum(tp, name, module, '') + else: + self._loading_struct_or_union(tp, '', name, module) + + def _loaded_gen_anonymous(self, tp, name, module, **kwds): + if isinstance(tp, model.EnumType): + self._loaded_gen_enum(tp, name, module, **kwds) + else: + self._loaded_struct_or_union(tp) + + # ---------- + # constants, likely declared with '#define' + + def _generate_gen_const(self, is_int, name, tp=None, category='const', + check_value=None): + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + self.export_symbols.append(funcname) + if check_value is not None: + assert is_int + assert category == 'const' + prnt('int %s(char *out_error)' % funcname) + prnt('{') + self._check_int_constant_value(name, check_value) + prnt(' return 0;') + prnt('}') + elif is_int: + assert category == 'const' + prnt('int %s(long long *out_value)' % funcname) + prnt('{') + prnt(' *out_value = (long long)(%s);' % (name,)) + prnt(' return (%s) <= 0;' % (name,)) + prnt('}') + else: + assert tp is not None + assert check_value is None + if category == 'var': + ampersand = '&' + else: + ampersand = '' + extra = '' + if category == 'const' and isinstance(tp, model.StructOrUnion): + extra = 'const *' + ampersand = '&' + prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name)) + prnt('{') + prnt(' return (%s%s);' % (ampersand, name)) + prnt('}') + prnt() + + def _generate_gen_constant_decl(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + self._generate_gen_const(is_int, name, tp) + + _loading_gen_constant = _loaded_noop + + def _load_constant(self, is_int, tp, name, module, check_value=None): + funcname = '_cffi_const_%s' % name + if check_value is not None: + assert is_int + self._load_known_int_constant(module, funcname) + value = check_value + elif is_int: + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType) + negative = function(p) + value = int(p[0]) + if value < 0 and not negative: + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) + else: + assert check_value is None + fntypeextra = '(*)(void)' + if isinstance(tp, model.StructOrUnion): + fntypeextra = '*' + fntypeextra + BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0] + function = module.load_function(BFunc, funcname) + value = function() + if isinstance(tp, model.StructOrUnion): + value = value[0] + return value + + def _loaded_gen_constant(self, tp, name, module, library): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + value = self._load_constant(is_int, tp, name, module) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + + # ---------- + # enums + + def _check_int_constant_value(self, name, value): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' sprintf(buf, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' sprintf(buf, "%%lu", (unsigned long)(%s));' % + name) + prnt(' sprintf(out_error, "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % (name[:100], value)) + prnt(' return -1;') + prnt(' }') + + def _load_known_int_constant(self, module, funcname): + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType, 256) + if function(p) < 0: + error = self.ffi.string(p) + if sys.version_info >= (3,): + error = str(error, 'utf-8') + raise VerificationError(error) + + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + + def _generate_gen_enum_decl(self, tp, name, prefix='enum'): + if tp.partial: + for enumerator in tp.enumerators: + self._generate_gen_const(True, enumerator) + return + # + funcname = self._enum_funcname(prefix, name) + self.export_symbols.append(funcname) + prnt = self._prnt + prnt('int %s(char *out_error)' % funcname) + prnt('{') + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._check_int_constant_value(enumerator, enumvalue) + prnt(' return 0;') + prnt('}') + prnt() + + def _loading_gen_enum(self, tp, name, module, prefix='enum'): + if tp.partial: + enumvalues = [self._load_constant(True, tp, enumerator, module) + for enumerator in tp.enumerators] + tp.enumvalues = tuple(enumvalues) + tp.partial_resolved = True + else: + funcname = self._enum_funcname(prefix, name) + self._load_known_int_constant(module, funcname) + + def _loaded_gen_enum(self, tp, name, module, library): + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + setattr(library, enumerator, enumvalue) + type(library)._cffi_dir.append(enumerator) + + # ---------- + # macros: for now only for integers + + def _generate_gen_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_gen_const(True, name, check_value=check_value) + + _loading_gen_macro = _loaded_noop + + def _loaded_gen_macro(self, tp, name, module, library): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + value = self._load_constant(True, tp, name, module, + check_value=check_value) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + + # ---------- + # global variables + + def _generate_gen_variable_decl(self, tp, name): + if isinstance(tp, model.ArrayType): + if tp.length_is_unknown(): + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") + tp_ptr = model.PointerType(tp.item) + self._generate_gen_const(False, name, tp_ptr) + else: + tp_ptr = model.PointerType(tp) + self._generate_gen_const(False, name, tp_ptr, category='var') + + _loading_gen_variable = _loaded_noop + + def _loaded_gen_variable(self, tp, name, module, library): + if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the + # sense that "a=..." is forbidden + if tp.length_is_unknown(): + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + tp_ptr = model.PointerType(tp.item) + value = self._load_constant(False, tp_ptr, name, module) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + return + # remove ptr= from the library instance, and replace + # it by a property on the class, which reads/writes into ptr[0]. + funcname = '_cffi_var_%s' % name + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] + function = module.load_function(BFunc, funcname) + ptr = function() + def getter(library): + return ptr[0] + def setter(library, value): + ptr[0] = value + setattr(type(library), name, property(getter, setter)) + type(library)._cffi_dir.append(name) + +cffimod_header = r''' +#include +#include +#include +#include +#include /* XXX for ssize_t on some platforms */ + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +# define _cffi_float_complex_t _Fcomplex /* include for it */ +# define _cffi_double_complex_t _Dcomplex /* include for it */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +# define _cffi_float_complex_t float _Complex +# define _cffi_double_complex_t double _Complex +#endif +''' diff --git a/lib/cffi/verifier.py b/lib/cffi/verifier.py new file mode 100644 index 0000000..e392a2b --- /dev/null +++ b/lib/cffi/verifier.py @@ -0,0 +1,306 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, os, binascii, shutil, io +from . import __version_verifier_modules__ +from . import ffiplatform +from .error import VerificationError + +if sys.version_info >= (3, 3): + import importlib.machinery + def _extension_suffixes(): + return importlib.machinery.EXTENSION_SUFFIXES[:] +else: + import imp + def _extension_suffixes(): + return [suffix for suffix, _, type in imp.get_suffixes() + if type == imp.C_EXTENSION] + + +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + + +class Verifier(object): + + def __init__(self, ffi, preamble, tmpdir=None, modulename=None, + ext_package=None, tag='', force_generic_engine=False, + source_extension='.c', flags=None, relative_to=None, **kwds): + if ffi._parser._uses_new_feature: + raise VerificationError( + "feature not supported with ffi.verify(), but only " + "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,)) + self.ffi = ffi + self.preamble = preamble + if not modulename: + flattened_kwds = ffiplatform.flatten(kwds) + vengine_class = _locate_engine_class(ffi, force_generic_engine) + self._vengine = vengine_class(self) + self._vengine.patch_extension_kwds(kwds) + self.flags = flags + self.kwds = self.make_relative_to(kwds, relative_to) + # + if modulename: + if tag: + raise TypeError("can't specify both 'modulename' and 'tag'") + else: + key = '\x00'.join(['%d.%d' % sys.version_info[:2], + __version_verifier_modules__, + preamble, flattened_kwds] + + ffi._cdefsources) + if sys.version_info >= (3,): + key = key.encode('utf-8') + k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) + k1 = k1.lstrip('0x').rstrip('L') + k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) + k2 = k2.lstrip('0').rstrip('L') + modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, + k1, k2) + suffix = _get_so_suffixes()[0] + self.tmpdir = tmpdir or _caller_dir_pycache() + self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension) + self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) + self.ext_package = ext_package + self._has_source = False + self._has_module = False + + def write_source(self, file=None): + """Write the C source code. It is produced in 'self.sourcefilename', + which can be tweaked beforehand.""" + with self.ffi._lock: + if self._has_source and file is None: + raise VerificationError( + "source code already written") + self._write_source(file) + + def compile_module(self): + """Write the C source code (if not done already) and compile it. + This produces a dynamic link library in 'self.modulefilename'.""" + with self.ffi._lock: + if self._has_module: + raise VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() + + def load_library(self): + """Get a C module from this Verifier instance. + Returns an instance of a FFILibrary class that behaves like the + objects returned by ffi.dlopen(), but that delegates all + operations to the C module. If necessary, the C code is written + and compiled first. + """ + with self.ffi._lock: + if not self._has_module: + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() + + def get_module_name(self): + basename = os.path.basename(self.modulefilename) + # kill both the .so extension and the other .'s, as introduced + # by Python 3: 'basename.cpython-33m.so' + basename = basename.split('.', 1)[0] + # and the _d added in Python 2 debug builds --- but try to be + # conservative and not kill a legitimate _d + if basename.endswith('_d') and hasattr(sys, 'gettotalrefcount'): + basename = basename[:-2] + return basename + + def get_extension(self): + if not self._has_source: + with self.ffi._lock: + if not self._has_source: + self._write_source() + sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) + modname = self.get_module_name() + return ffiplatform.get_extension(sourcename, modname, **self.kwds) + + def generates_python_module(self): + return self._vengine._gen_python_module + + def make_relative_to(self, kwds, relative_to): + if relative_to and os.path.dirname(relative_to): + dirname = os.path.dirname(relative_to) + kwds = kwds.copy() + for key in ffiplatform.LIST_OF_FILE_NAMES: + if key in kwds: + lst = kwds[key] + if not isinstance(lst, (list, tuple)): + raise TypeError("keyword '%s' should be a list or tuple" + % (key,)) + lst = [os.path.join(dirname, fn) for fn in lst] + kwds[key] = lst + return kwds + + # ---------- + + def _locate_module(self): + if not os.path.isfile(self.modulefilename): + if self.ext_package: + try: + pkg = __import__(self.ext_package, None, None, ['__doc__']) + except ImportError: + return # cannot import the package itself, give up + # (e.g. it might be called differently before installation) + path = pkg.__path__ + else: + path = None + filename = self._vengine.find_module(self.get_module_name(), path, + _get_so_suffixes()) + if filename is None: + return + self.modulefilename = filename + self._vengine.collect_types() + self._has_module = True + + def _write_source_to(self, file): + self._vengine._f = file + try: + self._vengine.write_source_to_f() + finally: + del self._vengine._f + + def _write_source(self, file=None): + if file is not None: + self._write_source_to(file) + else: + # Write our source file to an in memory file. + f = NativeIO() + self._write_source_to(f) + source_data = f.getvalue() + + # Determine if this matches the current file + if os.path.exists(self.sourcefilename): + with open(self.sourcefilename, "r") as fp: + needs_written = not (fp.read() == source_data) + else: + needs_written = True + + # Actually write the file out if it doesn't match + if needs_written: + _ensure_dir(self.sourcefilename) + with open(self.sourcefilename, "w") as fp: + fp.write(source_data) + + # Set this flag + self._has_source = True + + def _compile_module(self): + # compile this C source + tmpdir = os.path.dirname(self.sourcefilename) + outputfilename = ffiplatform.compile(tmpdir, self.get_extension()) + try: + same = ffiplatform.samefile(outputfilename, self.modulefilename) + except OSError: + same = False + if not same: + _ensure_dir(self.modulefilename) + shutil.move(outputfilename, self.modulefilename) + self._has_module = True + + def _load_library(self): + assert self._has_module + if self.flags is not None: + return self._vengine.load_library(self.flags) + else: + return self._vengine.load_library() + +# ____________________________________________________________ + +_FORCE_GENERIC_ENGINE = False # for tests + +def _locate_engine_class(ffi, force_generic_engine): + if _FORCE_GENERIC_ENGINE: + force_generic_engine = True + if not force_generic_engine: + if '__pypy__' in sys.builtin_module_names: + force_generic_engine = True + else: + try: + import _cffi_backend + except ImportError: + _cffi_backend = '?' + if ffi._backend is not _cffi_backend: + force_generic_engine = True + if force_generic_engine: + from . import vengine_gen + return vengine_gen.VGenericEngine + else: + from . import vengine_cpy + return vengine_cpy.VCPythonEngine + +# ____________________________________________________________ + +_TMPDIR = None + +def _caller_dir_pycache(): + if _TMPDIR: + return _TMPDIR + result = os.environ.get('CFFI_TMPDIR') + if result: + return result + filename = sys._getframe(2).f_code.co_filename + return os.path.abspath(os.path.join(os.path.dirname(filename), + '__pycache__')) + +def set_tmpdir(dirname): + """Set the temporary directory to use instead of __pycache__.""" + global _TMPDIR + _TMPDIR = dirname + +def cleanup_tmpdir(tmpdir=None, keep_so=False): + """Clean up the temporary directory by removing all files in it + called `_cffi_*.{c,so}` as well as the `build` subdirectory.""" + tmpdir = tmpdir or _caller_dir_pycache() + try: + filelist = os.listdir(tmpdir) + except OSError: + return + if keep_so: + suffix = '.c' # only remove .c files + else: + suffix = _get_so_suffixes()[0].lower() + for fn in filelist: + if fn.lower().startswith('_cffi_') and ( + fn.lower().endswith(suffix) or fn.lower().endswith('.c')): + try: + os.unlink(os.path.join(tmpdir, fn)) + except OSError: + pass + clean_dir = [os.path.join(tmpdir, 'build')] + for dir in clean_dir: + try: + for fn in os.listdir(dir): + fn = os.path.join(dir, fn) + if os.path.isdir(fn): + clean_dir.append(fn) + else: + os.unlink(fn) + except OSError: + pass + +def _get_so_suffixes(): + suffixes = _extension_suffixes() + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes + +def _ensure_dir(filename): + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib/cryptography-46.0.5.dist-info/INSTALLER b/lib/cryptography-46.0.5.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/cryptography-46.0.5.dist-info/METADATA b/lib/cryptography-46.0.5.dist-info/METADATA new file mode 100644 index 0000000..15080bb --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/METADATA @@ -0,0 +1,139 @@ +Metadata-Version: 2.4 +Name: cryptography +Version: 46.0.5 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: POSIX +Classifier: Operating System :: POSIX :: BSD +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable +Classifier: Topic :: Security :: Cryptography +Requires-Dist: cffi>=1.14 ; python_full_version == '3.8.*' and platform_python_implementation != 'PyPy' +Requires-Dist: cffi>=2.0.0 ; python_full_version >= '3.9' and platform_python_implementation != 'PyPy' +Requires-Dist: typing-extensions>=4.13.2 ; python_full_version < '3.11' +Requires-Dist: bcrypt>=3.1.5 ; extra == 'ssh' +Requires-Dist: nox[uv]>=2024.4.15 ; extra == 'nox' +Requires-Dist: cryptography-vectors==46.0.5 ; extra == 'test' +Requires-Dist: pytest>=7.4.0 ; extra == 'test' +Requires-Dist: pytest-benchmark>=4.0 ; extra == 'test' +Requires-Dist: pytest-cov>=2.10.1 ; extra == 'test' +Requires-Dist: pytest-xdist>=3.5.0 ; extra == 'test' +Requires-Dist: pretend>=0.7 ; extra == 'test' +Requires-Dist: certifi>=2024 ; extra == 'test' +Requires-Dist: pytest-randomly ; extra == 'test-randomorder' +Requires-Dist: sphinx>=5.3.0 ; extra == 'docs' +Requires-Dist: sphinx-rtd-theme>=3.0.0 ; extra == 'docs' +Requires-Dist: sphinx-inline-tabs ; extra == 'docs' +Requires-Dist: pyenchant>=3 ; extra == 'docstest' +Requires-Dist: readme-renderer>=30.0 ; extra == 'docstest' +Requires-Dist: sphinxcontrib-spelling>=7.3.1 ; extra == 'docstest' +Requires-Dist: build>=1.0.0 ; extra == 'sdist' +Requires-Dist: ruff>=0.11.11 ; extra == 'pep8test' +Requires-Dist: mypy>=1.14 ; extra == 'pep8test' +Requires-Dist: check-sdist ; extra == 'pep8test' +Requires-Dist: click>=8.0.1 ; extra == 'pep8test' +Provides-Extra: ssh +Provides-Extra: nox +Provides-Extra: test +Provides-Extra: test-randomorder +Provides-Extra: docs +Provides-Extra: docstest +Provides-Extra: sdist +Provides-Extra: pep8test +License-File: LICENSE +License-File: LICENSE.APACHE +License-File: LICENSE.BSD +Summary: cryptography is a package which provides cryptographic recipes and primitives to Python developers. +Author-email: The Python Cryptographic Authority and individual contributors +License-Expression: Apache-2.0 OR BSD-3-Clause +Requires-Python: >=3.8, !=3.9.0, !=3.9.1 +Description-Content-Type: text/x-rst; charset=UTF-8 +Project-URL: homepage, https://github.com/pyca/cryptography +Project-URL: documentation, https://cryptography.io/ +Project-URL: source, https://github.com/pyca/cryptography/ +Project-URL: issues, https://github.com/pyca/cryptography/issues +Project-URL: changelog, https://cryptography.io/en/latest/changelog/ + +pyca/cryptography +================= + +.. image:: https://img.shields.io/pypi/v/cryptography.svg + :target: https://pypi.org/project/cryptography/ + :alt: Latest Version + +.. image:: https://readthedocs.org/projects/cryptography/badge/?version=latest + :target: https://cryptography.io + :alt: Latest Docs + +.. image:: https://github.com/pyca/cryptography/actions/workflows/ci.yml/badge.svg + :target: https://github.com/pyca/cryptography/actions/workflows/ci.yml?query=branch%3Amain + +``cryptography`` is a package which provides cryptographic recipes and +primitives to Python developers. Our goal is for it to be your "cryptographic +standard library". It supports Python 3.8+ and PyPy3 7.3.11+. + +``cryptography`` includes both high level recipes and low level interfaces to +common cryptographic algorithms such as symmetric ciphers, message digests, and +key derivation functions. For example, to encrypt something with +``cryptography``'s high level symmetric encryption recipe: + +.. code-block:: pycon + + >>> from cryptography.fernet import Fernet + >>> # Put this somewhere safe! + >>> key = Fernet.generate_key() + >>> f = Fernet(key) + >>> token = f.encrypt(b"A really secret message. Not for prying eyes.") + >>> token + b'...' + >>> f.decrypt(token) + b'A really secret message. Not for prying eyes.' + +You can find more information in the `documentation`_. + +You can install ``cryptography`` with: + +.. code-block:: console + + $ pip install cryptography + +For full details see `the installation documentation`_. + +Discussion +~~~~~~~~~~ + +If you run into bugs, you can file them in our `issue tracker`_. + +We maintain a `cryptography-dev`_ mailing list for development discussion. + +You can also join ``#pyca`` on ``irc.libera.chat`` to ask questions or get +involved. + +Security +~~~~~~~~ + +Need to report a security issue? Please consult our `security reporting`_ +documentation. + + +.. _`documentation`: https://cryptography.io/ +.. _`the installation documentation`: https://cryptography.io/en/latest/installation/ +.. _`issue tracker`: https://github.com/pyca/cryptography/issues +.. _`cryptography-dev`: https://mail.python.org/mailman/listinfo/cryptography-dev +.. _`security reporting`: https://cryptography.io/en/latest/security/ + diff --git a/lib/cryptography-46.0.5.dist-info/RECORD b/lib/cryptography-46.0.5.dist-info/RECORD new file mode 100644 index 0000000..47d1561 --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/RECORD @@ -0,0 +1,180 @@ +cryptography-46.0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +cryptography-46.0.5.dist-info/METADATA,sha256=aOYB9_B-Ccske76ncMz-w9c_VnzYihv_7kxZlt2i2WQ,5748 +cryptography-46.0.5.dist-info/RECORD,, +cryptography-46.0.5.dist-info/WHEEL,sha256=jkxrJemT4jZpYSr-u9xPalWqoow8benNmiXfjKXLlJw,108 +cryptography-46.0.5.dist-info/licenses/LICENSE,sha256=Pgx8CRqUi4JTO6mP18u0BDLW8amsv4X1ki0vmak65rs,197 +cryptography-46.0.5.dist-info/licenses/LICENSE.APACHE,sha256=qsc7MUj20dcRHbyjIJn2jSbGRMaBOuHk8F9leaomY_4,11360 +cryptography-46.0.5.dist-info/licenses/LICENSE.BSD,sha256=YCxMdILeZHndLpeTzaJ15eY9dz2s0eymiSMqtwCPtPs,1532 +cryptography/__about__.py,sha256=GWg4NAxg4vsSKUwmDy1HjUeAOhqTA46wIhiY6i03NSU,445 +cryptography/__init__.py,sha256=mthuUrTd4FROCpUYrTIqhjz6s6T9djAZrV7nZ1oMm2o,364 +cryptography/__pycache__/__about__.cpython-314.pyc,, +cryptography/__pycache__/__init__.cpython-314.pyc,, +cryptography/__pycache__/exceptions.cpython-314.pyc,, +cryptography/__pycache__/fernet.cpython-314.pyc,, +cryptography/__pycache__/utils.cpython-314.pyc,, +cryptography/exceptions.py,sha256=835EWILc2fwxw-gyFMriciC2SqhViETB10LBSytnDIc,1087 +cryptography/fernet.py,sha256=3Cvxkh0KJSbX8HbnCHu4wfCW7U0GgfUA3v_qQ8a8iWc,6963 +cryptography/hazmat/__init__.py,sha256=5IwrLWrVp0AjEr_4FdWG_V057NSJGY_W4egNNsuct0g,455 +cryptography/hazmat/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/__pycache__/_oid.cpython-314.pyc,, +cryptography/hazmat/_oid.py,sha256=p8ThjwJB56Ci_rAIrjyJ1f8VjgD6e39es2dh8JIUBOw,17240 +cryptography/hazmat/asn1/__init__.py,sha256=hS_EWx3wVvZzfbCcNV8hzcDnyMM8H-BhIoS1TipUosk,293 +cryptography/hazmat/asn1/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/asn1/__pycache__/asn1.cpython-314.pyc,, +cryptography/hazmat/asn1/asn1.py,sha256=eMEThEXa19LQjcyVofgHsW6tsZnjp3ddH7bWkkcxfLM,3860 +cryptography/hazmat/backends/__init__.py,sha256=O5jvKFQdZnXhKeqJ-HtulaEL9Ni7mr1mDzZY5kHlYhI,361 +cryptography/hazmat/backends/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/backends/openssl/__init__.py,sha256=p3jmJfnCag9iE5sdMrN6VvVEu55u46xaS_IjoI0SrmA,305 +cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-314.pyc,, +cryptography/hazmat/backends/openssl/backend.py,sha256=tV5AxBoFJ2GfA0DMWSY-0TxQJrpQoexzI9R4Kybb--4,10215 +cryptography/hazmat/bindings/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180 +cryptography/hazmat/bindings/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/bindings/_rust.abi3.so,sha256=vnU--g2fSh9-jyB5m6kxpbaqNoiK-fQLIR088ShrjOs,12807728 +cryptography/hazmat/bindings/_rust/__init__.pyi,sha256=KhqLhXFPArPzzJ7DYO9Fl8FoXB_BagAd_r4Dm_Ze9Xo,1257 +cryptography/hazmat/bindings/_rust/_openssl.pyi,sha256=mpNJLuYLbCVrd5i33FBTmWwL_55Dw7JPkSLlSX9Q7oI,230 +cryptography/hazmat/bindings/_rust/asn1.pyi,sha256=BrGjC8J6nwuS-r3EVcdXJB8ndotfY9mbQYOfpbPG0HA,354 +cryptography/hazmat/bindings/_rust/declarative_asn1.pyi,sha256=2ECFmYue1EPkHEE2Bm7aLwkjB0mSUTpr23v9MN4pri4,892 +cryptography/hazmat/bindings/_rust/exceptions.pyi,sha256=exXr2xw_0pB1kk93cYbM3MohbzoUkjOms1ZMUi0uQZE,640 +cryptography/hazmat/bindings/_rust/ocsp.pyi,sha256=VPVWuKHI9EMs09ZLRYAGvR0Iz0mCMmEzXAkgJHovpoM,4020 +cryptography/hazmat/bindings/_rust/openssl/__init__.pyi,sha256=iOAMDyHoNwwCSZfZzuXDr64g4GpGUeDgEN-LjXqdrBM,1522 +cryptography/hazmat/bindings/_rust/openssl/aead.pyi,sha256=4Nddw6-ynzIB3w2W86WvkGKTLlTDk_6F5l54RHCuy3E,2688 +cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi,sha256=LhPzHWSXJq4grAJXn6zSvSSdV-aYIIscHDwIPlJGGPs,1315 +cryptography/hazmat/bindings/_rust/openssl/cmac.pyi,sha256=nPH0X57RYpsAkRowVpjQiHE566ThUTx7YXrsadmrmHk,564 +cryptography/hazmat/bindings/_rust/openssl/dh.pyi,sha256=Z3TC-G04-THtSdAOPLM1h2G7ml5bda1ElZUcn5wpuhk,1564 +cryptography/hazmat/bindings/_rust/openssl/dsa.pyi,sha256=qBtkgj2albt2qFcnZ9UDrhzoNhCVO7HTby5VSf1EXMI,1299 +cryptography/hazmat/bindings/_rust/openssl/ec.pyi,sha256=zJy0pRa5n-_p2dm45PxECB_-B6SVZyNKfjxFDpPqT38,1691 +cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi,sha256=VXfXd5G6hUivg399R1DYdmW3eTb0EebzDTqjRC2gaRw,532 +cryptography/hazmat/bindings/_rust/openssl/ed448.pyi,sha256=Yx49lqdnjsD7bxiDV1kcaMrDktug5evi5a6zerMiy2s,514 +cryptography/hazmat/bindings/_rust/openssl/hashes.pyi,sha256=OWZvBx7xfo_HJl41Nc--DugVyCVPIprZ3HlOPTSWH9g,984 +cryptography/hazmat/bindings/_rust/openssl/hmac.pyi,sha256=BXZn7NDjL3JAbYW0SQ8pg1iyC5DbQXVhUAiwsi8DFR8,702 +cryptography/hazmat/bindings/_rust/openssl/kdf.pyi,sha256=xXfFBb9QehHfDtEaxV_65Z0YK7NquOVIChpTLkgAs_k,2029 +cryptography/hazmat/bindings/_rust/openssl/keys.pyi,sha256=teIt8M6ZEMJrn4s3W0UnW0DZ-30Jd68WnSsKKG124l0,912 +cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi,sha256=_SW9NtQ5FDlAbdclFtWpT4lGmxKIKHpN-4j8J2BzYfQ,585 +cryptography/hazmat/bindings/_rust/openssl/rsa.pyi,sha256=2OQCNSXkxgc-3uw1xiCCloIQTV6p9_kK79Yu0rhZgPc,1364 +cryptography/hazmat/bindings/_rust/openssl/x25519.pyi,sha256=ewn4GpQyb7zPwE-ni7GtyQgMC0A1mLuqYsSyqv6nI_s,523 +cryptography/hazmat/bindings/_rust/openssl/x448.pyi,sha256=juTZTmli8jO_5Vcufg-vHvx_tCyezmSLIh_9PU3TczI,505 +cryptography/hazmat/bindings/_rust/pkcs12.pyi,sha256=vEEd5wDiZvb8ZGFaziLCaWLzAwoG_tvPUxLQw5_uOl8,1605 +cryptography/hazmat/bindings/_rust/pkcs7.pyi,sha256=txGBJijqZshEcqra6byPNbnisIdlxzOSIHP2hl9arPs,1601 +cryptography/hazmat/bindings/_rust/test_support.pyi,sha256=PPhld-WkO743iXFPebeG0LtgK0aTzGdjcIsay1Gm5GE,757 +cryptography/hazmat/bindings/_rust/x509.pyi,sha256=n9X0IQ6ICbdIi-ExdCFZoBgeY6njm3QOVAVZwDQdnbk,9784 +cryptography/hazmat/bindings/openssl/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180 +cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-314.pyc,, +cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-314.pyc,, +cryptography/hazmat/bindings/openssl/_conditional.py,sha256=DMOpA_XN4l70zTc5_J9DpwlbQeUBRTWpfIJ4yRIn1-U,5791 +cryptography/hazmat/bindings/openssl/binding.py,sha256=x8eocEmukO4cm7cHqfVmOoYY7CCXdoF1v1WhZQt9neo,4610 +cryptography/hazmat/decrepit/__init__.py,sha256=wHCbWfaefa-fk6THSw9th9fJUsStJo7245wfFBqmduA,216 +cryptography/hazmat/decrepit/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/decrepit/ciphers/__init__.py,sha256=wHCbWfaefa-fk6THSw9th9fJUsStJo7245wfFBqmduA,216 +cryptography/hazmat/decrepit/ciphers/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/decrepit/ciphers/__pycache__/algorithms.cpython-314.pyc,, +cryptography/hazmat/decrepit/ciphers/algorithms.py,sha256=YrKgHS4MfwWaMmPBYRymRRlC0phwWp9ycICFezeJPGk,2595 +cryptography/hazmat/primitives/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180 +cryptography/hazmat/primitives/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/_asymmetric.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/_cipheralgorithm.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/_serialization.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/cmac.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/constant_time.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/hashes.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/hmac.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/keywrap.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/padding.cpython-314.pyc,, +cryptography/hazmat/primitives/__pycache__/poly1305.cpython-314.pyc,, +cryptography/hazmat/primitives/_asymmetric.py,sha256=RhgcouUB6HTiFDBrR1LxqkMjpUxIiNvQ1r_zJjRG6qQ,532 +cryptography/hazmat/primitives/_cipheralgorithm.py,sha256=Eh3i7lwedHfi0eLSsH93PZxQKzY9I6lkK67vL4V5tOc,1522 +cryptography/hazmat/primitives/_serialization.py,sha256=chgPCSF2jxI2Cr5gB-qbWXOvOfupBh4CARS0KAhv9AM,5123 +cryptography/hazmat/primitives/asymmetric/__init__.py,sha256=s9oKCQ2ycFdXoERdS1imafueSkBsL9kvbyfghaauZ9Y,180 +cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/types.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-314.pyc,, +cryptography/hazmat/primitives/asymmetric/dh.py,sha256=0v_vEFFz5pQ1QG-FkWDyvgv7IfuVZSH5Q6LyFI5A8rg,3645 +cryptography/hazmat/primitives/asymmetric/dsa.py,sha256=Ld_bbbqQFz12dObHxIkzEQzX0SWWP41RLSWkYSaKhqE,4213 +cryptography/hazmat/primitives/asymmetric/ec.py,sha256=dj0ZR_jTVI1wojjipjbXNVccPSIRObWxSZcTGQKGbHc,13437 +cryptography/hazmat/primitives/asymmetric/ed25519.py,sha256=jZW5cs472wXXV3eB0sE1b8w64gdazwwU0_MT5UOTiXs,3700 +cryptography/hazmat/primitives/asymmetric/ed448.py,sha256=yAetgn2f2JYf0BO8MapGzXeThsvSMG5LmUCrxVOidAA,3729 +cryptography/hazmat/primitives/asymmetric/padding.py,sha256=vQ6l6gOg9HqcbOsvHrSiJRVLdEj9L4m4HkRGYziTyFA,2854 +cryptography/hazmat/primitives/asymmetric/rsa.py,sha256=ZnKOo2f34MCCOupC03Y1uR-_jiSG5IrelHEmxaME3D4,8303 +cryptography/hazmat/primitives/asymmetric/types.py,sha256=LnsOJym-wmPUJ7Knu_7bCNU3kIiELCd6krOaW_JU08I,2996 +cryptography/hazmat/primitives/asymmetric/utils.py,sha256=DPTs6T4F-UhwzFQTh-1fSEpQzazH2jf2xpIro3ItF4o,790 +cryptography/hazmat/primitives/asymmetric/x25519.py,sha256=_4nQeZ3yJ3Lg0RpXnaqA-1yt6vbx1F-wzLcaZHwSpeE,3613 +cryptography/hazmat/primitives/asymmetric/x448.py,sha256=WKBLtuVfJqiBRro654fGaQAlvsKbqbNkK7c4A_ZCdV0,3642 +cryptography/hazmat/primitives/ciphers/__init__.py,sha256=eyEXmjk6_CZXaOPYDr7vAYGXr29QvzgWL2-4CSolLFs,680 +cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-314.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-314.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-314.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-314.pyc,, +cryptography/hazmat/primitives/ciphers/aead.py,sha256=Fzlyx7w8KYQakzDp1zWgJnIr62zgZrgVh1u2h4exB54,634 +cryptography/hazmat/primitives/ciphers/algorithms.py,sha256=Q7ZJwcsx83Mgxv5y7r6CyJKSdsOwC-my-5A67-ma2vw,3407 +cryptography/hazmat/primitives/ciphers/base.py,sha256=aBC7HHBBoixebmparVr0UlODs3VD0A7B6oz_AaRjDv8,4253 +cryptography/hazmat/primitives/ciphers/modes.py,sha256=20stpwhDtbAvpH0SMf9EDHIciwmTF-JMBUOZ9bU8WiQ,8318 +cryptography/hazmat/primitives/cmac.py,sha256=sz_s6H_cYnOvx-VNWdIKhRhe3Ymp8z8J0D3CBqOX3gg,338 +cryptography/hazmat/primitives/constant_time.py,sha256=xdunWT0nf8OvKdcqUhhlFKayGp4_PgVJRU2W1wLSr_A,422 +cryptography/hazmat/primitives/hashes.py,sha256=M8BrlKB3U6DEtHvWTV5VRjpteHv1kS3Zxm_Bsk04cr8,5184 +cryptography/hazmat/primitives/hmac.py,sha256=RpB3z9z5skirCQrm7zQbtnp9pLMnAjrlTUvKqF5aDDc,423 +cryptography/hazmat/primitives/kdf/__init__.py,sha256=4XibZnrYq4hh5xBjWiIXzaYW6FKx8hPbVaa_cB9zS64,750 +cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/argon2.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-314.pyc,, +cryptography/hazmat/primitives/kdf/argon2.py,sha256=UFDNXG0v-rw3DqAQTB1UQAsQC2M5Ejg0k_6OCyhLKus,460 +cryptography/hazmat/primitives/kdf/concatkdf.py,sha256=Ua8KoLXXnzgsrAUmHpyKymaPt8aPRP0EHEaBz7QCQ9I,3737 +cryptography/hazmat/primitives/kdf/hkdf.py,sha256=M0lAEfRoc4kpp4-nwDj9yB-vNZukIOYEQrUlWsBNn9o,543 +cryptography/hazmat/primitives/kdf/kbkdf.py,sha256=oZepvo4evhKkkJQWRDwaPoIbyTaFmDc5NPimxg6lfKg,9165 +cryptography/hazmat/primitives/kdf/pbkdf2.py,sha256=1WIwhELR0w8ztTpTu8BrFiYWmK3hUfJq08I79TxwieE,1957 +cryptography/hazmat/primitives/kdf/scrypt.py,sha256=XyWUdUUmhuI9V6TqAPOvujCSMGv1XQdg0a21IWCmO-U,590 +cryptography/hazmat/primitives/kdf/x963kdf.py,sha256=zLTcF665QFvXX2f8TS7fmBZTteXpFjKahzfjjQcCJyw,1999 +cryptography/hazmat/primitives/keywrap.py,sha256=XV4Pj2fqSeD-RqZVvY2cA3j5_7RwJSFygYuLfk2ujCo,5650 +cryptography/hazmat/primitives/padding.py,sha256=QT-U-NvV2eQGO1wVPbDiNGNSc9keRDS-ig5cQOrLz0E,1865 +cryptography/hazmat/primitives/poly1305.py,sha256=P5EPQV-RB_FJPahpg01u0Ts4S_PnAmsroxIGXbGeRRo,355 +cryptography/hazmat/primitives/serialization/__init__.py,sha256=Q7uTgDlt7n3WfsMT6jYwutC6DIg_7SEeoAm1GHZ5B5E,1705 +cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-314.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-314.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-314.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-314.pyc,, +cryptography/hazmat/primitives/serialization/base.py,sha256=ikq5MJIwp_oUnjiaBco_PmQwOTYuGi-XkYUYHKy8Vo0,615 +cryptography/hazmat/primitives/serialization/pkcs12.py,sha256=mS9cFNG4afzvseoc5e1MWoY2VskfL8N8Y_OFjl67luY,5104 +cryptography/hazmat/primitives/serialization/pkcs7.py,sha256=5OR_Tkysxaprn4FegvJIfbep9rJ9wok6FLWvWwQ5-Mg,13943 +cryptography/hazmat/primitives/serialization/ssh.py,sha256=hPV5obFznz0QhFfXFPOeQ8y6MsurA0xVMQiLnLESEs8,53700 +cryptography/hazmat/primitives/twofactor/__init__.py,sha256=tmMZGB-g4IU1r7lIFqASU019zr0uPp_wEBYcwdDCKCA,258 +cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-314.pyc,, +cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-314.pyc,, +cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-314.pyc,, +cryptography/hazmat/primitives/twofactor/hotp.py,sha256=ivZo5BrcCGWLsqql4nZV0XXCjyGPi_iHfDFltGlOJwk,3256 +cryptography/hazmat/primitives/twofactor/totp.py,sha256=m5LPpRL00kp4zY8gTjr55Hfz9aMlPS53kHmVkSQCmdY,1652 +cryptography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +cryptography/utils.py,sha256=nFHkPQZycOQGeBtBRkWSA4WjOHFo7pwummQt-PPSkZc,4349 +cryptography/x509/__init__.py,sha256=xloN0swseNx-m2WFZmCA17gOoxQWqeU82UVjEdJBePQ,8257 +cryptography/x509/__pycache__/__init__.cpython-314.pyc,, +cryptography/x509/__pycache__/base.cpython-314.pyc,, +cryptography/x509/__pycache__/certificate_transparency.cpython-314.pyc,, +cryptography/x509/__pycache__/extensions.cpython-314.pyc,, +cryptography/x509/__pycache__/general_name.cpython-314.pyc,, +cryptography/x509/__pycache__/name.cpython-314.pyc,, +cryptography/x509/__pycache__/ocsp.cpython-314.pyc,, +cryptography/x509/__pycache__/oid.cpython-314.pyc,, +cryptography/x509/__pycache__/verification.cpython-314.pyc,, +cryptography/x509/base.py,sha256=OrmTw3y8B6AE_nGXQPN8x9kq-d7rDWeH13gCq6T6D6U,27997 +cryptography/x509/certificate_transparency.py,sha256=JqoOIDhlwInrYMFW6IFn77WJ0viF-PB_rlZV3vs9MYc,797 +cryptography/x509/extensions.py,sha256=QxYrqR6SF1qzR9ZraP8wDiIczlEVlAFuwDRVcltB6Tk,77724 +cryptography/x509/general_name.py,sha256=sP_rV11Qlpsk4x3XXGJY_Mv0Q_s9dtjeLckHsjpLQoQ,7836 +cryptography/x509/name.py,sha256=ty0_xf0LnHwZAdEf-d8FLO1K4hGqx_7DsD3CHwoLJiY,15101 +cryptography/x509/ocsp.py,sha256=Yey6NdFV1MPjop24Mj_VenjEpg3kUaMopSWOK0AbeBs,12699 +cryptography/x509/oid.py,sha256=BUzgXXGVWilkBkdKPTm9R4qElE9gAGHgdYPMZAp7PJo,931 +cryptography/x509/verification.py,sha256=gR2C2c-XZQtblZhT5T5vjSKOtCb74ef2alPVmEcwFlM,958 diff --git a/lib/cryptography-46.0.5.dist-info/WHEEL b/lib/cryptography-46.0.5.dist-info/WHEEL new file mode 100644 index 0000000..8e48aa1 --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: maturin (1.9.4) +Root-Is-Purelib: false +Tag: cp311-abi3-manylinux_2_34_x86_64 + diff --git a/lib/cryptography-46.0.5.dist-info/licenses/LICENSE b/lib/cryptography-46.0.5.dist-info/licenses/LICENSE new file mode 100644 index 0000000..b11f379 --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/licenses/LICENSE @@ -0,0 +1,3 @@ +This software is made available under the terms of *either* of the licenses +found in LICENSE.APACHE or LICENSE.BSD. Contributions to cryptography are made +under the terms of *both* these licenses. diff --git a/lib/cryptography-46.0.5.dist-info/licenses/LICENSE.APACHE b/lib/cryptography-46.0.5.dist-info/licenses/LICENSE.APACHE new file mode 100644 index 0000000..62589ed --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/licenses/LICENSE.APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/cryptography-46.0.5.dist-info/licenses/LICENSE.BSD b/lib/cryptography-46.0.5.dist-info/licenses/LICENSE.BSD new file mode 100644 index 0000000..ec1a29d --- /dev/null +++ b/lib/cryptography-46.0.5.dist-info/licenses/LICENSE.BSD @@ -0,0 +1,27 @@ +Copyright (c) Individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of PyCA Cryptography nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/cryptography/__about__.py b/lib/cryptography/__about__.py new file mode 100644 index 0000000..43b3024 --- /dev/null +++ b/lib/cryptography/__about__.py @@ -0,0 +1,17 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +__all__ = [ + "__author__", + "__copyright__", + "__version__", +] + +__version__ = "46.0.5" + + +__author__ = "The Python Cryptographic Authority and individual contributors" +__copyright__ = f"Copyright 2013-2025 {__author__}" diff --git a/lib/cryptography/__init__.py b/lib/cryptography/__init__.py new file mode 100644 index 0000000..d374f75 --- /dev/null +++ b/lib/cryptography/__init__.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.__about__ import __author__, __copyright__, __version__ + +__all__ = [ + "__author__", + "__copyright__", + "__version__", +] diff --git a/lib/cryptography/__pycache__/__about__.cpython-314.pyc b/lib/cryptography/__pycache__/__about__.cpython-314.pyc new file mode 100644 index 0000000..f404c3e Binary files /dev/null and b/lib/cryptography/__pycache__/__about__.cpython-314.pyc differ diff --git a/lib/cryptography/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..e079804 Binary files /dev/null and b/lib/cryptography/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/__pycache__/exceptions.cpython-314.pyc b/lib/cryptography/__pycache__/exceptions.cpython-314.pyc new file mode 100644 index 0000000..8bb77af Binary files /dev/null and b/lib/cryptography/__pycache__/exceptions.cpython-314.pyc differ diff --git a/lib/cryptography/__pycache__/fernet.cpython-314.pyc b/lib/cryptography/__pycache__/fernet.cpython-314.pyc new file mode 100644 index 0000000..ee89cd4 Binary files /dev/null and b/lib/cryptography/__pycache__/fernet.cpython-314.pyc differ diff --git a/lib/cryptography/__pycache__/utils.cpython-314.pyc b/lib/cryptography/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..d193880 Binary files /dev/null and b/lib/cryptography/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/cryptography/exceptions.py b/lib/cryptography/exceptions.py new file mode 100644 index 0000000..fe125ea --- /dev/null +++ b/lib/cryptography/exceptions.py @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.bindings._rust import exceptions as rust_exceptions + +if typing.TYPE_CHECKING: + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +_Reasons = rust_exceptions._Reasons + + +class UnsupportedAlgorithm(Exception): + def __init__(self, message: str, reason: _Reasons | None = None) -> None: + super().__init__(message) + self._reason = reason + + +class AlreadyFinalized(Exception): + pass + + +class AlreadyUpdated(Exception): + pass + + +class NotYetFinalized(Exception): + pass + + +class InvalidTag(Exception): + pass + + +class InvalidSignature(Exception): + pass + + +class InternalError(Exception): + def __init__( + self, msg: str, err_code: list[rust_openssl.OpenSSLError] + ) -> None: + super().__init__(msg) + self.err_code = err_code + + +class InvalidKey(Exception): + pass diff --git a/lib/cryptography/fernet.py b/lib/cryptography/fernet.py new file mode 100644 index 0000000..c6744ae --- /dev/null +++ b/lib/cryptography/fernet.py @@ -0,0 +1,224 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import base64 +import binascii +import os +import time +import typing +from collections.abc import Iterable + +from cryptography import utils +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.hmac import HMAC + + +class InvalidToken(Exception): + pass + + +_MAX_CLOCK_SKEW = 60 + + +class Fernet: + def __init__( + self, + key: bytes | str, + backend: typing.Any = None, + ) -> None: + try: + key = base64.urlsafe_b64decode(key) + except binascii.Error as exc: + raise ValueError( + "Fernet key must be 32 url-safe base64-encoded bytes." + ) from exc + if len(key) != 32: + raise ValueError( + "Fernet key must be 32 url-safe base64-encoded bytes." + ) + + self._signing_key = key[:16] + self._encryption_key = key[16:] + + @classmethod + def generate_key(cls) -> bytes: + return base64.urlsafe_b64encode(os.urandom(32)) + + def encrypt(self, data: bytes) -> bytes: + return self.encrypt_at_time(data, int(time.time())) + + def encrypt_at_time(self, data: bytes, current_time: int) -> bytes: + iv = os.urandom(16) + return self._encrypt_from_parts(data, current_time, iv) + + def _encrypt_from_parts( + self, data: bytes, current_time: int, iv: bytes + ) -> bytes: + utils._check_bytes("data", data) + + padder = padding.PKCS7(algorithms.AES.block_size).padder() + padded_data = padder.update(data) + padder.finalize() + encryptor = Cipher( + algorithms.AES(self._encryption_key), + modes.CBC(iv), + ).encryptor() + ciphertext = encryptor.update(padded_data) + encryptor.finalize() + + basic_parts = ( + b"\x80" + + current_time.to_bytes(length=8, byteorder="big") + + iv + + ciphertext + ) + + h = HMAC(self._signing_key, hashes.SHA256()) + h.update(basic_parts) + hmac = h.finalize() + return base64.urlsafe_b64encode(basic_parts + hmac) + + def decrypt(self, token: bytes | str, ttl: int | None = None) -> bytes: + timestamp, data = Fernet._get_unverified_token_data(token) + if ttl is None: + time_info = None + else: + time_info = (ttl, int(time.time())) + return self._decrypt_data(data, timestamp, time_info) + + def decrypt_at_time( + self, token: bytes | str, ttl: int, current_time: int + ) -> bytes: + if ttl is None: + raise ValueError( + "decrypt_at_time() can only be used with a non-None ttl" + ) + timestamp, data = Fernet._get_unverified_token_data(token) + return self._decrypt_data(data, timestamp, (ttl, current_time)) + + def extract_timestamp(self, token: bytes | str) -> int: + timestamp, data = Fernet._get_unverified_token_data(token) + # Verify the token was not tampered with. + self._verify_signature(data) + return timestamp + + @staticmethod + def _get_unverified_token_data(token: bytes | str) -> tuple[int, bytes]: + if not isinstance(token, (str, bytes)): + raise TypeError("token must be bytes or str") + + try: + data = base64.urlsafe_b64decode(token) + except (TypeError, binascii.Error): + raise InvalidToken + + if not data or data[0] != 0x80: + raise InvalidToken + + if len(data) < 9: + raise InvalidToken + + timestamp = int.from_bytes(data[1:9], byteorder="big") + return timestamp, data + + def _verify_signature(self, data: bytes) -> None: + h = HMAC(self._signing_key, hashes.SHA256()) + h.update(data[:-32]) + try: + h.verify(data[-32:]) + except InvalidSignature: + raise InvalidToken + + def _decrypt_data( + self, + data: bytes, + timestamp: int, + time_info: tuple[int, int] | None, + ) -> bytes: + if time_info is not None: + ttl, current_time = time_info + if timestamp + ttl < current_time: + raise InvalidToken + + if current_time + _MAX_CLOCK_SKEW < timestamp: + raise InvalidToken + + self._verify_signature(data) + + iv = data[9:25] + ciphertext = data[25:-32] + decryptor = Cipher( + algorithms.AES(self._encryption_key), modes.CBC(iv) + ).decryptor() + plaintext_padded = decryptor.update(ciphertext) + try: + plaintext_padded += decryptor.finalize() + except ValueError: + raise InvalidToken + unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder() + + unpadded = unpadder.update(plaintext_padded) + try: + unpadded += unpadder.finalize() + except ValueError: + raise InvalidToken + return unpadded + + +class MultiFernet: + def __init__(self, fernets: Iterable[Fernet]): + fernets = list(fernets) + if not fernets: + raise ValueError( + "MultiFernet requires at least one Fernet instance" + ) + self._fernets = fernets + + def encrypt(self, msg: bytes) -> bytes: + return self.encrypt_at_time(msg, int(time.time())) + + def encrypt_at_time(self, msg: bytes, current_time: int) -> bytes: + return self._fernets[0].encrypt_at_time(msg, current_time) + + def rotate(self, msg: bytes | str) -> bytes: + timestamp, data = Fernet._get_unverified_token_data(msg) + for f in self._fernets: + try: + p = f._decrypt_data(data, timestamp, None) + break + except InvalidToken: + pass + else: + raise InvalidToken + + iv = os.urandom(16) + return self._fernets[0]._encrypt_from_parts(p, timestamp, iv) + + def decrypt(self, msg: bytes | str, ttl: int | None = None) -> bytes: + for f in self._fernets: + try: + return f.decrypt(msg, ttl) + except InvalidToken: + pass + raise InvalidToken + + def decrypt_at_time( + self, msg: bytes | str, ttl: int, current_time: int + ) -> bytes: + for f in self._fernets: + try: + return f.decrypt_at_time(msg, ttl, current_time) + except InvalidToken: + pass + raise InvalidToken + + def extract_timestamp(self, msg: bytes | str) -> int: + for f in self._fernets: + try: + return f.extract_timestamp(msg) + except InvalidToken: + pass + raise InvalidToken diff --git a/lib/cryptography/hazmat/__init__.py b/lib/cryptography/hazmat/__init__.py new file mode 100644 index 0000000..b9f1187 --- /dev/null +++ b/lib/cryptography/hazmat/__init__.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +""" +Hazardous Materials + +This is a "Hazardous Materials" module. You should ONLY use it if you're +100% absolutely sure that you know what you're doing because this module +is full of land mines, dragons, and dinosaurs with laser guns. +""" diff --git a/lib/cryptography/hazmat/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..067aa8f Binary files /dev/null and b/lib/cryptography/hazmat/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/__pycache__/_oid.cpython-314.pyc b/lib/cryptography/hazmat/__pycache__/_oid.cpython-314.pyc new file mode 100644 index 0000000..fef49e2 Binary files /dev/null and b/lib/cryptography/hazmat/__pycache__/_oid.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/_oid.py b/lib/cryptography/hazmat/_oid.py new file mode 100644 index 0000000..4bf138d --- /dev/null +++ b/lib/cryptography/hazmat/_oid.py @@ -0,0 +1,356 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import ( + ObjectIdentifier as ObjectIdentifier, +) +from cryptography.hazmat.primitives import hashes + + +class ExtensionOID: + SUBJECT_DIRECTORY_ATTRIBUTES = ObjectIdentifier("2.5.29.9") + SUBJECT_KEY_IDENTIFIER = ObjectIdentifier("2.5.29.14") + KEY_USAGE = ObjectIdentifier("2.5.29.15") + PRIVATE_KEY_USAGE_PERIOD = ObjectIdentifier("2.5.29.16") + SUBJECT_ALTERNATIVE_NAME = ObjectIdentifier("2.5.29.17") + ISSUER_ALTERNATIVE_NAME = ObjectIdentifier("2.5.29.18") + BASIC_CONSTRAINTS = ObjectIdentifier("2.5.29.19") + NAME_CONSTRAINTS = ObjectIdentifier("2.5.29.30") + CRL_DISTRIBUTION_POINTS = ObjectIdentifier("2.5.29.31") + CERTIFICATE_POLICIES = ObjectIdentifier("2.5.29.32") + POLICY_MAPPINGS = ObjectIdentifier("2.5.29.33") + AUTHORITY_KEY_IDENTIFIER = ObjectIdentifier("2.5.29.35") + POLICY_CONSTRAINTS = ObjectIdentifier("2.5.29.36") + EXTENDED_KEY_USAGE = ObjectIdentifier("2.5.29.37") + FRESHEST_CRL = ObjectIdentifier("2.5.29.46") + INHIBIT_ANY_POLICY = ObjectIdentifier("2.5.29.54") + ISSUING_DISTRIBUTION_POINT = ObjectIdentifier("2.5.29.28") + AUTHORITY_INFORMATION_ACCESS = ObjectIdentifier("1.3.6.1.5.5.7.1.1") + SUBJECT_INFORMATION_ACCESS = ObjectIdentifier("1.3.6.1.5.5.7.1.11") + OCSP_NO_CHECK = ObjectIdentifier("1.3.6.1.5.5.7.48.1.5") + TLS_FEATURE = ObjectIdentifier("1.3.6.1.5.5.7.1.24") + CRL_NUMBER = ObjectIdentifier("2.5.29.20") + DELTA_CRL_INDICATOR = ObjectIdentifier("2.5.29.27") + PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS = ObjectIdentifier( + "1.3.6.1.4.1.11129.2.4.2" + ) + PRECERT_POISON = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.3") + SIGNED_CERTIFICATE_TIMESTAMPS = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.5") + MS_CERTIFICATE_TEMPLATE = ObjectIdentifier("1.3.6.1.4.1.311.21.7") + ADMISSIONS = ObjectIdentifier("1.3.36.8.3.3") + + +class OCSPExtensionOID: + NONCE = ObjectIdentifier("1.3.6.1.5.5.7.48.1.2") + ACCEPTABLE_RESPONSES = ObjectIdentifier("1.3.6.1.5.5.7.48.1.4") + + +class CRLEntryExtensionOID: + CERTIFICATE_ISSUER = ObjectIdentifier("2.5.29.29") + CRL_REASON = ObjectIdentifier("2.5.29.21") + INVALIDITY_DATE = ObjectIdentifier("2.5.29.24") + + +class NameOID: + COMMON_NAME = ObjectIdentifier("2.5.4.3") + COUNTRY_NAME = ObjectIdentifier("2.5.4.6") + LOCALITY_NAME = ObjectIdentifier("2.5.4.7") + STATE_OR_PROVINCE_NAME = ObjectIdentifier("2.5.4.8") + STREET_ADDRESS = ObjectIdentifier("2.5.4.9") + ORGANIZATION_IDENTIFIER = ObjectIdentifier("2.5.4.97") + ORGANIZATION_NAME = ObjectIdentifier("2.5.4.10") + ORGANIZATIONAL_UNIT_NAME = ObjectIdentifier("2.5.4.11") + SERIAL_NUMBER = ObjectIdentifier("2.5.4.5") + SURNAME = ObjectIdentifier("2.5.4.4") + GIVEN_NAME = ObjectIdentifier("2.5.4.42") + TITLE = ObjectIdentifier("2.5.4.12") + INITIALS = ObjectIdentifier("2.5.4.43") + GENERATION_QUALIFIER = ObjectIdentifier("2.5.4.44") + X500_UNIQUE_IDENTIFIER = ObjectIdentifier("2.5.4.45") + DN_QUALIFIER = ObjectIdentifier("2.5.4.46") + PSEUDONYM = ObjectIdentifier("2.5.4.65") + USER_ID = ObjectIdentifier("0.9.2342.19200300.100.1.1") + DOMAIN_COMPONENT = ObjectIdentifier("0.9.2342.19200300.100.1.25") + EMAIL_ADDRESS = ObjectIdentifier("1.2.840.113549.1.9.1") + JURISDICTION_COUNTRY_NAME = ObjectIdentifier("1.3.6.1.4.1.311.60.2.1.3") + JURISDICTION_LOCALITY_NAME = ObjectIdentifier("1.3.6.1.4.1.311.60.2.1.1") + JURISDICTION_STATE_OR_PROVINCE_NAME = ObjectIdentifier( + "1.3.6.1.4.1.311.60.2.1.2" + ) + BUSINESS_CATEGORY = ObjectIdentifier("2.5.4.15") + POSTAL_ADDRESS = ObjectIdentifier("2.5.4.16") + POSTAL_CODE = ObjectIdentifier("2.5.4.17") + INN = ObjectIdentifier("1.2.643.3.131.1.1") + OGRN = ObjectIdentifier("1.2.643.100.1") + SNILS = ObjectIdentifier("1.2.643.100.3") + UNSTRUCTURED_NAME = ObjectIdentifier("1.2.840.113549.1.9.2") + + +class SignatureAlgorithmOID: + RSA_WITH_MD5 = ObjectIdentifier("1.2.840.113549.1.1.4") + RSA_WITH_SHA1 = ObjectIdentifier("1.2.840.113549.1.1.5") + # This is an alternate OID for RSA with SHA1 that is occasionally seen + _RSA_WITH_SHA1 = ObjectIdentifier("1.3.14.3.2.29") + RSA_WITH_SHA224 = ObjectIdentifier("1.2.840.113549.1.1.14") + RSA_WITH_SHA256 = ObjectIdentifier("1.2.840.113549.1.1.11") + RSA_WITH_SHA384 = ObjectIdentifier("1.2.840.113549.1.1.12") + RSA_WITH_SHA512 = ObjectIdentifier("1.2.840.113549.1.1.13") + RSA_WITH_SHA3_224 = ObjectIdentifier("2.16.840.1.101.3.4.3.13") + RSA_WITH_SHA3_256 = ObjectIdentifier("2.16.840.1.101.3.4.3.14") + RSA_WITH_SHA3_384 = ObjectIdentifier("2.16.840.1.101.3.4.3.15") + RSA_WITH_SHA3_512 = ObjectIdentifier("2.16.840.1.101.3.4.3.16") + RSASSA_PSS = ObjectIdentifier("1.2.840.113549.1.1.10") + ECDSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10045.4.1") + ECDSA_WITH_SHA224 = ObjectIdentifier("1.2.840.10045.4.3.1") + ECDSA_WITH_SHA256 = ObjectIdentifier("1.2.840.10045.4.3.2") + ECDSA_WITH_SHA384 = ObjectIdentifier("1.2.840.10045.4.3.3") + ECDSA_WITH_SHA512 = ObjectIdentifier("1.2.840.10045.4.3.4") + ECDSA_WITH_SHA3_224 = ObjectIdentifier("2.16.840.1.101.3.4.3.9") + ECDSA_WITH_SHA3_256 = ObjectIdentifier("2.16.840.1.101.3.4.3.10") + ECDSA_WITH_SHA3_384 = ObjectIdentifier("2.16.840.1.101.3.4.3.11") + ECDSA_WITH_SHA3_512 = ObjectIdentifier("2.16.840.1.101.3.4.3.12") + DSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10040.4.3") + DSA_WITH_SHA224 = ObjectIdentifier("2.16.840.1.101.3.4.3.1") + DSA_WITH_SHA256 = ObjectIdentifier("2.16.840.1.101.3.4.3.2") + DSA_WITH_SHA384 = ObjectIdentifier("2.16.840.1.101.3.4.3.3") + DSA_WITH_SHA512 = ObjectIdentifier("2.16.840.1.101.3.4.3.4") + ED25519 = ObjectIdentifier("1.3.101.112") + ED448 = ObjectIdentifier("1.3.101.113") + GOSTR3411_94_WITH_3410_2001 = ObjectIdentifier("1.2.643.2.2.3") + GOSTR3410_2012_WITH_3411_2012_256 = ObjectIdentifier("1.2.643.7.1.1.3.2") + GOSTR3410_2012_WITH_3411_2012_512 = ObjectIdentifier("1.2.643.7.1.1.3.3") + + +_SIG_OIDS_TO_HASH: dict[ObjectIdentifier, hashes.HashAlgorithm | None] = { + SignatureAlgorithmOID.RSA_WITH_MD5: hashes.MD5(), + SignatureAlgorithmOID.RSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID._RSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.RSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.RSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.RSA_WITH_SHA384: hashes.SHA384(), + SignatureAlgorithmOID.RSA_WITH_SHA512: hashes.SHA512(), + SignatureAlgorithmOID.RSA_WITH_SHA3_224: hashes.SHA3_224(), + SignatureAlgorithmOID.RSA_WITH_SHA3_256: hashes.SHA3_256(), + SignatureAlgorithmOID.RSA_WITH_SHA3_384: hashes.SHA3_384(), + SignatureAlgorithmOID.RSA_WITH_SHA3_512: hashes.SHA3_512(), + SignatureAlgorithmOID.ECDSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.ECDSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.ECDSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ECDSA_WITH_SHA384: hashes.SHA384(), + SignatureAlgorithmOID.ECDSA_WITH_SHA512: hashes.SHA512(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_224: hashes.SHA3_224(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_256: hashes.SHA3_256(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_384: hashes.SHA3_384(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_512: hashes.SHA3_512(), + SignatureAlgorithmOID.DSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.DSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.DSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ED25519: None, + SignatureAlgorithmOID.ED448: None, + SignatureAlgorithmOID.GOSTR3411_94_WITH_3410_2001: None, + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_256: None, + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_512: None, +} + + +class HashAlgorithmOID: + SHA1 = ObjectIdentifier("1.3.14.3.2.26") + SHA224 = ObjectIdentifier("2.16.840.1.101.3.4.2.4") + SHA256 = ObjectIdentifier("2.16.840.1.101.3.4.2.1") + SHA384 = ObjectIdentifier("2.16.840.1.101.3.4.2.2") + SHA512 = ObjectIdentifier("2.16.840.1.101.3.4.2.3") + SHA3_224 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.224") + SHA3_256 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.256") + SHA3_384 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.384") + SHA3_512 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.512") + SHA3_224_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.7") + SHA3_256_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.8") + SHA3_384_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.9") + SHA3_512_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.10") + + +class PublicKeyAlgorithmOID: + DSA = ObjectIdentifier("1.2.840.10040.4.1") + EC_PUBLIC_KEY = ObjectIdentifier("1.2.840.10045.2.1") + RSAES_PKCS1_v1_5 = ObjectIdentifier("1.2.840.113549.1.1.1") + RSASSA_PSS = ObjectIdentifier("1.2.840.113549.1.1.10") + X25519 = ObjectIdentifier("1.3.101.110") + X448 = ObjectIdentifier("1.3.101.111") + ED25519 = ObjectIdentifier("1.3.101.112") + ED448 = ObjectIdentifier("1.3.101.113") + + +class ExtendedKeyUsageOID: + SERVER_AUTH = ObjectIdentifier("1.3.6.1.5.5.7.3.1") + CLIENT_AUTH = ObjectIdentifier("1.3.6.1.5.5.7.3.2") + CODE_SIGNING = ObjectIdentifier("1.3.6.1.5.5.7.3.3") + EMAIL_PROTECTION = ObjectIdentifier("1.3.6.1.5.5.7.3.4") + TIME_STAMPING = ObjectIdentifier("1.3.6.1.5.5.7.3.8") + OCSP_SIGNING = ObjectIdentifier("1.3.6.1.5.5.7.3.9") + ANY_EXTENDED_KEY_USAGE = ObjectIdentifier("2.5.29.37.0") + SMARTCARD_LOGON = ObjectIdentifier("1.3.6.1.4.1.311.20.2.2") + KERBEROS_PKINIT_KDC = ObjectIdentifier("1.3.6.1.5.2.3.5") + IPSEC_IKE = ObjectIdentifier("1.3.6.1.5.5.7.3.17") + BUNDLE_SECURITY = ObjectIdentifier("1.3.6.1.5.5.7.3.35") + CERTIFICATE_TRANSPARENCY = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.4") + + +class OtherNameFormOID: + PERMANENT_IDENTIFIER = ObjectIdentifier("1.3.6.1.5.5.7.8.3") + HW_MODULE_NAME = ObjectIdentifier("1.3.6.1.5.5.7.8.4") + DNS_SRV = ObjectIdentifier("1.3.6.1.5.5.7.8.7") + NAI_REALM = ObjectIdentifier("1.3.6.1.5.5.7.8.8") + SMTP_UTF8_MAILBOX = ObjectIdentifier("1.3.6.1.5.5.7.8.9") + ACP_NODE_NAME = ObjectIdentifier("1.3.6.1.5.5.7.8.10") + BUNDLE_EID = ObjectIdentifier("1.3.6.1.5.5.7.8.11") + + +class AuthorityInformationAccessOID: + CA_ISSUERS = ObjectIdentifier("1.3.6.1.5.5.7.48.2") + OCSP = ObjectIdentifier("1.3.6.1.5.5.7.48.1") + + +class SubjectInformationAccessOID: + CA_REPOSITORY = ObjectIdentifier("1.3.6.1.5.5.7.48.5") + + +class CertificatePoliciesOID: + CPS_QUALIFIER = ObjectIdentifier("1.3.6.1.5.5.7.2.1") + CPS_USER_NOTICE = ObjectIdentifier("1.3.6.1.5.5.7.2.2") + ANY_POLICY = ObjectIdentifier("2.5.29.32.0") + + +class AttributeOID: + CHALLENGE_PASSWORD = ObjectIdentifier("1.2.840.113549.1.9.7") + UNSTRUCTURED_NAME = ObjectIdentifier("1.2.840.113549.1.9.2") + + +_OID_NAMES = { + NameOID.COMMON_NAME: "commonName", + NameOID.COUNTRY_NAME: "countryName", + NameOID.LOCALITY_NAME: "localityName", + NameOID.STATE_OR_PROVINCE_NAME: "stateOrProvinceName", + NameOID.STREET_ADDRESS: "streetAddress", + NameOID.ORGANIZATION_NAME: "organizationName", + NameOID.ORGANIZATIONAL_UNIT_NAME: "organizationalUnitName", + NameOID.SERIAL_NUMBER: "serialNumber", + NameOID.SURNAME: "surname", + NameOID.GIVEN_NAME: "givenName", + NameOID.TITLE: "title", + NameOID.GENERATION_QUALIFIER: "generationQualifier", + NameOID.X500_UNIQUE_IDENTIFIER: "x500UniqueIdentifier", + NameOID.DN_QUALIFIER: "dnQualifier", + NameOID.PSEUDONYM: "pseudonym", + NameOID.USER_ID: "userID", + NameOID.DOMAIN_COMPONENT: "domainComponent", + NameOID.EMAIL_ADDRESS: "emailAddress", + NameOID.JURISDICTION_COUNTRY_NAME: "jurisdictionCountryName", + NameOID.JURISDICTION_LOCALITY_NAME: "jurisdictionLocalityName", + NameOID.JURISDICTION_STATE_OR_PROVINCE_NAME: ( + "jurisdictionStateOrProvinceName" + ), + NameOID.BUSINESS_CATEGORY: "businessCategory", + NameOID.POSTAL_ADDRESS: "postalAddress", + NameOID.POSTAL_CODE: "postalCode", + NameOID.INN: "INN", + NameOID.OGRN: "OGRN", + NameOID.SNILS: "SNILS", + NameOID.UNSTRUCTURED_NAME: "unstructuredName", + SignatureAlgorithmOID.RSA_WITH_MD5: "md5WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA1: "sha1WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA224: "sha224WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA256: "sha256WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA384: "sha384WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA512: "sha512WithRSAEncryption", + SignatureAlgorithmOID.RSASSA_PSS: "rsassaPss", + SignatureAlgorithmOID.ECDSA_WITH_SHA1: "ecdsa-with-SHA1", + SignatureAlgorithmOID.ECDSA_WITH_SHA224: "ecdsa-with-SHA224", + SignatureAlgorithmOID.ECDSA_WITH_SHA256: "ecdsa-with-SHA256", + SignatureAlgorithmOID.ECDSA_WITH_SHA384: "ecdsa-with-SHA384", + SignatureAlgorithmOID.ECDSA_WITH_SHA512: "ecdsa-with-SHA512", + SignatureAlgorithmOID.DSA_WITH_SHA1: "dsa-with-sha1", + SignatureAlgorithmOID.DSA_WITH_SHA224: "dsa-with-sha224", + SignatureAlgorithmOID.DSA_WITH_SHA256: "dsa-with-sha256", + SignatureAlgorithmOID.ED25519: "ed25519", + SignatureAlgorithmOID.ED448: "ed448", + SignatureAlgorithmOID.GOSTR3411_94_WITH_3410_2001: ( + "GOST R 34.11-94 with GOST R 34.10-2001" + ), + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_256: ( + "GOST R 34.10-2012 with GOST R 34.11-2012 (256 bit)" + ), + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_512: ( + "GOST R 34.10-2012 with GOST R 34.11-2012 (512 bit)" + ), + HashAlgorithmOID.SHA1: "sha1", + HashAlgorithmOID.SHA224: "sha224", + HashAlgorithmOID.SHA256: "sha256", + HashAlgorithmOID.SHA384: "sha384", + HashAlgorithmOID.SHA512: "sha512", + HashAlgorithmOID.SHA3_224: "sha3_224", + HashAlgorithmOID.SHA3_256: "sha3_256", + HashAlgorithmOID.SHA3_384: "sha3_384", + HashAlgorithmOID.SHA3_512: "sha3_512", + HashAlgorithmOID.SHA3_224_NIST: "sha3_224", + HashAlgorithmOID.SHA3_256_NIST: "sha3_256", + HashAlgorithmOID.SHA3_384_NIST: "sha3_384", + HashAlgorithmOID.SHA3_512_NIST: "sha3_512", + PublicKeyAlgorithmOID.DSA: "dsaEncryption", + PublicKeyAlgorithmOID.EC_PUBLIC_KEY: "id-ecPublicKey", + PublicKeyAlgorithmOID.RSAES_PKCS1_v1_5: "rsaEncryption", + PublicKeyAlgorithmOID.X25519: "X25519", + PublicKeyAlgorithmOID.X448: "X448", + ExtendedKeyUsageOID.SERVER_AUTH: "serverAuth", + ExtendedKeyUsageOID.CLIENT_AUTH: "clientAuth", + ExtendedKeyUsageOID.CODE_SIGNING: "codeSigning", + ExtendedKeyUsageOID.EMAIL_PROTECTION: "emailProtection", + ExtendedKeyUsageOID.TIME_STAMPING: "timeStamping", + ExtendedKeyUsageOID.OCSP_SIGNING: "OCSPSigning", + ExtendedKeyUsageOID.SMARTCARD_LOGON: "msSmartcardLogin", + ExtendedKeyUsageOID.KERBEROS_PKINIT_KDC: "pkInitKDC", + ExtensionOID.SUBJECT_DIRECTORY_ATTRIBUTES: "subjectDirectoryAttributes", + ExtensionOID.SUBJECT_KEY_IDENTIFIER: "subjectKeyIdentifier", + ExtensionOID.KEY_USAGE: "keyUsage", + ExtensionOID.PRIVATE_KEY_USAGE_PERIOD: "privateKeyUsagePeriod", + ExtensionOID.SUBJECT_ALTERNATIVE_NAME: "subjectAltName", + ExtensionOID.ISSUER_ALTERNATIVE_NAME: "issuerAltName", + ExtensionOID.BASIC_CONSTRAINTS: "basicConstraints", + ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS: ( + "signedCertificateTimestampList" + ), + ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS: ( + "signedCertificateTimestampList" + ), + ExtensionOID.PRECERT_POISON: "ctPoison", + ExtensionOID.MS_CERTIFICATE_TEMPLATE: "msCertificateTemplate", + ExtensionOID.ADMISSIONS: "Admissions", + CRLEntryExtensionOID.CRL_REASON: "cRLReason", + CRLEntryExtensionOID.INVALIDITY_DATE: "invalidityDate", + CRLEntryExtensionOID.CERTIFICATE_ISSUER: "certificateIssuer", + ExtensionOID.NAME_CONSTRAINTS: "nameConstraints", + ExtensionOID.CRL_DISTRIBUTION_POINTS: "cRLDistributionPoints", + ExtensionOID.CERTIFICATE_POLICIES: "certificatePolicies", + ExtensionOID.POLICY_MAPPINGS: "policyMappings", + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: "authorityKeyIdentifier", + ExtensionOID.POLICY_CONSTRAINTS: "policyConstraints", + ExtensionOID.EXTENDED_KEY_USAGE: "extendedKeyUsage", + ExtensionOID.FRESHEST_CRL: "freshestCRL", + ExtensionOID.INHIBIT_ANY_POLICY: "inhibitAnyPolicy", + ExtensionOID.ISSUING_DISTRIBUTION_POINT: "issuingDistributionPoint", + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: "authorityInfoAccess", + ExtensionOID.SUBJECT_INFORMATION_ACCESS: "subjectInfoAccess", + ExtensionOID.OCSP_NO_CHECK: "OCSPNoCheck", + ExtensionOID.CRL_NUMBER: "cRLNumber", + ExtensionOID.DELTA_CRL_INDICATOR: "deltaCRLIndicator", + ExtensionOID.TLS_FEATURE: "TLSFeature", + AuthorityInformationAccessOID.OCSP: "OCSP", + AuthorityInformationAccessOID.CA_ISSUERS: "caIssuers", + SubjectInformationAccessOID.CA_REPOSITORY: "caRepository", + CertificatePoliciesOID.CPS_QUALIFIER: "id-qt-cps", + CertificatePoliciesOID.CPS_USER_NOTICE: "id-qt-unotice", + OCSPExtensionOID.NONCE: "OCSPNonce", + AttributeOID.CHALLENGE_PASSWORD: "challengePassword", +} diff --git a/lib/cryptography/hazmat/asn1/__init__.py b/lib/cryptography/hazmat/asn1/__init__.py new file mode 100644 index 0000000..be68373 --- /dev/null +++ b/lib/cryptography/hazmat/asn1/__init__.py @@ -0,0 +1,10 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.asn1.asn1 import encode_der, sequence + +__all__ = [ + "encode_der", + "sequence", +] diff --git a/lib/cryptography/hazmat/asn1/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/asn1/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..5386a0f Binary files /dev/null and b/lib/cryptography/hazmat/asn1/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/asn1/__pycache__/asn1.cpython-314.pyc b/lib/cryptography/hazmat/asn1/__pycache__/asn1.cpython-314.pyc new file mode 100644 index 0000000..2ab3bfe Binary files /dev/null and b/lib/cryptography/hazmat/asn1/__pycache__/asn1.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/asn1/asn1.py b/lib/cryptography/hazmat/asn1/asn1.py new file mode 100644 index 0000000..dedad6f --- /dev/null +++ b/lib/cryptography/hazmat/asn1/asn1.py @@ -0,0 +1,116 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import dataclasses +import sys +import typing + +if sys.version_info < (3, 11): + import typing_extensions + + # We use the `include_extras` parameter of `get_type_hints`, which was + # added in Python 3.9. This can be replaced by the `typing` version + # once the min version is >= 3.9 + if sys.version_info < (3, 9): + get_type_hints = typing_extensions.get_type_hints + else: + get_type_hints = typing.get_type_hints +else: + get_type_hints = typing.get_type_hints + +from cryptography.hazmat.bindings._rust import declarative_asn1 + +T = typing.TypeVar("T", covariant=True) +U = typing.TypeVar("U") + + +encode_der = declarative_asn1.encode_der + + +def _normalize_field_type( + field_type: typing.Any, field_name: str +) -> declarative_asn1.AnnotatedType: + annotation = declarative_asn1.Annotation() + + if hasattr(field_type, "__asn1_root__"): + annotated_root = field_type.__asn1_root__ + if not isinstance(annotated_root, declarative_asn1.AnnotatedType): + raise TypeError(f"unsupported root type: {annotated_root}") + return annotated_root + else: + rust_field_type = declarative_asn1.non_root_python_to_rust(field_type) + + return declarative_asn1.AnnotatedType(rust_field_type, annotation) + + +def _annotate_fields( + raw_fields: dict[str, type], +) -> dict[str, declarative_asn1.AnnotatedType]: + fields = {} + for field_name, field_type in raw_fields.items(): + # Recursively normalize the field type into something that the + # Rust code can understand. + annotated_field_type = _normalize_field_type(field_type, field_name) + fields[field_name] = annotated_field_type + + return fields + + +def _register_asn1_sequence(cls: type[U]) -> None: + raw_fields = get_type_hints(cls, include_extras=True) + root = declarative_asn1.AnnotatedType( + declarative_asn1.Type.Sequence(cls, _annotate_fields(raw_fields)), + declarative_asn1.Annotation(), + ) + + setattr(cls, "__asn1_root__", root) + + +# Due to https://github.com/python/mypy/issues/19731, we can't define an alias +# for `dataclass_transform` that conditionally points to `typing` or +# `typing_extensions` depending on the Python version (like we do for +# `get_type_hints`). +# We work around it by making the whole decorated class conditional on the +# Python version. +if sys.version_info < (3, 11): + + @typing_extensions.dataclass_transform(kw_only_default=True) + def sequence(cls: type[U]) -> type[U]: + # We use `dataclasses.dataclass` to add an __init__ method + # to the class with keyword-only parameters. + if sys.version_info >= (3, 10): + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + # `match_args` was added in Python 3.10 and defaults + # to True + match_args=False, + # `kw_only` was added in Python 3.10 and defaults to + # False + kw_only=True, + )(cls) + else: + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + )(cls) + _register_asn1_sequence(dataclass_cls) + return dataclass_cls + +else: + + @typing.dataclass_transform(kw_only_default=True) + def sequence(cls: type[U]) -> type[U]: + # Only add an __init__ method, with keyword-only + # parameters. + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + match_args=False, + kw_only=True, + )(cls) + _register_asn1_sequence(dataclass_cls) + return dataclass_cls diff --git a/lib/cryptography/hazmat/backends/__init__.py b/lib/cryptography/hazmat/backends/__init__.py new file mode 100644 index 0000000..b4400aa --- /dev/null +++ b/lib/cryptography/hazmat/backends/__init__.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from typing import Any + + +def default_backend() -> Any: + from cryptography.hazmat.backends.openssl.backend import backend + + return backend diff --git a/lib/cryptography/hazmat/backends/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/backends/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..d584e42 Binary files /dev/null and b/lib/cryptography/hazmat/backends/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/backends/openssl/__init__.py b/lib/cryptography/hazmat/backends/openssl/__init__.py new file mode 100644 index 0000000..51b0447 --- /dev/null +++ b/lib/cryptography/hazmat/backends/openssl/__init__.py @@ -0,0 +1,9 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.backends.openssl.backend import backend + +__all__ = ["backend"] diff --git a/lib/cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..54eee6e Binary files /dev/null and b/lib/cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-314.pyc b/lib/cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-314.pyc new file mode 100644 index 0000000..cb83ee2 Binary files /dev/null and b/lib/cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/backends/openssl/backend.py b/lib/cryptography/hazmat/backends/openssl/backend.py new file mode 100644 index 0000000..248b8c5 --- /dev/null +++ b/lib/cryptography/hazmat/backends/openssl/backend.py @@ -0,0 +1,302 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.bindings.openssl import binding +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives._asymmetric import AsymmetricPadding +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils +from cryptography.hazmat.primitives.asymmetric.padding import ( + MGF1, + OAEP, + PSS, + PKCS1v15, +) +from cryptography.hazmat.primitives.ciphers import ( + CipherAlgorithm, +) +from cryptography.hazmat.primitives.ciphers.algorithms import ( + AES, +) +from cryptography.hazmat.primitives.ciphers.modes import ( + CBC, + Mode, +) + + +class Backend: + """ + OpenSSL API binding interfaces. + """ + + name = "openssl" + + # TripleDES encryption is disallowed/deprecated throughout 2023 in + # FIPS 140-3. To keep it simple we denylist any use of TripleDES (TDEA). + _fips_ciphers = (AES,) + # Sometimes SHA1 is still permissible. That logic is contained + # within the various *_supported methods. + _fips_hashes = ( + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA512_224, + hashes.SHA512_256, + hashes.SHA3_224, + hashes.SHA3_256, + hashes.SHA3_384, + hashes.SHA3_512, + hashes.SHAKE128, + hashes.SHAKE256, + ) + _fips_ecdh_curves = ( + ec.SECP224R1, + ec.SECP256R1, + ec.SECP384R1, + ec.SECP521R1, + ) + _fips_rsa_min_key_size = 2048 + _fips_rsa_min_public_exponent = 65537 + _fips_dsa_min_modulus = 1 << 2048 + _fips_dh_min_key_size = 2048 + _fips_dh_min_modulus = 1 << _fips_dh_min_key_size + + def __init__(self) -> None: + self._binding = binding.Binding() + self._ffi = self._binding.ffi + self._lib = self._binding.lib + self._fips_enabled = rust_openssl.is_fips_enabled() + + def __repr__(self) -> str: + return ( + f"" + ) + + def openssl_assert(self, ok: bool) -> None: + return binding._openssl_assert(ok) + + def _enable_fips(self) -> None: + # This function enables FIPS mode for OpenSSL 3.0.0 on installs that + # have the FIPS provider installed properly. + rust_openssl.enable_fips(rust_openssl._providers) + assert rust_openssl.is_fips_enabled() + self._fips_enabled = rust_openssl.is_fips_enabled() + + def openssl_version_text(self) -> str: + """ + Friendly string name of the loaded OpenSSL library. This is not + necessarily the same version as it was compiled against. + + Example: OpenSSL 3.2.1 30 Jan 2024 + """ + return rust_openssl.openssl_version_text() + + def openssl_version_number(self) -> int: + return rust_openssl.openssl_version() + + def hash_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + if self._fips_enabled and not isinstance(algorithm, self._fips_hashes): + return False + + return rust_openssl.hashes.hash_supported(algorithm) + + def signature_hash_supported( + self, algorithm: hashes.HashAlgorithm + ) -> bool: + # Dedicated check for hashing algorithm use in message digest for + # signatures, e.g. RSA PKCS#1 v1.5 SHA1 (sha1WithRSAEncryption). + if self._fips_enabled and isinstance(algorithm, hashes.SHA1): + return False + return self.hash_supported(algorithm) + + def scrypt_supported(self) -> bool: + if self._fips_enabled: + return False + else: + return hasattr(rust_openssl.kdf.Scrypt, "derive") + + def argon2_supported(self) -> bool: + if self._fips_enabled: + return False + else: + return hasattr(rust_openssl.kdf.Argon2id, "derive") + + def hmac_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + # FIPS mode still allows SHA1 for HMAC + if self._fips_enabled and isinstance(algorithm, hashes.SHA1): + return True + if rust_openssl.CRYPTOGRAPHY_IS_AWSLC: + return isinstance( + algorithm, + ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA512_224, + hashes.SHA512_256, + ), + ) + return self.hash_supported(algorithm) + + def cipher_supported(self, cipher: CipherAlgorithm, mode: Mode) -> bool: + if self._fips_enabled: + # FIPS mode requires AES. TripleDES is disallowed/deprecated in + # FIPS 140-3. + if not isinstance(cipher, self._fips_ciphers): + return False + + return rust_openssl.ciphers.cipher_supported(cipher, mode) + + def pbkdf2_hmac_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + return self.hmac_supported(algorithm) + + def _consume_errors(self) -> list[rust_openssl.OpenSSLError]: + return rust_openssl.capture_error_stack() + + def _oaep_hash_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + if self._fips_enabled and isinstance(algorithm, hashes.SHA1): + return False + + return isinstance( + algorithm, + ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + ), + ) + + def rsa_padding_supported(self, padding: AsymmetricPadding) -> bool: + if isinstance(padding, PKCS1v15): + return True + elif isinstance(padding, PSS) and isinstance(padding._mgf, MGF1): + # FIPS 186-4 only allows salt length == digest length for PSS + # It is technically acceptable to set an explicit salt length + # equal to the digest length and this will incorrectly fail, but + # since we don't do that in the tests and this method is + # private, we'll ignore that until we need to do otherwise. + if ( + self._fips_enabled + and padding._salt_length != PSS.DIGEST_LENGTH + ): + return False + return self.hash_supported(padding._mgf._algorithm) + elif isinstance(padding, OAEP) and isinstance(padding._mgf, MGF1): + return self._oaep_hash_supported( + padding._mgf._algorithm + ) and self._oaep_hash_supported(padding._algorithm) + else: + return False + + def rsa_encryption_supported(self, padding: AsymmetricPadding) -> bool: + if self._fips_enabled and isinstance(padding, PKCS1v15): + return False + else: + return self.rsa_padding_supported(padding) + + def dsa_supported(self) -> bool: + return ( + not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not self._fips_enabled + ) + + def dsa_hash_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + if not self.dsa_supported(): + return False + return self.signature_hash_supported(algorithm) + + def cmac_algorithm_supported(self, algorithm) -> bool: + return self.cipher_supported( + algorithm, CBC(b"\x00" * algorithm.block_size) + ) + + def elliptic_curve_supported(self, curve: ec.EllipticCurve) -> bool: + if self._fips_enabled and not isinstance( + curve, self._fips_ecdh_curves + ): + return False + + return rust_openssl.ec.curve_supported(curve) + + def elliptic_curve_signature_algorithm_supported( + self, + signature_algorithm: ec.EllipticCurveSignatureAlgorithm, + curve: ec.EllipticCurve, + ) -> bool: + # We only support ECDSA right now. + if not isinstance(signature_algorithm, ec.ECDSA): + return False + + return self.elliptic_curve_supported(curve) and ( + isinstance(signature_algorithm.algorithm, asym_utils.Prehashed) + or self.hash_supported(signature_algorithm.algorithm) + ) + + def elliptic_curve_exchange_algorithm_supported( + self, algorithm: ec.ECDH, curve: ec.EllipticCurve + ) -> bool: + return self.elliptic_curve_supported(curve) and isinstance( + algorithm, ec.ECDH + ) + + def dh_supported(self) -> bool: + return ( + not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not rust_openssl.CRYPTOGRAPHY_IS_AWSLC + ) + + def dh_x942_serialization_supported(self) -> bool: + return self._lib.Cryptography_HAS_EVP_PKEY_DHX == 1 + + def x25519_supported(self) -> bool: + return not self._fips_enabled + + def x448_supported(self) -> bool: + if self._fips_enabled: + return False + return ( + not rust_openssl.CRYPTOGRAPHY_IS_LIBRESSL + and not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not rust_openssl.CRYPTOGRAPHY_IS_AWSLC + ) + + def ed25519_supported(self) -> bool: + return not self._fips_enabled + + def ed448_supported(self) -> bool: + if self._fips_enabled: + return False + return ( + not rust_openssl.CRYPTOGRAPHY_IS_LIBRESSL + and not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not rust_openssl.CRYPTOGRAPHY_IS_AWSLC + ) + + def ecdsa_deterministic_supported(self) -> bool: + return ( + rust_openssl.CRYPTOGRAPHY_OPENSSL_320_OR_GREATER + and not self._fips_enabled + ) + + def poly1305_supported(self) -> bool: + return not self._fips_enabled + + def pkcs7_supported(self) -> bool: + return ( + not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not rust_openssl.CRYPTOGRAPHY_IS_AWSLC + ) + + +backend = Backend() diff --git a/lib/cryptography/hazmat/bindings/__init__.py b/lib/cryptography/hazmat/bindings/__init__.py new file mode 100644 index 0000000..b509336 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/__init__.py @@ -0,0 +1,3 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. diff --git a/lib/cryptography/hazmat/bindings/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/bindings/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..490e2cb Binary files /dev/null and b/lib/cryptography/hazmat/bindings/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/bindings/_rust.abi3.so b/lib/cryptography/hazmat/bindings/_rust.abi3.so new file mode 100755 index 0000000..edefdfc Binary files /dev/null and b/lib/cryptography/hazmat/bindings/_rust.abi3.so differ diff --git a/lib/cryptography/hazmat/bindings/_rust/__init__.pyi b/lib/cryptography/hazmat/bindings/_rust/__init__.pyi new file mode 100644 index 0000000..2f4eef4 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/__init__.pyi @@ -0,0 +1,37 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import padding +from cryptography.utils import Buffer + +class PKCS7PaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class ANSIX923PaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class PKCS7UnpaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class ANSIX923UnpaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class ObjectIdentifier: + def __init__(self, value: str) -> None: ... + @property + def dotted_string(self) -> str: ... + @property + def _name(self) -> str: ... + +T = typing.TypeVar("T") diff --git a/lib/cryptography/hazmat/bindings/_rust/_openssl.pyi b/lib/cryptography/hazmat/bindings/_rust/_openssl.pyi new file mode 100644 index 0000000..8010008 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/_openssl.pyi @@ -0,0 +1,8 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +lib = typing.Any +ffi = typing.Any diff --git a/lib/cryptography/hazmat/bindings/_rust/asn1.pyi b/lib/cryptography/hazmat/bindings/_rust/asn1.pyi new file mode 100644 index 0000000..3b5f208 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/asn1.pyi @@ -0,0 +1,7 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +def decode_dss_signature(signature: bytes) -> tuple[int, int]: ... +def encode_dss_signature(r: int, s: int) -> bytes: ... +def parse_spki_for_data(data: bytes) -> bytes: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/declarative_asn1.pyi b/lib/cryptography/hazmat/bindings/_rust/declarative_asn1.pyi new file mode 100644 index 0000000..8563c11 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/declarative_asn1.pyi @@ -0,0 +1,32 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +import typing + +def encode_der(value: typing.Any) -> bytes: ... +def non_root_python_to_rust(cls: type) -> Type: ... + +# Type is a Rust enum with tuple variants. For now, we express the type +# annotations like this: +class Type: + Sequence: typing.ClassVar[type] + PyInt: typing.ClassVar[type] + +class Annotation: + def __new__( + cls, + ) -> Annotation: ... + +class AnnotatedType: + inner: Type + annotation: Annotation + + def __new__(cls, inner: Type, annotation: Annotation) -> AnnotatedType: ... + +class AnnotatedTypeObject: + annotated_type: AnnotatedType + value: typing.Any + + def __new__( + cls, annotated_type: AnnotatedType, value: typing.Any + ) -> AnnotatedTypeObject: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/exceptions.pyi b/lib/cryptography/hazmat/bindings/_rust/exceptions.pyi new file mode 100644 index 0000000..09f46b1 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/exceptions.pyi @@ -0,0 +1,17 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +class _Reasons: + BACKEND_MISSING_INTERFACE: _Reasons + UNSUPPORTED_HASH: _Reasons + UNSUPPORTED_CIPHER: _Reasons + UNSUPPORTED_PADDING: _Reasons + UNSUPPORTED_MGF: _Reasons + UNSUPPORTED_PUBLIC_KEY_ALGORITHM: _Reasons + UNSUPPORTED_ELLIPTIC_CURVE: _Reasons + UNSUPPORTED_SERIALIZATION: _Reasons + UNSUPPORTED_X509: _Reasons + UNSUPPORTED_EXCHANGE_ALGORITHM: _Reasons + UNSUPPORTED_DIFFIE_HELLMAN: _Reasons + UNSUPPORTED_MAC: _Reasons diff --git a/lib/cryptography/hazmat/bindings/_rust/ocsp.pyi b/lib/cryptography/hazmat/bindings/_rust/ocsp.pyi new file mode 100644 index 0000000..103e96c --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/ocsp.pyi @@ -0,0 +1,117 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import datetime +from collections.abc import Iterator + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes +from cryptography.x509 import ocsp + +class OCSPRequest: + @property + def issuer_key_hash(self) -> bytes: ... + @property + def issuer_name_hash(self) -> bytes: ... + @property + def hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def serial_number(self) -> int: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + @property + def extensions(self) -> x509.Extensions: ... + +class OCSPResponse: + @property + def responses(self) -> Iterator[OCSPSingleResponse]: ... + @property + def response_status(self) -> ocsp.OCSPResponseStatus: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_response_bytes(self) -> bytes: ... + @property + def certificates(self) -> list[x509.Certificate]: ... + @property + def responder_key_hash(self) -> bytes | None: ... + @property + def responder_name(self) -> x509.Name | None: ... + @property + def produced_at(self) -> datetime.datetime: ... + @property + def produced_at_utc(self) -> datetime.datetime: ... + @property + def certificate_status(self) -> ocsp.OCSPCertStatus: ... + @property + def revocation_time(self) -> datetime.datetime | None: ... + @property + def revocation_time_utc(self) -> datetime.datetime | None: ... + @property + def revocation_reason(self) -> x509.ReasonFlags | None: ... + @property + def this_update(self) -> datetime.datetime: ... + @property + def this_update_utc(self) -> datetime.datetime: ... + @property + def next_update(self) -> datetime.datetime | None: ... + @property + def next_update_utc(self) -> datetime.datetime | None: ... + @property + def issuer_key_hash(self) -> bytes: ... + @property + def issuer_name_hash(self) -> bytes: ... + @property + def hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def serial_number(self) -> int: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def single_extensions(self) -> x509.Extensions: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + +class OCSPSingleResponse: + @property + def certificate_status(self) -> ocsp.OCSPCertStatus: ... + @property + def revocation_time(self) -> datetime.datetime | None: ... + @property + def revocation_time_utc(self) -> datetime.datetime | None: ... + @property + def revocation_reason(self) -> x509.ReasonFlags | None: ... + @property + def this_update(self) -> datetime.datetime: ... + @property + def this_update_utc(self) -> datetime.datetime: ... + @property + def next_update(self) -> datetime.datetime | None: ... + @property + def next_update_utc(self) -> datetime.datetime | None: ... + @property + def issuer_key_hash(self) -> bytes: ... + @property + def issuer_name_hash(self) -> bytes: ... + @property + def hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def serial_number(self) -> int: ... + +def load_der_ocsp_request(data: bytes) -> ocsp.OCSPRequest: ... +def load_der_ocsp_response(data: bytes) -> ocsp.OCSPResponse: ... +def create_ocsp_request( + builder: ocsp.OCSPRequestBuilder, +) -> ocsp.OCSPRequest: ... +def create_ocsp_response( + status: ocsp.OCSPResponseStatus, + builder: ocsp.OCSPResponseBuilder | None, + private_key: PrivateKeyTypes | None, + hash_algorithm: hashes.HashAlgorithm | None, +) -> ocsp.OCSPResponse: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi new file mode 100644 index 0000000..5fb3cb2 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi @@ -0,0 +1,75 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.bindings._rust.openssl import ( + aead, + ciphers, + cmac, + dh, + dsa, + ec, + ed448, + ed25519, + hashes, + hmac, + kdf, + keys, + poly1305, + rsa, + x448, + x25519, +) + +__all__ = [ + "aead", + "ciphers", + "cmac", + "dh", + "dsa", + "ec", + "ed448", + "ed25519", + "hashes", + "hmac", + "kdf", + "keys", + "openssl_version", + "openssl_version_text", + "poly1305", + "raise_openssl_error", + "rsa", + "x448", + "x25519", +] + +CRYPTOGRAPHY_IS_LIBRESSL: bool +CRYPTOGRAPHY_IS_BORINGSSL: bool +CRYPTOGRAPHY_IS_AWSLC: bool +CRYPTOGRAPHY_OPENSSL_300_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_309_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_320_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_330_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_350_OR_GREATER: bool + +class Providers: ... + +_legacy_provider_loaded: bool +_providers: Providers + +def openssl_version() -> int: ... +def openssl_version_text() -> str: ... +def raise_openssl_error() -> typing.NoReturn: ... +def capture_error_stack() -> list[OpenSSLError]: ... +def is_fips_enabled() -> bool: ... +def enable_fips(providers: Providers) -> None: ... + +class OpenSSLError: + @property + def lib(self) -> int: ... + @property + def reason(self) -> int: ... + @property + def reason_text(self) -> bytes: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/aead.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/aead.pyi new file mode 100644 index 0000000..831fcd1 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/aead.pyi @@ -0,0 +1,107 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from collections.abc import Sequence + +from cryptography.utils import Buffer + +class AESGCM: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + +class ChaCha20Poly1305: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key() -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + +class AESCCM: + def __init__(self, key: Buffer, tag_length: int = 16) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + +class AESSIV: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + data: Buffer, + associated_data: Sequence[Buffer] | None, + ) -> bytes: ... + def decrypt( + self, + data: Buffer, + associated_data: Sequence[Buffer] | None, + ) -> bytes: ... + +class AESOCB3: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + +class AESGCMSIV: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi new file mode 100644 index 0000000..a48fb01 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi @@ -0,0 +1,38 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import ciphers +from cryptography.hazmat.primitives.ciphers import modes + +@typing.overload +def create_encryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.ModeWithAuthenticationTag +) -> ciphers.AEADEncryptionContext: ... +@typing.overload +def create_encryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.Mode | None +) -> ciphers.CipherContext: ... +@typing.overload +def create_decryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.ModeWithAuthenticationTag +) -> ciphers.AEADDecryptionContext: ... +@typing.overload +def create_decryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.Mode | None +) -> ciphers.CipherContext: ... +def cipher_supported( + algorithm: ciphers.CipherAlgorithm, mode: modes.Mode +) -> bool: ... +def _advance( + ctx: ciphers.AEADEncryptionContext | ciphers.AEADDecryptionContext, n: int +) -> None: ... +def _advance_aad( + ctx: ciphers.AEADEncryptionContext | ciphers.AEADDecryptionContext, n: int +) -> None: ... + +class CipherContext: ... +class AEADEncryptionContext: ... +class AEADDecryptionContext: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi new file mode 100644 index 0000000..9c03508 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi @@ -0,0 +1,18 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import ciphers + +class CMAC: + def __init__( + self, + algorithm: ciphers.BlockCipherAlgorithm, + backend: typing.Any = None, + ) -> None: ... + def update(self, data: bytes) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, signature: bytes) -> None: ... + def copy(self) -> CMAC: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/dh.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/dh.pyi new file mode 100644 index 0000000..08733d7 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/dh.pyi @@ -0,0 +1,51 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import dh + +MIN_MODULUS_SIZE: int + +class DHPrivateKey: ... +class DHPublicKey: ... +class DHParameters: ... + +class DHPrivateNumbers: + def __init__(self, x: int, public_numbers: DHPublicNumbers) -> None: ... + def private_key(self, backend: typing.Any = None) -> dh.DHPrivateKey: ... + @property + def x(self) -> int: ... + @property + def public_numbers(self) -> DHPublicNumbers: ... + +class DHPublicNumbers: + def __init__( + self, y: int, parameter_numbers: DHParameterNumbers + ) -> None: ... + def public_key(self, backend: typing.Any = None) -> dh.DHPublicKey: ... + @property + def y(self) -> int: ... + @property + def parameter_numbers(self) -> DHParameterNumbers: ... + +class DHParameterNumbers: + def __init__(self, p: int, g: int, q: int | None = None) -> None: ... + def parameters(self, backend: typing.Any = None) -> dh.DHParameters: ... + @property + def p(self) -> int: ... + @property + def g(self) -> int: ... + @property + def q(self) -> int | None: ... + +def generate_parameters( + generator: int, key_size: int, backend: typing.Any = None +) -> dh.DHParameters: ... +def from_pem_parameters( + data: bytes, backend: typing.Any = None +) -> dh.DHParameters: ... +def from_der_parameters( + data: bytes, backend: typing.Any = None +) -> dh.DHParameters: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/dsa.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/dsa.pyi new file mode 100644 index 0000000..0922a4c --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/dsa.pyi @@ -0,0 +1,41 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import dsa + +class DSAPrivateKey: ... +class DSAPublicKey: ... +class DSAParameters: ... + +class DSAPrivateNumbers: + def __init__(self, x: int, public_numbers: DSAPublicNumbers) -> None: ... + @property + def x(self) -> int: ... + @property + def public_numbers(self) -> DSAPublicNumbers: ... + def private_key(self, backend: typing.Any = None) -> dsa.DSAPrivateKey: ... + +class DSAPublicNumbers: + def __init__( + self, y: int, parameter_numbers: DSAParameterNumbers + ) -> None: ... + @property + def y(self) -> int: ... + @property + def parameter_numbers(self) -> DSAParameterNumbers: ... + def public_key(self, backend: typing.Any = None) -> dsa.DSAPublicKey: ... + +class DSAParameterNumbers: + def __init__(self, p: int, q: int, g: int) -> None: ... + @property + def p(self) -> int: ... + @property + def q(self) -> int: ... + @property + def g(self) -> int: ... + def parameters(self, backend: typing.Any = None) -> dsa.DSAParameters: ... + +def generate_parameters(key_size: int) -> dsa.DSAParameters: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/ec.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/ec.pyi new file mode 100644 index 0000000..5c3b7bf --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/ec.pyi @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import ec + +class ECPrivateKey: ... +class ECPublicKey: ... + +class EllipticCurvePrivateNumbers: + def __init__( + self, private_value: int, public_numbers: EllipticCurvePublicNumbers + ) -> None: ... + def private_key( + self, backend: typing.Any = None + ) -> ec.EllipticCurvePrivateKey: ... + @property + def private_value(self) -> int: ... + @property + def public_numbers(self) -> EllipticCurvePublicNumbers: ... + +class EllipticCurvePublicNumbers: + def __init__(self, x: int, y: int, curve: ec.EllipticCurve) -> None: ... + def public_key( + self, backend: typing.Any = None + ) -> ec.EllipticCurvePublicKey: ... + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + @property + def curve(self) -> ec.EllipticCurve: ... + def __eq__(self, other: object) -> bool: ... + +def curve_supported(curve: ec.EllipticCurve) -> bool: ... +def generate_private_key( + curve: ec.EllipticCurve, backend: typing.Any = None +) -> ec.EllipticCurvePrivateKey: ... +def from_private_numbers( + numbers: ec.EllipticCurvePrivateNumbers, +) -> ec.EllipticCurvePrivateKey: ... +def from_public_numbers( + numbers: ec.EllipticCurvePublicNumbers, +) -> ec.EllipticCurvePublicKey: ... +def from_public_bytes( + curve: ec.EllipticCurve, data: bytes +) -> ec.EllipticCurvePublicKey: ... +def derive_private_key( + private_value: int, curve: ec.EllipticCurve +) -> ec.EllipticCurvePrivateKey: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi new file mode 100644 index 0000000..f85b3d1 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ed25519 +from cryptography.utils import Buffer + +class Ed25519PrivateKey: ... +class Ed25519PublicKey: ... + +def generate_key() -> ed25519.Ed25519PrivateKey: ... +def from_private_bytes(data: Buffer) -> ed25519.Ed25519PrivateKey: ... +def from_public_bytes(data: bytes) -> ed25519.Ed25519PublicKey: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi new file mode 100644 index 0000000..c8ca0ec --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ed448 +from cryptography.utils import Buffer + +class Ed448PrivateKey: ... +class Ed448PublicKey: ... + +def generate_key() -> ed448.Ed448PrivateKey: ... +def from_private_bytes(data: Buffer) -> ed448.Ed448PrivateKey: ... +def from_public_bytes(data: bytes) -> ed448.Ed448PublicKey: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi new file mode 100644 index 0000000..6bfd295 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi @@ -0,0 +1,28 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import hashes +from cryptography.utils import Buffer + +class Hash(hashes.HashContext): + def __init__( + self, algorithm: hashes.HashAlgorithm, backend: typing.Any = None + ) -> None: ... + @property + def algorithm(self) -> hashes.HashAlgorithm: ... + def update(self, data: Buffer) -> None: ... + def finalize(self) -> bytes: ... + def copy(self) -> Hash: ... + +def hash_supported(algorithm: hashes.HashAlgorithm) -> bool: ... + +class XOFHash: + def __init__(self, algorithm: hashes.ExtendableOutputFunction) -> None: ... + @property + def algorithm(self) -> hashes.ExtendableOutputFunction: ... + def update(self, data: Buffer) -> None: ... + def squeeze(self, length: int) -> bytes: ... + def copy(self) -> XOFHash: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi new file mode 100644 index 0000000..3883d1b --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi @@ -0,0 +1,22 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import hashes +from cryptography.utils import Buffer + +class HMAC(hashes.HashContext): + def __init__( + self, + key: Buffer, + algorithm: hashes.HashAlgorithm, + backend: typing.Any = None, + ) -> None: ... + @property + def algorithm(self) -> hashes.HashAlgorithm: ... + def update(self, data: Buffer) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, signature: bytes) -> None: ... + def copy(self) -> HMAC: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi new file mode 100644 index 0000000..9e2d8d9 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi @@ -0,0 +1,72 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.hashes import HashAlgorithm +from cryptography.utils import Buffer + +def derive_pbkdf2_hmac( + key_material: Buffer, + algorithm: HashAlgorithm, + salt: bytes, + iterations: int, + length: int, +) -> bytes: ... + +class Scrypt: + def __init__( + self, + salt: bytes, + length: int, + n: int, + r: int, + p: int, + backend: typing.Any = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class Argon2id: + def __init__( + self, + *, + salt: bytes, + length: int, + iterations: int, + lanes: int, + memory_cost: int, + ad: bytes | None = None, + secret: bytes | None = None, + ) -> None: ... + def derive(self, key_material: bytes) -> bytes: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + def derive_phc_encoded(self, key_material: bytes) -> str: ... + @classmethod + def verify_phc_encoded( + cls, key_material: bytes, phc_encoded: str, secret: bytes | None = None + ) -> None: ... + +class HKDF: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + salt: bytes | None, + info: bytes | None, + backend: typing.Any = None, + ): ... + def derive(self, key_material: Buffer) -> bytes: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class HKDFExpand: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + info: bytes | None, + backend: typing.Any = None, + ): ... + def derive(self, key_material: Buffer) -> bytes: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/keys.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/keys.pyi new file mode 100644 index 0000000..404057e --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/keys.pyi @@ -0,0 +1,34 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric.types import ( + PrivateKeyTypes, + PublicKeyTypes, +) +from cryptography.utils import Buffer + +def load_der_private_key( + data: Buffer, + password: bytes | None, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, +) -> PrivateKeyTypes: ... +def load_pem_private_key( + data: Buffer, + password: bytes | None, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, +) -> PrivateKeyTypes: ... +def load_der_public_key( + data: bytes, + backend: typing.Any = None, +) -> PublicKeyTypes: ... +def load_pem_public_key( + data: bytes, + backend: typing.Any = None, +) -> PublicKeyTypes: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi new file mode 100644 index 0000000..45a2a39 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi @@ -0,0 +1,15 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.utils import Buffer + +class Poly1305: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_tag(key: Buffer, data: Buffer) -> bytes: ... + @staticmethod + def verify_tag(key: Buffer, data: Buffer, tag: bytes) -> None: ... + def update(self, data: Buffer) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, tag: bytes) -> None: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi new file mode 100644 index 0000000..ef7752d --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi @@ -0,0 +1,55 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import rsa + +class RSAPrivateKey: ... +class RSAPublicKey: ... + +class RSAPrivateNumbers: + def __init__( + self, + p: int, + q: int, + d: int, + dmp1: int, + dmq1: int, + iqmp: int, + public_numbers: RSAPublicNumbers, + ) -> None: ... + @property + def p(self) -> int: ... + @property + def q(self) -> int: ... + @property + def d(self) -> int: ... + @property + def dmp1(self) -> int: ... + @property + def dmq1(self) -> int: ... + @property + def iqmp(self) -> int: ... + @property + def public_numbers(self) -> RSAPublicNumbers: ... + def private_key( + self, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, + ) -> rsa.RSAPrivateKey: ... + +class RSAPublicNumbers: + def __init__(self, e: int, n: int) -> None: ... + @property + def n(self) -> int: ... + @property + def e(self) -> int: ... + def public_key(self, backend: typing.Any = None) -> rsa.RSAPublicKey: ... + +def generate_private_key( + public_exponent: int, + key_size: int, +) -> rsa.RSAPrivateKey: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/x25519.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/x25519.pyi new file mode 100644 index 0000000..38d2add --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/x25519.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.utils import Buffer + +class X25519PrivateKey: ... +class X25519PublicKey: ... + +def generate_key() -> x25519.X25519PrivateKey: ... +def from_private_bytes(data: Buffer) -> x25519.X25519PrivateKey: ... +def from_public_bytes(data: bytes) -> x25519.X25519PublicKey: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/openssl/x448.pyi b/lib/cryptography/hazmat/bindings/_rust/openssl/x448.pyi new file mode 100644 index 0000000..3ac0980 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/openssl/x448.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import x448 +from cryptography.utils import Buffer + +class X448PrivateKey: ... +class X448PublicKey: ... + +def generate_key() -> x448.X448PrivateKey: ... +def from_private_bytes(data: Buffer) -> x448.X448PrivateKey: ... +def from_public_bytes(data: bytes) -> x448.X448PublicKey: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/pkcs12.pyi b/lib/cryptography/hazmat/bindings/_rust/pkcs12.pyi new file mode 100644 index 0000000..b25becb --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/pkcs12.pyi @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing +from collections.abc import Iterable + +from cryptography import x509 +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes +from cryptography.hazmat.primitives.serialization import ( + KeySerializationEncryption, +) +from cryptography.hazmat.primitives.serialization.pkcs12 import ( + PKCS12KeyAndCertificates, + PKCS12PrivateKeyTypes, +) +from cryptography.utils import Buffer + +class PKCS12Certificate: + def __init__( + self, cert: x509.Certificate, friendly_name: bytes | None + ) -> None: ... + @property + def friendly_name(self) -> bytes | None: ... + @property + def certificate(self) -> x509.Certificate: ... + +def load_key_and_certificates( + data: Buffer, + password: Buffer | None, + backend: typing.Any = None, +) -> tuple[ + PrivateKeyTypes | None, + x509.Certificate | None, + list[x509.Certificate], +]: ... +def load_pkcs12( + data: bytes, + password: bytes | None, + backend: typing.Any = None, +) -> PKCS12KeyAndCertificates: ... +def serialize_java_truststore( + certs: Iterable[PKCS12Certificate], + encryption_algorithm: KeySerializationEncryption, +) -> bytes: ... +def serialize_key_and_certificates( + name: bytes | None, + key: PKCS12PrivateKeyTypes | None, + cert: x509.Certificate | None, + cas: Iterable[x509.Certificate | PKCS12Certificate] | None, + encryption_algorithm: KeySerializationEncryption, +) -> bytes: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/pkcs7.pyi b/lib/cryptography/hazmat/bindings/_rust/pkcs7.pyi new file mode 100644 index 0000000..358b135 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/pkcs7.pyi @@ -0,0 +1,50 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from collections.abc import Iterable + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import pkcs7 + +def serialize_certificates( + certs: list[x509.Certificate], + encoding: serialization.Encoding, +) -> bytes: ... +def encrypt_and_serialize( + builder: pkcs7.PKCS7EnvelopeBuilder, + content_encryption_algorithm: pkcs7.ContentEncryptionAlgorithm, + encoding: serialization.Encoding, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def sign_and_serialize( + builder: pkcs7.PKCS7SignatureBuilder, + encoding: serialization.Encoding, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def decrypt_der( + data: bytes, + certificate: x509.Certificate, + private_key: rsa.RSAPrivateKey, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def decrypt_pem( + data: bytes, + certificate: x509.Certificate, + private_key: rsa.RSAPrivateKey, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def decrypt_smime( + data: bytes, + certificate: x509.Certificate, + private_key: rsa.RSAPrivateKey, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def load_pem_pkcs7_certificates( + data: bytes, +) -> list[x509.Certificate]: ... +def load_der_pkcs7_certificates( + data: bytes, +) -> list[x509.Certificate]: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/test_support.pyi b/lib/cryptography/hazmat/bindings/_rust/test_support.pyi new file mode 100644 index 0000000..c6c6d0b --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/test_support.pyi @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs7 +from cryptography.utils import Buffer + +class TestCertificate: + not_after_tag: int + not_before_tag: int + issuer_value_tags: list[int] + subject_value_tags: list[int] + +def test_parse_certificate(data: bytes) -> TestCertificate: ... +def pkcs7_verify( + encoding: serialization.Encoding, + sig: bytes, + msg: Buffer | None, + certs: list[x509.Certificate], + options: list[pkcs7.PKCS7Options], +) -> None: ... diff --git a/lib/cryptography/hazmat/bindings/_rust/x509.pyi b/lib/cryptography/hazmat/bindings/_rust/x509.pyi new file mode 100644 index 0000000..83c3441 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/_rust/x509.pyi @@ -0,0 +1,301 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import datetime +import typing +from collections.abc import Iterator + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.padding import PSS, PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPublicKeyTypes, + CertificatePublicKeyTypes, + PrivateKeyTypes, +) +from cryptography.x509 import certificate_transparency + +def load_pem_x509_certificate( + data: bytes, backend: typing.Any = None +) -> x509.Certificate: ... +def load_der_x509_certificate( + data: bytes, backend: typing.Any = None +) -> x509.Certificate: ... +def load_pem_x509_certificates( + data: bytes, +) -> list[x509.Certificate]: ... +def load_pem_x509_crl( + data: bytes, backend: typing.Any = None +) -> x509.CertificateRevocationList: ... +def load_der_x509_crl( + data: bytes, backend: typing.Any = None +) -> x509.CertificateRevocationList: ... +def load_pem_x509_csr( + data: bytes, backend: typing.Any = None +) -> x509.CertificateSigningRequest: ... +def load_der_x509_csr( + data: bytes, backend: typing.Any = None +) -> x509.CertificateSigningRequest: ... +def encode_name_bytes(name: x509.Name) -> bytes: ... +def encode_extension_value(extension: x509.ExtensionType) -> bytes: ... +def create_x509_certificate( + builder: x509.CertificateBuilder, + private_key: PrivateKeyTypes, + hash_algorithm: hashes.HashAlgorithm | None, + rsa_padding: PKCS1v15 | PSS | None, + ecdsa_deterministic: bool | None, +) -> x509.Certificate: ... +def create_x509_csr( + builder: x509.CertificateSigningRequestBuilder, + private_key: PrivateKeyTypes, + hash_algorithm: hashes.HashAlgorithm | None, + rsa_padding: PKCS1v15 | PSS | None, + ecdsa_deterministic: bool | None, +) -> x509.CertificateSigningRequest: ... +def create_x509_crl( + builder: x509.CertificateRevocationListBuilder, + private_key: PrivateKeyTypes, + hash_algorithm: hashes.HashAlgorithm | None, + rsa_padding: PKCS1v15 | PSS | None, + ecdsa_deterministic: bool | None, +) -> x509.CertificateRevocationList: ... + +class Sct: + @property + def version(self) -> certificate_transparency.Version: ... + @property + def log_id(self) -> bytes: ... + @property + def timestamp(self) -> datetime.datetime: ... + @property + def entry_type(self) -> certificate_transparency.LogEntryType: ... + @property + def signature_hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def signature_algorithm( + self, + ) -> certificate_transparency.SignatureAlgorithm: ... + @property + def signature(self) -> bytes: ... + @property + def extension_bytes(self) -> bytes: ... + +class Certificate: + def fingerprint(self, algorithm: hashes.HashAlgorithm) -> bytes: ... + @property + def serial_number(self) -> int: ... + @property + def version(self) -> x509.Version: ... + def public_key(self) -> CertificatePublicKeyTypes: ... + @property + def public_key_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def not_valid_before(self) -> datetime.datetime: ... + @property + def not_valid_before_utc(self) -> datetime.datetime: ... + @property + def not_valid_after(self) -> datetime.datetime: ... + @property + def not_valid_after_utc(self) -> datetime.datetime: ... + @property + def issuer(self) -> x509.Name: ... + @property + def subject(self) -> x509.Name: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_algorithm_parameters( + self, + ) -> PSS | PKCS1v15 | ECDSA | None: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_certificate_bytes(self) -> bytes: ... + @property + def tbs_precertificate_bytes(self) -> bytes: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + def verify_directly_issued_by(self, issuer: Certificate) -> None: ... + +class RevokedCertificate: ... + +class CertificateRevocationList: + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + def fingerprint(self, algorithm: hashes.HashAlgorithm) -> bytes: ... + def get_revoked_certificate_by_serial_number( + self, serial_number: int + ) -> x509.RevokedCertificate | None: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_algorithm_parameters( + self, + ) -> PSS | PKCS1v15 | ECDSA | None: ... + @property + def issuer(self) -> x509.Name: ... + @property + def next_update(self) -> datetime.datetime | None: ... + @property + def next_update_utc(self) -> datetime.datetime | None: ... + @property + def last_update(self) -> datetime.datetime: ... + @property + def last_update_utc(self) -> datetime.datetime: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_certlist_bytes(self) -> bytes: ... + def __eq__(self, other: object) -> bool: ... + def __len__(self) -> int: ... + @typing.overload + def __getitem__(self, idx: int) -> x509.RevokedCertificate: ... + @typing.overload + def __getitem__(self, idx: slice) -> list[x509.RevokedCertificate]: ... + def __iter__(self) -> Iterator[x509.RevokedCertificate]: ... + def is_signature_valid( + self, public_key: CertificateIssuerPublicKeyTypes + ) -> bool: ... + +class CertificateSigningRequest: + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def public_key(self) -> CertificatePublicKeyTypes: ... + @property + def subject(self) -> x509.Name: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_algorithm_parameters( + self, + ) -> PSS | PKCS1v15 | ECDSA | None: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def attributes(self) -> x509.Attributes: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_certrequest_bytes(self) -> bytes: ... + @property + def is_signature_valid(self) -> bool: ... + +class PolicyBuilder: + def time(self, time: datetime.datetime) -> PolicyBuilder: ... + def store(self, store: Store) -> PolicyBuilder: ... + def max_chain_depth(self, max_chain_depth: int) -> PolicyBuilder: ... + def extension_policies( + self, *, ca_policy: ExtensionPolicy, ee_policy: ExtensionPolicy + ) -> PolicyBuilder: ... + def build_client_verifier(self) -> ClientVerifier: ... + def build_server_verifier( + self, subject: x509.verification.Subject + ) -> ServerVerifier: ... + +class Policy: + @property + def max_chain_depth(self) -> int: ... + @property + def subject(self) -> x509.verification.Subject | None: ... + @property + def validation_time(self) -> datetime.datetime: ... + @property + def extended_key_usage(self) -> x509.ObjectIdentifier: ... + @property + def minimum_rsa_modulus(self) -> int: ... + +class Criticality: + CRITICAL: Criticality + AGNOSTIC: Criticality + NON_CRITICAL: Criticality + +T = typing.TypeVar("T", contravariant=True, bound=x509.ExtensionType) + +MaybeExtensionValidatorCallback = typing.Callable[ + [ + Policy, + x509.Certificate, + T | None, + ], + None, +] + +PresentExtensionValidatorCallback = typing.Callable[ + [Policy, x509.Certificate, T], + None, +] + +class ExtensionPolicy: + @staticmethod + def permit_all() -> ExtensionPolicy: ... + @staticmethod + def webpki_defaults_ca() -> ExtensionPolicy: ... + @staticmethod + def webpki_defaults_ee() -> ExtensionPolicy: ... + def require_not_present( + self, extension_type: type[x509.ExtensionType] + ) -> ExtensionPolicy: ... + def may_be_present( + self, + extension_type: type[T], + criticality: Criticality, + validator: MaybeExtensionValidatorCallback[T] | None, + ) -> ExtensionPolicy: ... + def require_present( + self, + extension_type: type[T], + criticality: Criticality, + validator: PresentExtensionValidatorCallback[T] | None, + ) -> ExtensionPolicy: ... + +class VerifiedClient: + @property + def subjects(self) -> list[x509.GeneralName] | None: ... + @property + def chain(self) -> list[x509.Certificate]: ... + +class ClientVerifier: + @property + def policy(self) -> Policy: ... + @property + def store(self) -> Store: ... + def verify( + self, + leaf: x509.Certificate, + intermediates: list[x509.Certificate], + ) -> VerifiedClient: ... + +class ServerVerifier: + @property + def policy(self) -> Policy: ... + @property + def store(self) -> Store: ... + def verify( + self, + leaf: x509.Certificate, + intermediates: list[x509.Certificate], + ) -> list[x509.Certificate]: ... + +class Store: + def __init__(self, certs: list[x509.Certificate]) -> None: ... + +class VerificationError(Exception): ... diff --git a/lib/cryptography/hazmat/bindings/openssl/__init__.py b/lib/cryptography/hazmat/bindings/openssl/__init__.py new file mode 100644 index 0000000..b509336 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/openssl/__init__.py @@ -0,0 +1,3 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. diff --git a/lib/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..516a619 Binary files /dev/null and b/lib/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-314.pyc b/lib/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-314.pyc new file mode 100644 index 0000000..07fbf59 Binary files /dev/null and b/lib/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-314.pyc b/lib/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-314.pyc new file mode 100644 index 0000000..ab0f75d Binary files /dev/null and b/lib/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/bindings/openssl/_conditional.py b/lib/cryptography/hazmat/bindings/openssl/_conditional.py new file mode 100644 index 0000000..063bcf5 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/openssl/_conditional.py @@ -0,0 +1,207 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + + +def cryptography_has_set_cert_cb() -> list[str]: + return [ + "SSL_CTX_set_cert_cb", + "SSL_set_cert_cb", + ] + + +def cryptography_has_ssl_st() -> list[str]: + return [ + "SSL_ST_BEFORE", + "SSL_ST_OK", + "SSL_ST_INIT", + "SSL_ST_RENEGOTIATE", + ] + + +def cryptography_has_tls_st() -> list[str]: + return [ + "TLS_ST_BEFORE", + "TLS_ST_OK", + ] + + +def cryptography_has_ssl_sigalgs() -> list[str]: + return [ + "SSL_CTX_set1_sigalgs_list", + ] + + +def cryptography_has_psk() -> list[str]: + return [ + "SSL_CTX_use_psk_identity_hint", + "SSL_CTX_set_psk_server_callback", + "SSL_CTX_set_psk_client_callback", + ] + + +def cryptography_has_psk_tlsv13() -> list[str]: + return [ + "SSL_CTX_set_psk_find_session_callback", + "SSL_CTX_set_psk_use_session_callback", + "Cryptography_SSL_SESSION_new", + "SSL_CIPHER_find", + "SSL_SESSION_set1_master_key", + "SSL_SESSION_set_cipher", + "SSL_SESSION_set_protocol_version", + ] + + +def cryptography_has_custom_ext() -> list[str]: + return [ + "SSL_CTX_add_client_custom_ext", + "SSL_CTX_add_server_custom_ext", + "SSL_extension_supported", + ] + + +def cryptography_has_tlsv13_functions() -> list[str]: + return [ + "SSL_CTX_set_ciphersuites", + ] + + +def cryptography_has_tlsv13_hs_functions() -> list[str]: + return [ + "SSL_VERIFY_POST_HANDSHAKE", + "SSL_verify_client_post_handshake", + "SSL_CTX_set_post_handshake_auth", + "SSL_set_post_handshake_auth", + "SSL_SESSION_get_max_early_data", + "SSL_write_early_data", + "SSL_read_early_data", + "SSL_CTX_set_max_early_data", + ] + + +def cryptography_has_ssl_verify_client_post_handshake() -> list[str]: + return [ + "SSL_verify_client_post_handshake", + ] + + +def cryptography_has_engine() -> list[str]: + return [ + "ENGINE_by_id", + "ENGINE_init", + "ENGINE_finish", + "ENGINE_get_default_RAND", + "ENGINE_set_default_RAND", + "ENGINE_unregister_RAND", + "ENGINE_ctrl_cmd", + "ENGINE_free", + "ENGINE_get_name", + "ENGINE_ctrl_cmd_string", + "ENGINE_load_builtin_engines", + "ENGINE_load_private_key", + "ENGINE_load_public_key", + "SSL_CTX_set_client_cert_engine", + ] + + +def cryptography_has_verified_chain() -> list[str]: + return [ + "SSL_get0_verified_chain", + ] + + +def cryptography_has_srtp() -> list[str]: + return [ + "SSL_CTX_set_tlsext_use_srtp", + "SSL_set_tlsext_use_srtp", + "SSL_get_selected_srtp_profile", + ] + + +def cryptography_has_op_no_renegotiation() -> list[str]: + return [ + "SSL_OP_NO_RENEGOTIATION", + ] + + +def cryptography_has_dtls_get_data_mtu() -> list[str]: + return [ + "DTLS_get_data_mtu", + ] + + +def cryptography_has_ssl_cookie() -> list[str]: + return [ + "SSL_OP_COOKIE_EXCHANGE", + "DTLSv1_listen", + "SSL_CTX_set_cookie_generate_cb", + "SSL_CTX_set_cookie_verify_cb", + ] + + +def cryptography_has_prime_checks() -> list[str]: + return [ + "BN_prime_checks_for_size", + ] + + +def cryptography_has_unexpected_eof_while_reading() -> list[str]: + return ["SSL_R_UNEXPECTED_EOF_WHILE_READING"] + + +def cryptography_has_ssl_op_ignore_unexpected_eof() -> list[str]: + return [ + "SSL_OP_IGNORE_UNEXPECTED_EOF", + ] + + +def cryptography_has_get_extms_support() -> list[str]: + return ["SSL_get_extms_support"] + + +def cryptography_has_ssl_get0_group_name() -> list[str]: + return ["SSL_get0_group_name"] + + +# This is a mapping of +# {condition: function-returning-names-dependent-on-that-condition} so we can +# loop over them and delete unsupported names at runtime. It will be removed +# when cffi supports #if in cdef. We use functions instead of just a dict of +# lists so we can use coverage to measure which are used. +CONDITIONAL_NAMES = { + "Cryptography_HAS_SET_CERT_CB": cryptography_has_set_cert_cb, + "Cryptography_HAS_SSL_ST": cryptography_has_ssl_st, + "Cryptography_HAS_TLS_ST": cryptography_has_tls_st, + "Cryptography_HAS_SIGALGS": cryptography_has_ssl_sigalgs, + "Cryptography_HAS_PSK": cryptography_has_psk, + "Cryptography_HAS_PSK_TLSv1_3": cryptography_has_psk_tlsv13, + "Cryptography_HAS_CUSTOM_EXT": cryptography_has_custom_ext, + "Cryptography_HAS_TLSv1_3_FUNCTIONS": cryptography_has_tlsv13_functions, + "Cryptography_HAS_TLSv1_3_HS_FUNCTIONS": ( + cryptography_has_tlsv13_hs_functions + ), + "Cryptography_HAS_SSL_VERIFY_CLIENT_POST_HANDSHAKE": ( + cryptography_has_ssl_verify_client_post_handshake + ), + "Cryptography_HAS_ENGINE": cryptography_has_engine, + "Cryptography_HAS_VERIFIED_CHAIN": cryptography_has_verified_chain, + "Cryptography_HAS_SRTP": cryptography_has_srtp, + "Cryptography_HAS_OP_NO_RENEGOTIATION": ( + cryptography_has_op_no_renegotiation + ), + "Cryptography_HAS_DTLS_GET_DATA_MTU": cryptography_has_dtls_get_data_mtu, + "Cryptography_HAS_SSL_COOKIE": cryptography_has_ssl_cookie, + "Cryptography_HAS_PRIME_CHECKS": cryptography_has_prime_checks, + "Cryptography_HAS_UNEXPECTED_EOF_WHILE_READING": ( + cryptography_has_unexpected_eof_while_reading + ), + "Cryptography_HAS_SSL_OP_IGNORE_UNEXPECTED_EOF": ( + cryptography_has_ssl_op_ignore_unexpected_eof + ), + "Cryptography_HAS_GET_EXTMS_SUPPORT": cryptography_has_get_extms_support, + "Cryptography_HAS_SSL_GET0_GROUP_NAME": ( + cryptography_has_ssl_get0_group_name + ), +} diff --git a/lib/cryptography/hazmat/bindings/openssl/binding.py b/lib/cryptography/hazmat/bindings/openssl/binding.py new file mode 100644 index 0000000..4494c71 --- /dev/null +++ b/lib/cryptography/hazmat/bindings/openssl/binding.py @@ -0,0 +1,137 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import os +import sys +import threading +import types +import typing +import warnings +from collections.abc import Callable + +import cryptography +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._rust import _openssl, openssl +from cryptography.hazmat.bindings.openssl._conditional import CONDITIONAL_NAMES +from cryptography.utils import CryptographyDeprecationWarning + + +def _openssl_assert(ok: bool) -> None: + if not ok: + errors = openssl.capture_error_stack() + + raise InternalError( + "Unknown OpenSSL error. This error is commonly encountered when " + "another library is not cleaning up the OpenSSL error stack. If " + "you are using cryptography with another library that uses " + "OpenSSL try disabling it before reporting a bug. Otherwise " + "please file an issue at https://github.com/pyca/cryptography/" + "issues with information on how to reproduce " + f"this. ({errors!r})", + errors, + ) + + +def build_conditional_library( + lib: typing.Any, + conditional_names: dict[str, Callable[[], list[str]]], +) -> typing.Any: + conditional_lib = types.ModuleType("lib") + conditional_lib._original_lib = lib # type: ignore[attr-defined] + excluded_names = set() + for condition, names_cb in conditional_names.items(): + if not getattr(lib, condition): + excluded_names.update(names_cb()) + + for attr in dir(lib): + if attr not in excluded_names: + setattr(conditional_lib, attr, getattr(lib, attr)) + + return conditional_lib + + +class Binding: + """ + OpenSSL API wrapper. + """ + + lib: typing.ClassVar[typing.Any] = None + ffi = _openssl.ffi + _lib_loaded = False + _init_lock = threading.Lock() + + def __init__(self) -> None: + self._ensure_ffi_initialized() + + @classmethod + def _ensure_ffi_initialized(cls) -> None: + with cls._init_lock: + if not cls._lib_loaded: + cls.lib = build_conditional_library( + _openssl.lib, CONDITIONAL_NAMES + ) + cls._lib_loaded = True + + @classmethod + def init_static_locks(cls) -> None: + cls._ensure_ffi_initialized() + + +def _verify_package_version(version: str) -> None: + # Occasionally we run into situations where the version of the Python + # package does not match the version of the shared object that is loaded. + # This may occur in environments where multiple versions of cryptography + # are installed and available in the python path. To avoid errors cropping + # up later this code checks that the currently imported package and the + # shared object that were loaded have the same version and raise an + # ImportError if they do not + so_package_version = _openssl.ffi.string( + _openssl.lib.CRYPTOGRAPHY_PACKAGE_VERSION + ) + if version.encode("ascii") != so_package_version: + raise ImportError( + "The version of cryptography does not match the loaded " + "shared object. This can happen if you have multiple copies of " + "cryptography installed in your Python path. Please try creating " + "a new virtual environment to resolve this issue. " + f"Loaded python version: {version}, " + f"shared object version: {so_package_version}" + ) + + _openssl_assert( + _openssl.lib.OpenSSL_version_num() == openssl.openssl_version(), + ) + + +_verify_package_version(cryptography.__version__) + +Binding.init_static_locks() + +if ( + sys.platform == "win32" + and os.environ.get("PROCESSOR_ARCHITEW6432") is not None +): + warnings.warn( + "You are using cryptography on a 32-bit Python on a 64-bit Windows " + "Operating System. Cryptography will be significantly faster if you " + "switch to using a 64-bit Python.", + UserWarning, + stacklevel=2, + ) + +if ( + not openssl.CRYPTOGRAPHY_IS_LIBRESSL + and not openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not openssl.CRYPTOGRAPHY_IS_AWSLC + and not openssl.CRYPTOGRAPHY_OPENSSL_300_OR_GREATER +): + warnings.warn( + "You are using OpenSSL < 3.0. Support for OpenSSL < 3.0 is deprecated " + "and will be removed in the next release. Please upgrade to OpenSSL " + "3.0 or later.", + CryptographyDeprecationWarning, + stacklevel=2, + ) diff --git a/lib/cryptography/hazmat/decrepit/__init__.py b/lib/cryptography/hazmat/decrepit/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/lib/cryptography/hazmat/decrepit/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/lib/cryptography/hazmat/decrepit/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/decrepit/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..099774b Binary files /dev/null and b/lib/cryptography/hazmat/decrepit/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/decrepit/ciphers/__init__.py b/lib/cryptography/hazmat/decrepit/ciphers/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/lib/cryptography/hazmat/decrepit/ciphers/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/lib/cryptography/hazmat/decrepit/ciphers/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/decrepit/ciphers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..9e4a59c Binary files /dev/null and b/lib/cryptography/hazmat/decrepit/ciphers/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/decrepit/ciphers/__pycache__/algorithms.cpython-314.pyc b/lib/cryptography/hazmat/decrepit/ciphers/__pycache__/algorithms.cpython-314.pyc new file mode 100644 index 0000000..6ef1c08 Binary files /dev/null and b/lib/cryptography/hazmat/decrepit/ciphers/__pycache__/algorithms.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/decrepit/ciphers/algorithms.py b/lib/cryptography/hazmat/decrepit/ciphers/algorithms.py new file mode 100644 index 0000000..072a991 --- /dev/null +++ b/lib/cryptography/hazmat/decrepit/ciphers/algorithms.py @@ -0,0 +1,112 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, + _verify_key_size, +) + + +class ARC4(CipherAlgorithm): + name = "RC4" + key_sizes = frozenset([40, 56, 64, 80, 128, 160, 192, 256]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class TripleDES(BlockCipherAlgorithm): + name = "3DES" + block_size = 64 + key_sizes = frozenset([64, 128, 192]) + + def __init__(self, key: bytes): + if len(key) == 8: + key += key + key + elif len(key) == 16: + key += key[:8] + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +# Not actually supported, marker for tests +class _DES: + key_size = 64 + + +class Blowfish(BlockCipherAlgorithm): + name = "Blowfish" + block_size = 64 + key_sizes = frozenset(range(32, 449, 8)) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class CAST5(BlockCipherAlgorithm): + name = "CAST5" + block_size = 64 + key_sizes = frozenset(range(40, 129, 8)) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class SEED(BlockCipherAlgorithm): + name = "SEED" + block_size = 128 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class IDEA(BlockCipherAlgorithm): + name = "IDEA" + block_size = 64 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +# This class only allows RC2 with a 128-bit key. No support for +# effective key bits or other key sizes is provided. +class RC2(BlockCipherAlgorithm): + name = "RC2" + block_size = 64 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 diff --git a/lib/cryptography/hazmat/primitives/__init__.py b/lib/cryptography/hazmat/primitives/__init__.py new file mode 100644 index 0000000..b509336 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/__init__.py @@ -0,0 +1,3 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. diff --git a/lib/cryptography/hazmat/primitives/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..5879ecc Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/_asymmetric.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/_asymmetric.cpython-314.pyc new file mode 100644 index 0000000..03490f1 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/_asymmetric.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/_cipheralgorithm.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/_cipheralgorithm.cpython-314.pyc new file mode 100644 index 0000000..b4c1a52 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/_cipheralgorithm.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/_serialization.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/_serialization.cpython-314.pyc new file mode 100644 index 0000000..ecad360 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/_serialization.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/cmac.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/cmac.cpython-314.pyc new file mode 100644 index 0000000..31266f7 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/cmac.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/constant_time.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/constant_time.cpython-314.pyc new file mode 100644 index 0000000..0c59bb6 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/constant_time.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/hashes.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/hashes.cpython-314.pyc new file mode 100644 index 0000000..4cb0d9d Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/hashes.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/hmac.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/hmac.cpython-314.pyc new file mode 100644 index 0000000..99cc589 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/hmac.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/keywrap.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/keywrap.cpython-314.pyc new file mode 100644 index 0000000..0681836 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/keywrap.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/padding.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/padding.cpython-314.pyc new file mode 100644 index 0000000..7b541d2 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/padding.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/__pycache__/poly1305.cpython-314.pyc b/lib/cryptography/hazmat/primitives/__pycache__/poly1305.cpython-314.pyc new file mode 100644 index 0000000..6181724 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/__pycache__/poly1305.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/_asymmetric.py b/lib/cryptography/hazmat/primitives/_asymmetric.py new file mode 100644 index 0000000..ea55ffd --- /dev/null +++ b/lib/cryptography/hazmat/primitives/_asymmetric.py @@ -0,0 +1,19 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +# This exists to break an import cycle. It is normally accessible from the +# asymmetric padding module. + + +class AsymmetricPadding(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this padding (e.g. "PSS", "PKCS1"). + """ diff --git a/lib/cryptography/hazmat/primitives/_cipheralgorithm.py b/lib/cryptography/hazmat/primitives/_cipheralgorithm.py new file mode 100644 index 0000000..305a9fd --- /dev/null +++ b/lib/cryptography/hazmat/primitives/_cipheralgorithm.py @@ -0,0 +1,60 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils + +# This exists to break an import cycle. It is normally accessible from the +# ciphers module. + + +class CipherAlgorithm(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this mode (e.g. "AES", "Camellia"). + """ + + @property + @abc.abstractmethod + def key_sizes(self) -> frozenset[int]: + """ + Valid key sizes for this algorithm in bits + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The size of the key being used as an integer in bits (e.g. 128, 256). + """ + + +class BlockCipherAlgorithm(CipherAlgorithm): + key: utils.Buffer + + @property + @abc.abstractmethod + def block_size(self) -> int: + """ + The size of a block as an integer in bits (e.g. 64, 128). + """ + + +def _verify_key_size( + algorithm: CipherAlgorithm, key: utils.Buffer +) -> utils.Buffer: + # Verify that the key is instance of bytes + utils._check_byteslike("key", key) + + # Verify that the key size matches the expected key size + if len(key) * 8 not in algorithm.key_sizes: + raise ValueError( + f"Invalid key size ({len(key) * 8}) for {algorithm.name}." + ) + return key diff --git a/lib/cryptography/hazmat/primitives/_serialization.py b/lib/cryptography/hazmat/primitives/_serialization.py new file mode 100644 index 0000000..e998865 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/_serialization.py @@ -0,0 +1,168 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.hazmat.primitives.hashes import HashAlgorithm + +# This exists to break an import cycle. These classes are normally accessible +# from the serialization module. + + +class PBES(utils.Enum): + PBESv1SHA1And3KeyTripleDESCBC = "PBESv1 using SHA1 and 3-Key TripleDES" + PBESv2SHA256AndAES256CBC = "PBESv2 using SHA256 PBKDF2 and AES256 CBC" + + +class Encoding(utils.Enum): + PEM = "PEM" + DER = "DER" + OpenSSH = "OpenSSH" + Raw = "Raw" + X962 = "ANSI X9.62" + SMIME = "S/MIME" + + +class PrivateFormat(utils.Enum): + PKCS8 = "PKCS8" + TraditionalOpenSSL = "TraditionalOpenSSL" + Raw = "Raw" + OpenSSH = "OpenSSH" + PKCS12 = "PKCS12" + + def encryption_builder(self) -> KeySerializationEncryptionBuilder: + if self not in (PrivateFormat.OpenSSH, PrivateFormat.PKCS12): + raise ValueError( + "encryption_builder only supported with PrivateFormat.OpenSSH" + " and PrivateFormat.PKCS12" + ) + return KeySerializationEncryptionBuilder(self) + + +class PublicFormat(utils.Enum): + SubjectPublicKeyInfo = "X.509 subjectPublicKeyInfo with PKCS#1" + PKCS1 = "Raw PKCS#1" + OpenSSH = "OpenSSH" + Raw = "Raw" + CompressedPoint = "X9.62 Compressed Point" + UncompressedPoint = "X9.62 Uncompressed Point" + + +class ParameterFormat(utils.Enum): + PKCS3 = "PKCS3" + + +class KeySerializationEncryption(metaclass=abc.ABCMeta): + pass + + +class BestAvailableEncryption(KeySerializationEncryption): + def __init__(self, password: bytes): + if not isinstance(password, bytes) or len(password) == 0: + raise ValueError("Password must be 1 or more bytes.") + + self.password = password + + +class NoEncryption(KeySerializationEncryption): + pass + + +class KeySerializationEncryptionBuilder: + def __init__( + self, + format: PrivateFormat, + *, + _kdf_rounds: int | None = None, + _hmac_hash: HashAlgorithm | None = None, + _key_cert_algorithm: PBES | None = None, + ) -> None: + self._format = format + + self._kdf_rounds = _kdf_rounds + self._hmac_hash = _hmac_hash + self._key_cert_algorithm = _key_cert_algorithm + + def kdf_rounds(self, rounds: int) -> KeySerializationEncryptionBuilder: + if self._kdf_rounds is not None: + raise ValueError("kdf_rounds already set") + + if not isinstance(rounds, int): + raise TypeError("kdf_rounds must be an integer") + + if rounds < 1: + raise ValueError("kdf_rounds must be a positive integer") + + return KeySerializationEncryptionBuilder( + self._format, + _kdf_rounds=rounds, + _hmac_hash=self._hmac_hash, + _key_cert_algorithm=self._key_cert_algorithm, + ) + + def hmac_hash( + self, algorithm: HashAlgorithm + ) -> KeySerializationEncryptionBuilder: + if self._format is not PrivateFormat.PKCS12: + raise TypeError( + "hmac_hash only supported with PrivateFormat.PKCS12" + ) + + if self._hmac_hash is not None: + raise ValueError("hmac_hash already set") + return KeySerializationEncryptionBuilder( + self._format, + _kdf_rounds=self._kdf_rounds, + _hmac_hash=algorithm, + _key_cert_algorithm=self._key_cert_algorithm, + ) + + def key_cert_algorithm( + self, algorithm: PBES + ) -> KeySerializationEncryptionBuilder: + if self._format is not PrivateFormat.PKCS12: + raise TypeError( + "key_cert_algorithm only supported with PrivateFormat.PKCS12" + ) + if self._key_cert_algorithm is not None: + raise ValueError("key_cert_algorithm already set") + return KeySerializationEncryptionBuilder( + self._format, + _kdf_rounds=self._kdf_rounds, + _hmac_hash=self._hmac_hash, + _key_cert_algorithm=algorithm, + ) + + def build(self, password: bytes) -> KeySerializationEncryption: + if not isinstance(password, bytes) or len(password) == 0: + raise ValueError("Password must be 1 or more bytes.") + + return _KeySerializationEncryption( + self._format, + password, + kdf_rounds=self._kdf_rounds, + hmac_hash=self._hmac_hash, + key_cert_algorithm=self._key_cert_algorithm, + ) + + +class _KeySerializationEncryption(KeySerializationEncryption): + def __init__( + self, + format: PrivateFormat, + password: bytes, + *, + kdf_rounds: int | None, + hmac_hash: HashAlgorithm | None, + key_cert_algorithm: PBES | None, + ): + self._format = format + self.password = password + + self._kdf_rounds = kdf_rounds + self._hmac_hash = hmac_hash + self._key_cert_algorithm = key_cert_algorithm diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__init__.py b/lib/cryptography/hazmat/primitives/asymmetric/__init__.py new file mode 100644 index 0000000..b509336 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/__init__.py @@ -0,0 +1,3 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..143816a Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-314.pyc new file mode 100644 index 0000000..861ac03 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-314.pyc new file mode 100644 index 0000000..4db1d9c Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-314.pyc new file mode 100644 index 0000000..791f2c1 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-314.pyc new file mode 100644 index 0000000..c244235 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-314.pyc new file mode 100644 index 0000000..aaefafe Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-314.pyc new file mode 100644 index 0000000..569631b Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-314.pyc new file mode 100644 index 0000000..3d9a41b Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/types.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/types.cpython-314.pyc new file mode 100644 index 0000000..6dc2130 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/types.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..ed8b6f1 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-314.pyc new file mode 100644 index 0000000..29fe3e5 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-314.pyc b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-314.pyc new file mode 100644 index 0000000..32019e7 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/asymmetric/dh.py b/lib/cryptography/hazmat/primitives/asymmetric/dh.py new file mode 100644 index 0000000..1822e99 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/dh.py @@ -0,0 +1,147 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization + +generate_parameters = rust_openssl.dh.generate_parameters + + +DHPrivateNumbers = rust_openssl.dh.DHPrivateNumbers +DHPublicNumbers = rust_openssl.dh.DHPublicNumbers +DHParameterNumbers = rust_openssl.dh.DHParameterNumbers + + +class DHParameters(metaclass=abc.ABCMeta): + @abc.abstractmethod + def generate_private_key(self) -> DHPrivateKey: + """ + Generates and returns a DHPrivateKey. + """ + + @abc.abstractmethod + def parameter_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.ParameterFormat, + ) -> bytes: + """ + Returns the parameters serialized as bytes. + """ + + @abc.abstractmethod + def parameter_numbers(self) -> DHParameterNumbers: + """ + Returns a DHParameterNumbers. + """ + + +DHParametersWithSerialization = DHParameters +DHParameters.register(rust_openssl.dh.DHParameters) + + +class DHPublicKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def parameters(self) -> DHParameters: + """ + The DHParameters object associated with this public key. + """ + + @abc.abstractmethod + def public_numbers(self) -> DHPublicNumbers: + """ + Returns a DHPublicNumbers. + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> DHPublicKey: + """ + Returns a copy. + """ + + +DHPublicKeyWithSerialization = DHPublicKey +DHPublicKey.register(rust_openssl.dh.DHPublicKey) + + +class DHPrivateKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def public_key(self) -> DHPublicKey: + """ + The DHPublicKey associated with this private key. + """ + + @abc.abstractmethod + def parameters(self) -> DHParameters: + """ + The DHParameters object associated with this private key. + """ + + @abc.abstractmethod + def exchange(self, peer_public_key: DHPublicKey) -> bytes: + """ + Given peer's DHPublicKey, carry out the key exchange and + return shared key as bytes. + """ + + @abc.abstractmethod + def private_numbers(self) -> DHPrivateNumbers: + """ + Returns a DHPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> DHPrivateKey: + """ + Returns a copy. + """ + + +DHPrivateKeyWithSerialization = DHPrivateKey +DHPrivateKey.register(rust_openssl.dh.DHPrivateKey) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/dsa.py b/lib/cryptography/hazmat/primitives/asymmetric/dsa.py new file mode 100644 index 0000000..21d78ba --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/dsa.py @@ -0,0 +1,167 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import typing + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization, hashes +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils +from cryptography.utils import Buffer + + +class DSAParameters(metaclass=abc.ABCMeta): + @abc.abstractmethod + def generate_private_key(self) -> DSAPrivateKey: + """ + Generates and returns a DSAPrivateKey. + """ + + @abc.abstractmethod + def parameter_numbers(self) -> DSAParameterNumbers: + """ + Returns a DSAParameterNumbers. + """ + + +DSAParametersWithNumbers = DSAParameters +DSAParameters.register(rust_openssl.dsa.DSAParameters) + + +class DSAPrivateKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def public_key(self) -> DSAPublicKey: + """ + The DSAPublicKey associated with this private key. + """ + + @abc.abstractmethod + def parameters(self) -> DSAParameters: + """ + The DSAParameters object associated with this private key. + """ + + @abc.abstractmethod + def sign( + self, + data: Buffer, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> bytes: + """ + Signs the data + """ + + @abc.abstractmethod + def private_numbers(self) -> DSAPrivateNumbers: + """ + Returns a DSAPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> DSAPrivateKey: + """ + Returns a copy. + """ + + +DSAPrivateKeyWithSerialization = DSAPrivateKey +DSAPrivateKey.register(rust_openssl.dsa.DSAPrivateKey) + + +class DSAPublicKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def parameters(self) -> DSAParameters: + """ + The DSAParameters object associated with this public key. + """ + + @abc.abstractmethod + def public_numbers(self) -> DSAPublicNumbers: + """ + Returns a DSAPublicNumbers. + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify( + self, + signature: Buffer, + data: Buffer, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> None: + """ + Verifies the signature of the data. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> DSAPublicKey: + """ + Returns a copy. + """ + + +DSAPublicKeyWithSerialization = DSAPublicKey +DSAPublicKey.register(rust_openssl.dsa.DSAPublicKey) + +DSAPrivateNumbers = rust_openssl.dsa.DSAPrivateNumbers +DSAPublicNumbers = rust_openssl.dsa.DSAPublicNumbers +DSAParameterNumbers = rust_openssl.dsa.DSAParameterNumbers + + +def generate_parameters( + key_size: int, backend: typing.Any = None +) -> DSAParameters: + if key_size not in (1024, 2048, 3072, 4096): + raise ValueError("Key size must be 1024, 2048, 3072, or 4096 bits.") + + return rust_openssl.dsa.generate_parameters(key_size) + + +def generate_private_key( + key_size: int, backend: typing.Any = None +) -> DSAPrivateKey: + parameters = generate_parameters(key_size) + return parameters.generate_private_key() diff --git a/lib/cryptography/hazmat/primitives/asymmetric/ec.py b/lib/cryptography/hazmat/primitives/asymmetric/ec.py new file mode 100644 index 0000000..8638d20 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/ec.py @@ -0,0 +1,470 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import typing + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat._oid import ObjectIdentifier +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization, hashes +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils + + +class EllipticCurveOID: + SECP192R1 = ObjectIdentifier("1.2.840.10045.3.1.1") + SECP224R1 = ObjectIdentifier("1.3.132.0.33") + SECP256K1 = ObjectIdentifier("1.3.132.0.10") + SECP256R1 = ObjectIdentifier("1.2.840.10045.3.1.7") + SECP384R1 = ObjectIdentifier("1.3.132.0.34") + SECP521R1 = ObjectIdentifier("1.3.132.0.35") + BRAINPOOLP256R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.7") + BRAINPOOLP384R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.11") + BRAINPOOLP512R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.13") + SECT163K1 = ObjectIdentifier("1.3.132.0.1") + SECT163R2 = ObjectIdentifier("1.3.132.0.15") + SECT233K1 = ObjectIdentifier("1.3.132.0.26") + SECT233R1 = ObjectIdentifier("1.3.132.0.27") + SECT283K1 = ObjectIdentifier("1.3.132.0.16") + SECT283R1 = ObjectIdentifier("1.3.132.0.17") + SECT409K1 = ObjectIdentifier("1.3.132.0.36") + SECT409R1 = ObjectIdentifier("1.3.132.0.37") + SECT571K1 = ObjectIdentifier("1.3.132.0.38") + SECT571R1 = ObjectIdentifier("1.3.132.0.39") + + +class EllipticCurve(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + The name of the curve. e.g. secp256r1. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + Bit size of a secret scalar for the curve. + """ + + @property + @abc.abstractmethod + def group_order(self) -> int: + """ + The order of the curve's group. + """ + + +class EllipticCurveSignatureAlgorithm(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def algorithm( + self, + ) -> asym_utils.Prehashed | hashes.HashAlgorithm: + """ + The digest algorithm used with this signature. + """ + + +class EllipticCurvePrivateKey(metaclass=abc.ABCMeta): + @abc.abstractmethod + def exchange( + self, algorithm: ECDH, peer_public_key: EllipticCurvePublicKey + ) -> bytes: + """ + Performs a key exchange operation using the provided algorithm with the + provided peer's public key. + """ + + @abc.abstractmethod + def public_key(self) -> EllipticCurvePublicKey: + """ + The EllipticCurvePublicKey for this private key. + """ + + @property + @abc.abstractmethod + def curve(self) -> EllipticCurve: + """ + The EllipticCurve that this key is on. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + Bit size of a secret scalar for the curve. + """ + + @abc.abstractmethod + def sign( + self, + data: utils.Buffer, + signature_algorithm: EllipticCurveSignatureAlgorithm, + ) -> bytes: + """ + Signs the data + """ + + @abc.abstractmethod + def private_numbers(self) -> EllipticCurvePrivateNumbers: + """ + Returns an EllipticCurvePrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> EllipticCurvePrivateKey: + """ + Returns a copy. + """ + + +EllipticCurvePrivateKeyWithSerialization = EllipticCurvePrivateKey +EllipticCurvePrivateKey.register(rust_openssl.ec.ECPrivateKey) + + +class EllipticCurvePublicKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def curve(self) -> EllipticCurve: + """ + The EllipticCurve that this key is on. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + Bit size of a secret scalar for the curve. + """ + + @abc.abstractmethod + def public_numbers(self) -> EllipticCurvePublicNumbers: + """ + Returns an EllipticCurvePublicNumbers. + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify( + self, + signature: utils.Buffer, + data: utils.Buffer, + signature_algorithm: EllipticCurveSignatureAlgorithm, + ) -> None: + """ + Verifies the signature of the data. + """ + + @classmethod + def from_encoded_point( + cls, curve: EllipticCurve, data: bytes + ) -> EllipticCurvePublicKey: + utils._check_bytes("data", data) + + if len(data) == 0: + raise ValueError("data must not be an empty byte string") + + if data[0] not in [0x02, 0x03, 0x04]: + raise ValueError("Unsupported elliptic curve point type") + + return rust_openssl.ec.from_public_bytes(curve, data) + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> EllipticCurvePublicKey: + """ + Returns a copy. + """ + + +EllipticCurvePublicKeyWithSerialization = EllipticCurvePublicKey +EllipticCurvePublicKey.register(rust_openssl.ec.ECPublicKey) + +EllipticCurvePrivateNumbers = rust_openssl.ec.EllipticCurvePrivateNumbers +EllipticCurvePublicNumbers = rust_openssl.ec.EllipticCurvePublicNumbers + + +class SECT571R1(EllipticCurve): + name = "sect571r1" + key_size = 570 + group_order = 0x3FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE661CE18FF55987308059B186823851EC7DD9CA1161DE93D5174D66E8382E9BB2FE84E47 # noqa: E501 + + +class SECT409R1(EllipticCurve): + name = "sect409r1" + key_size = 409 + group_order = 0x10000000000000000000000000000000000000000000000000001E2AAD6A612F33307BE5FA47C3C9E052F838164CD37D9A21173 # noqa: E501 + + +class SECT283R1(EllipticCurve): + name = "sect283r1" + key_size = 283 + group_order = 0x3FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEF90399660FC938A90165B042A7CEFADB307 # noqa: E501 + + +class SECT233R1(EllipticCurve): + name = "sect233r1" + key_size = 233 + group_order = 0x1000000000000000000000000000013E974E72F8A6922031D2603CFE0D7 + + +class SECT163R2(EllipticCurve): + name = "sect163r2" + key_size = 163 + group_order = 0x40000000000000000000292FE77E70C12A4234C33 + + +class SECT571K1(EllipticCurve): + name = "sect571k1" + key_size = 571 + group_order = 0x20000000000000000000000000000000000000000000000000000000000000000000000131850E1F19A63E4B391A8DB917F4138B630D84BE5D639381E91DEB45CFE778F637C1001 # noqa: E501 + + +class SECT409K1(EllipticCurve): + name = "sect409k1" + key_size = 409 + group_order = 0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE5F83B2D4EA20400EC4557D5ED3E3E7CA5B4B5C83B8E01E5FCF # noqa: E501 + + +class SECT283K1(EllipticCurve): + name = "sect283k1" + key_size = 283 + group_order = 0x1FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE9AE2ED07577265DFF7F94451E061E163C61 # noqa: E501 + + +class SECT233K1(EllipticCurve): + name = "sect233k1" + key_size = 233 + group_order = 0x8000000000000000000000000000069D5BB915BCD46EFB1AD5F173ABDF + + +class SECT163K1(EllipticCurve): + name = "sect163k1" + key_size = 163 + group_order = 0x4000000000000000000020108A2E0CC0D99F8A5EF + + +class SECP521R1(EllipticCurve): + name = "secp521r1" + key_size = 521 + group_order = 0x1FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA51868783BF2F966B7FCC0148F709A5D03BB5C9B8899C47AEBB6FB71E91386409 # noqa: E501 + + +class SECP384R1(EllipticCurve): + name = "secp384r1" + key_size = 384 + group_order = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC7634D81F4372DDF581A0DB248B0A77AECEC196ACCC52973 # noqa: E501 + + +class SECP256R1(EllipticCurve): + name = "secp256r1" + key_size = 256 + group_order = ( + 0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551 + ) + + +class SECP256K1(EllipticCurve): + name = "secp256k1" + key_size = 256 + group_order = ( + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 + ) + + +class SECP224R1(EllipticCurve): + name = "secp224r1" + key_size = 224 + group_order = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFF16A2E0B8F03E13DD29455C5C2A3D + + +class SECP192R1(EllipticCurve): + name = "secp192r1" + key_size = 192 + group_order = 0xFFFFFFFFFFFFFFFFFFFFFFFF99DEF836146BC9B1B4D22831 + + +class BrainpoolP256R1(EllipticCurve): + name = "brainpoolP256r1" + key_size = 256 + group_order = ( + 0xA9FB57DBA1EEA9BC3E660A909D838D718C397AA3B561A6F7901E0E82974856A7 + ) + + +class BrainpoolP384R1(EllipticCurve): + name = "brainpoolP384r1" + key_size = 384 + group_order = 0x8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B31F166E6CAC0425A7CF3AB6AF6B7FC3103B883202E9046565 # noqa: E501 + + +class BrainpoolP512R1(EllipticCurve): + name = "brainpoolP512r1" + key_size = 512 + group_order = 0xAADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA70330870553E5C414CA92619418661197FAC10471DB1D381085DDADDB58796829CA90069 # noqa: E501 + + +_CURVE_TYPES: dict[str, EllipticCurve] = { + "prime192v1": SECP192R1(), + "prime256v1": SECP256R1(), + "secp192r1": SECP192R1(), + "secp224r1": SECP224R1(), + "secp256r1": SECP256R1(), + "secp384r1": SECP384R1(), + "secp521r1": SECP521R1(), + "secp256k1": SECP256K1(), + "sect163k1": SECT163K1(), + "sect233k1": SECT233K1(), + "sect283k1": SECT283K1(), + "sect409k1": SECT409K1(), + "sect571k1": SECT571K1(), + "sect163r2": SECT163R2(), + "sect233r1": SECT233R1(), + "sect283r1": SECT283R1(), + "sect409r1": SECT409R1(), + "sect571r1": SECT571R1(), + "brainpoolP256r1": BrainpoolP256R1(), + "brainpoolP384r1": BrainpoolP384R1(), + "brainpoolP512r1": BrainpoolP512R1(), +} + + +class ECDSA(EllipticCurveSignatureAlgorithm): + def __init__( + self, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + deterministic_signing: bool = False, + ): + from cryptography.hazmat.backends.openssl.backend import backend + + if ( + deterministic_signing + and not backend.ecdsa_deterministic_supported() + ): + raise UnsupportedAlgorithm( + "ECDSA with deterministic signature (RFC 6979) is not " + "supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + self._algorithm = algorithm + self._deterministic_signing = deterministic_signing + + @property + def algorithm( + self, + ) -> asym_utils.Prehashed | hashes.HashAlgorithm: + return self._algorithm + + @property + def deterministic_signing( + self, + ) -> bool: + return self._deterministic_signing + + +generate_private_key = rust_openssl.ec.generate_private_key + + +def derive_private_key( + private_value: int, + curve: EllipticCurve, + backend: typing.Any = None, +) -> EllipticCurvePrivateKey: + if not isinstance(private_value, int): + raise TypeError("private_value must be an integer type.") + + if private_value <= 0: + raise ValueError("private_value must be a positive integer.") + + return rust_openssl.ec.derive_private_key(private_value, curve) + + +class ECDH: + pass + + +_OID_TO_CURVE = { + EllipticCurveOID.SECP192R1: SECP192R1, + EllipticCurveOID.SECP224R1: SECP224R1, + EllipticCurveOID.SECP256K1: SECP256K1, + EllipticCurveOID.SECP256R1: SECP256R1, + EllipticCurveOID.SECP384R1: SECP384R1, + EllipticCurveOID.SECP521R1: SECP521R1, + EllipticCurveOID.BRAINPOOLP256R1: BrainpoolP256R1, + EllipticCurveOID.BRAINPOOLP384R1: BrainpoolP384R1, + EllipticCurveOID.BRAINPOOLP512R1: BrainpoolP512R1, + EllipticCurveOID.SECT163K1: SECT163K1, + EllipticCurveOID.SECT163R2: SECT163R2, + EllipticCurveOID.SECT233K1: SECT233K1, + EllipticCurveOID.SECT233R1: SECT233R1, + EllipticCurveOID.SECT283K1: SECT283K1, + EllipticCurveOID.SECT283R1: SECT283R1, + EllipticCurveOID.SECT409K1: SECT409K1, + EllipticCurveOID.SECT409R1: SECT409R1, + EllipticCurveOID.SECT571K1: SECT571K1, + EllipticCurveOID.SECT571R1: SECT571R1, +} + + +def get_curve_for_oid(oid: ObjectIdentifier) -> type[EllipticCurve]: + try: + return _OID_TO_CURVE[oid] + except KeyError: + raise LookupError( + "The provided object identifier has no matching elliptic " + "curve class" + ) + + +_SECT_CURVES: tuple[type[EllipticCurve], ...] = ( + SECT163K1, + SECT163R2, + SECT233K1, + SECT233R1, + SECT283K1, + SECT283R1, + SECT409K1, + SECT409R1, + SECT571K1, + SECT571R1, +) + +for _curve_cls in _SECT_CURVES: + utils.deprecated( + _curve_cls, + __name__, + f"{_curve_cls.__name__} will be removed in the next release.", + utils.DeprecatedIn46, + name=_curve_cls.__name__, + ) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/ed25519.py b/lib/cryptography/hazmat/primitives/asymmetric/ed25519.py new file mode 100644 index 0000000..e576dc9 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/ed25519.py @@ -0,0 +1,129 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class Ed25519PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> Ed25519PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed25519.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def verify(self, signature: Buffer, data: Buffer) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> Ed25519PublicKey: + """ + Returns a copy. + """ + + +Ed25519PublicKey.register(rust_openssl.ed25519.Ed25519PublicKey) + + +class Ed25519PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> Ed25519PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed25519.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> Ed25519PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed25519.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> Ed25519PublicKey: + """ + The Ed25519PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def sign(self, data: Buffer) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def __copy__(self) -> Ed25519PrivateKey: + """ + Returns a copy. + """ + + +Ed25519PrivateKey.register(rust_openssl.ed25519.Ed25519PrivateKey) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/ed448.py b/lib/cryptography/hazmat/primitives/asymmetric/ed448.py new file mode 100644 index 0000000..89db209 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/ed448.py @@ -0,0 +1,131 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class Ed448PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> Ed448PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed448.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def verify(self, signature: Buffer, data: Buffer) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> Ed448PublicKey: + """ + Returns a copy. + """ + + +if hasattr(rust_openssl, "ed448"): + Ed448PublicKey.register(rust_openssl.ed448.Ed448PublicKey) + + +class Ed448PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> Ed448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed448.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> Ed448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed448.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> Ed448PublicKey: + """ + The Ed448PublicKey derived from the private key. + """ + + @abc.abstractmethod + def sign(self, data: Buffer) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def __copy__(self) -> Ed448PrivateKey: + """ + Returns a copy. + """ + + +if hasattr(rust_openssl, "x448"): + Ed448PrivateKey.register(rust_openssl.ed448.Ed448PrivateKey) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/padding.py b/lib/cryptography/hazmat/primitives/asymmetric/padding.py new file mode 100644 index 0000000..5121a28 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/padding.py @@ -0,0 +1,111 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives._asymmetric import ( + AsymmetricPadding as AsymmetricPadding, +) +from cryptography.hazmat.primitives.asymmetric import rsa + + +class PKCS1v15(AsymmetricPadding): + name = "EMSA-PKCS1-v1_5" + + +class _MaxLength: + "Sentinel value for `MAX_LENGTH`." + + +class _Auto: + "Sentinel value for `AUTO`." + + +class _DigestLength: + "Sentinel value for `DIGEST_LENGTH`." + + +class PSS(AsymmetricPadding): + MAX_LENGTH = _MaxLength() + AUTO = _Auto() + DIGEST_LENGTH = _DigestLength() + name = "EMSA-PSS" + _salt_length: int | _MaxLength | _Auto | _DigestLength + + def __init__( + self, + mgf: MGF, + salt_length: int | _MaxLength | _Auto | _DigestLength, + ) -> None: + self._mgf = mgf + + if not isinstance( + salt_length, (int, _MaxLength, _Auto, _DigestLength) + ): + raise TypeError( + "salt_length must be an integer, MAX_LENGTH, " + "DIGEST_LENGTH, or AUTO" + ) + + if isinstance(salt_length, int) and salt_length < 0: + raise ValueError("salt_length must be zero or greater.") + + self._salt_length = salt_length + + @property + def mgf(self) -> MGF: + return self._mgf + + +class OAEP(AsymmetricPadding): + name = "EME-OAEP" + + def __init__( + self, + mgf: MGF, + algorithm: hashes.HashAlgorithm, + label: bytes | None, + ): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + self._mgf = mgf + self._algorithm = algorithm + self._label = label + + @property + def algorithm(self) -> hashes.HashAlgorithm: + return self._algorithm + + @property + def mgf(self) -> MGF: + return self._mgf + + +class MGF(metaclass=abc.ABCMeta): + _algorithm: hashes.HashAlgorithm + + +class MGF1(MGF): + def __init__(self, algorithm: hashes.HashAlgorithm): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + self._algorithm = algorithm + + +def calculate_max_pss_salt_length( + key: rsa.RSAPrivateKey | rsa.RSAPublicKey, + hash_algorithm: hashes.HashAlgorithm, +) -> int: + if not isinstance(key, (rsa.RSAPrivateKey, rsa.RSAPublicKey)): + raise TypeError("key must be an RSA public or private key") + # bit length - 1 per RFC 3447 + emlen = (key.key_size + 6) // 8 + salt_length = emlen - hash_algorithm.digest_size - 2 + assert salt_length >= 0 + return salt_length diff --git a/lib/cryptography/hazmat/primitives/asymmetric/rsa.py b/lib/cryptography/hazmat/primitives/asymmetric/rsa.py new file mode 100644 index 0000000..f94812e --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/rsa.py @@ -0,0 +1,285 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import random +import typing +from math import gcd + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization, hashes +from cryptography.hazmat.primitives._asymmetric import AsymmetricPadding +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils + + +class RSAPrivateKey(metaclass=abc.ABCMeta): + @abc.abstractmethod + def decrypt(self, ciphertext: bytes, padding: AsymmetricPadding) -> bytes: + """ + Decrypts the provided ciphertext. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the public modulus. + """ + + @abc.abstractmethod + def public_key(self) -> RSAPublicKey: + """ + The RSAPublicKey associated with this private key. + """ + + @abc.abstractmethod + def sign( + self, + data: bytes, + padding: AsymmetricPadding, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def private_numbers(self) -> RSAPrivateNumbers: + """ + Returns an RSAPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> RSAPrivateKey: + """ + Returns a copy. + """ + + +RSAPrivateKeyWithSerialization = RSAPrivateKey +RSAPrivateKey.register(rust_openssl.rsa.RSAPrivateKey) + + +class RSAPublicKey(metaclass=abc.ABCMeta): + @abc.abstractmethod + def encrypt(self, plaintext: bytes, padding: AsymmetricPadding) -> bytes: + """ + Encrypts the given plaintext. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the public modulus. + """ + + @abc.abstractmethod + def public_numbers(self) -> RSAPublicNumbers: + """ + Returns an RSAPublicNumbers + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify( + self, + signature: bytes, + data: bytes, + padding: AsymmetricPadding, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> None: + """ + Verifies the signature of the data. + """ + + @abc.abstractmethod + def recover_data_from_signature( + self, + signature: bytes, + padding: AsymmetricPadding, + algorithm: hashes.HashAlgorithm | None, + ) -> bytes: + """ + Recovers the original data from the signature. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> RSAPublicKey: + """ + Returns a copy. + """ + + +RSAPublicKeyWithSerialization = RSAPublicKey +RSAPublicKey.register(rust_openssl.rsa.RSAPublicKey) + +RSAPrivateNumbers = rust_openssl.rsa.RSAPrivateNumbers +RSAPublicNumbers = rust_openssl.rsa.RSAPublicNumbers + + +def generate_private_key( + public_exponent: int, + key_size: int, + backend: typing.Any = None, +) -> RSAPrivateKey: + _verify_rsa_parameters(public_exponent, key_size) + return rust_openssl.rsa.generate_private_key(public_exponent, key_size) + + +def _verify_rsa_parameters(public_exponent: int, key_size: int) -> None: + if public_exponent not in (3, 65537): + raise ValueError( + "public_exponent must be either 3 (for legacy compatibility) or " + "65537. Almost everyone should choose 65537 here!" + ) + + if key_size < 1024: + raise ValueError("key_size must be at least 1024-bits.") + + +def _modinv(e: int, m: int) -> int: + """ + Modular Multiplicative Inverse. Returns x such that: (x*e) mod m == 1 + """ + x1, x2 = 1, 0 + a, b = e, m + while b > 0: + q, r = divmod(a, b) + xn = x1 - q * x2 + a, b, x1, x2 = b, r, x2, xn + return x1 % m + + +def rsa_crt_iqmp(p: int, q: int) -> int: + """ + Compute the CRT (q ** -1) % p value from RSA primes p and q. + """ + if p <= 1 or q <= 1: + raise ValueError("Values can't be <= 1") + return _modinv(q, p) + + +def rsa_crt_dmp1(private_exponent: int, p: int) -> int: + """ + Compute the CRT private_exponent % (p - 1) value from the RSA + private_exponent (d) and p. + """ + if private_exponent <= 1 or p <= 1: + raise ValueError("Values can't be <= 1") + return private_exponent % (p - 1) + + +def rsa_crt_dmq1(private_exponent: int, q: int) -> int: + """ + Compute the CRT private_exponent % (q - 1) value from the RSA + private_exponent (d) and q. + """ + if private_exponent <= 1 or q <= 1: + raise ValueError("Values can't be <= 1") + return private_exponent % (q - 1) + + +def rsa_recover_private_exponent(e: int, p: int, q: int) -> int: + """ + Compute the RSA private_exponent (d) given the public exponent (e) + and the RSA primes p and q. + + This uses the Carmichael totient function to generate the + smallest possible working value of the private exponent. + """ + # This lambda_n is the Carmichael totient function. + # The original RSA paper uses the Euler totient function + # here: phi_n = (p - 1) * (q - 1) + # Either version of the private exponent will work, but the + # one generated by the older formulation may be larger + # than necessary. (lambda_n always divides phi_n) + # + # TODO: Replace with lcm(p - 1, q - 1) once the minimum + # supported Python version is >= 3.9. + if e <= 1 or p <= 1 or q <= 1: + raise ValueError("Values can't be <= 1") + lambda_n = (p - 1) * (q - 1) // gcd(p - 1, q - 1) + return _modinv(e, lambda_n) + + +# Controls the number of iterations rsa_recover_prime_factors will perform +# to obtain the prime factors. +_MAX_RECOVERY_ATTEMPTS = 500 + + +def rsa_recover_prime_factors(n: int, e: int, d: int) -> tuple[int, int]: + """ + Compute factors p and q from the private exponent d. We assume that n has + no more than two factors. This function is adapted from code in PyCrypto. + """ + # reject invalid values early + if d <= 1 or e <= 1: + raise ValueError("d, e can't be <= 1") + if 17 != pow(17, e * d, n): + raise ValueError("n, d, e don't match") + # See 8.2.2(i) in Handbook of Applied Cryptography. + ktot = d * e - 1 + # The quantity d*e-1 is a multiple of phi(n), even, + # and can be represented as t*2^s. + t = ktot + while t % 2 == 0: + t = t // 2 + # Cycle through all multiplicative inverses in Zn. + # The algorithm is non-deterministic, but there is a 50% chance + # any candidate a leads to successful factoring. + # See "Digitalized Signatures and Public Key Functions as Intractable + # as Factorization", M. Rabin, 1979 + spotted = False + tries = 0 + while not spotted and tries < _MAX_RECOVERY_ATTEMPTS: + a = random.randint(2, n - 1) + tries += 1 + k = t + # Cycle through all values a^{t*2^i}=a^k + while k < ktot: + cand = pow(a, k, n) + # Check if a^k is a non-trivial root of unity (mod n) + if cand != 1 and cand != (n - 1) and pow(cand, 2, n) == 1: + # We have found a number such that (cand-1)(cand+1)=0 (mod n). + # Either of the terms divides n. + p = gcd(cand + 1, n) + spotted = True + break + k *= 2 + if not spotted: + raise ValueError("Unable to compute factors p and q from exponent d.") + # Found ! + q, r = divmod(n, p) + assert r == 0 + p, q = sorted((p, q), reverse=True) + return (p, q) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/types.py b/lib/cryptography/hazmat/primitives/asymmetric/types.py new file mode 100644 index 0000000..1fe4eaf --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/types.py @@ -0,0 +1,111 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography import utils +from cryptography.hazmat.primitives.asymmetric import ( + dh, + dsa, + ec, + ed448, + ed25519, + rsa, + x448, + x25519, +) + +# Every asymmetric key type +PublicKeyTypes = typing.Union[ + dh.DHPublicKey, + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + x25519.X25519PublicKey, + x448.X448PublicKey, +] +PUBLIC_KEY_TYPES = PublicKeyTypes +utils.deprecated( + PUBLIC_KEY_TYPES, + __name__, + "Use PublicKeyTypes instead", + utils.DeprecatedIn40, + name="PUBLIC_KEY_TYPES", +) +# Every asymmetric key type +PrivateKeyTypes = typing.Union[ + dh.DHPrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + x25519.X25519PrivateKey, + x448.X448PrivateKey, +] +PRIVATE_KEY_TYPES = PrivateKeyTypes +utils.deprecated( + PRIVATE_KEY_TYPES, + __name__, + "Use PrivateKeyTypes instead", + utils.DeprecatedIn40, + name="PRIVATE_KEY_TYPES", +) +# Just the key types we allow to be used for x509 signing. This mirrors +# the certificate public key types +CertificateIssuerPrivateKeyTypes = typing.Union[ + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, +] +CERTIFICATE_PRIVATE_KEY_TYPES = CertificateIssuerPrivateKeyTypes +utils.deprecated( + CERTIFICATE_PRIVATE_KEY_TYPES, + __name__, + "Use CertificateIssuerPrivateKeyTypes instead", + utils.DeprecatedIn40, + name="CERTIFICATE_PRIVATE_KEY_TYPES", +) +# Just the key types we allow to be used for x509 signing. This mirrors +# the certificate private key types +CertificateIssuerPublicKeyTypes = typing.Union[ + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, +] +CERTIFICATE_ISSUER_PUBLIC_KEY_TYPES = CertificateIssuerPublicKeyTypes +utils.deprecated( + CERTIFICATE_ISSUER_PUBLIC_KEY_TYPES, + __name__, + "Use CertificateIssuerPublicKeyTypes instead", + utils.DeprecatedIn40, + name="CERTIFICATE_ISSUER_PUBLIC_KEY_TYPES", +) +# This type removes DHPublicKey. x448/x25519 can be a public key +# but cannot be used in signing so they are allowed here. +CertificatePublicKeyTypes = typing.Union[ + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + x25519.X25519PublicKey, + x448.X448PublicKey, +] +CERTIFICATE_PUBLIC_KEY_TYPES = CertificatePublicKeyTypes +utils.deprecated( + CERTIFICATE_PUBLIC_KEY_TYPES, + __name__, + "Use CertificatePublicKeyTypes instead", + utils.DeprecatedIn40, + name="CERTIFICATE_PUBLIC_KEY_TYPES", +) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/utils.py b/lib/cryptography/hazmat/primitives/asymmetric/utils.py new file mode 100644 index 0000000..826b956 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/utils.py @@ -0,0 +1,24 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import asn1 +from cryptography.hazmat.primitives import hashes + +decode_dss_signature = asn1.decode_dss_signature +encode_dss_signature = asn1.encode_dss_signature + + +class Prehashed: + def __init__(self, algorithm: hashes.HashAlgorithm): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of HashAlgorithm.") + + self._algorithm = algorithm + self._digest_size = algorithm.digest_size + + @property + def digest_size(self) -> int: + return self._digest_size diff --git a/lib/cryptography/hazmat/primitives/asymmetric/x25519.py b/lib/cryptography/hazmat/primitives/asymmetric/x25519.py new file mode 100644 index 0000000..a499376 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/x25519.py @@ -0,0 +1,122 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class X25519PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> X25519PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x25519.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> X25519PublicKey: + """ + Returns a copy. + """ + + +X25519PublicKey.register(rust_openssl.x25519.X25519PublicKey) + + +class X25519PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> X25519PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + return rust_openssl.x25519.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> X25519PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x25519.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> X25519PublicKey: + """ + Returns the public key associated with this private key + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def exchange(self, peer_public_key: X25519PublicKey) -> bytes: + """ + Performs a key exchange operation using the provided peer's public key. + """ + + @abc.abstractmethod + def __copy__(self) -> X25519PrivateKey: + """ + Returns a copy. + """ + + +X25519PrivateKey.register(rust_openssl.x25519.X25519PrivateKey) diff --git a/lib/cryptography/hazmat/primitives/asymmetric/x448.py b/lib/cryptography/hazmat/primitives/asymmetric/x448.py new file mode 100644 index 0000000..c6fd71b --- /dev/null +++ b/lib/cryptography/hazmat/primitives/asymmetric/x448.py @@ -0,0 +1,125 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class X448PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> X448PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x448.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> X448PublicKey: + """ + Returns a copy. + """ + + +if hasattr(rust_openssl, "x448"): + X448PublicKey.register(rust_openssl.x448.X448PublicKey) + + +class X448PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> X448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x448.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> X448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x448.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> X448PublicKey: + """ + Returns the public key associated with this private key + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def exchange(self, peer_public_key: X448PublicKey) -> bytes: + """ + Performs a key exchange operation using the provided peer's public key. + """ + + @abc.abstractmethod + def __copy__(self) -> X448PrivateKey: + """ + Returns a copy. + """ + + +if hasattr(rust_openssl, "x448"): + X448PrivateKey.register(rust_openssl.x448.X448PrivateKey) diff --git a/lib/cryptography/hazmat/primitives/ciphers/__init__.py b/lib/cryptography/hazmat/primitives/ciphers/__init__.py new file mode 100644 index 0000000..10c15d0 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/ciphers/__init__.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) +from cryptography.hazmat.primitives.ciphers.base import ( + AEADCipherContext, + AEADDecryptionContext, + AEADEncryptionContext, + Cipher, + CipherContext, +) + +__all__ = [ + "AEADCipherContext", + "AEADDecryptionContext", + "AEADEncryptionContext", + "BlockCipherAlgorithm", + "Cipher", + "CipherAlgorithm", + "CipherContext", +] diff --git a/lib/cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..6a31783 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-314.pyc b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-314.pyc new file mode 100644 index 0000000..45b5f5a Binary files /dev/null and b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-314.pyc b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-314.pyc new file mode 100644 index 0000000..6c1d862 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-314.pyc b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..2c6b08a Binary files /dev/null and b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-314.pyc b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-314.pyc new file mode 100644 index 0000000..4b59041 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/ciphers/aead.py b/lib/cryptography/hazmat/primitives/ciphers/aead.py new file mode 100644 index 0000000..c8a582d --- /dev/null +++ b/lib/cryptography/hazmat/primitives/ciphers/aead.py @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = [ + "AESCCM", + "AESGCM", + "AESGCMSIV", + "AESOCB3", + "AESSIV", + "ChaCha20Poly1305", +] + +AESGCM = rust_openssl.aead.AESGCM +ChaCha20Poly1305 = rust_openssl.aead.ChaCha20Poly1305 +AESCCM = rust_openssl.aead.AESCCM +AESSIV = rust_openssl.aead.AESSIV +AESOCB3 = rust_openssl.aead.AESOCB3 +AESGCMSIV = rust_openssl.aead.AESGCMSIV diff --git a/lib/cryptography/hazmat/primitives/ciphers/algorithms.py b/lib/cryptography/hazmat/primitives/ciphers/algorithms.py new file mode 100644 index 0000000..1e402c7 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/ciphers/algorithms.py @@ -0,0 +1,136 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + ARC4 as ARC4, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + CAST5 as CAST5, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + IDEA as IDEA, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + SEED as SEED, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + Blowfish as Blowfish, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + TripleDES as TripleDES, +) +from cryptography.hazmat.primitives._cipheralgorithm import _verify_key_size +from cryptography.hazmat.primitives.ciphers import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) + + +class AES(BlockCipherAlgorithm): + name = "AES" + block_size = 128 + # 512 added to support AES-256-XTS, which uses 512-bit keys + key_sizes = frozenset([128, 192, 256, 512]) + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class AES128(BlockCipherAlgorithm): + name = "AES" + block_size = 128 + key_sizes = frozenset([128]) + key_size = 128 + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + +class AES256(BlockCipherAlgorithm): + name = "AES" + block_size = 128 + key_sizes = frozenset([256]) + key_size = 256 + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + +class Camellia(BlockCipherAlgorithm): + name = "camellia" + block_size = 128 + key_sizes = frozenset([128, 192, 256]) + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +utils.deprecated( + ARC4, + __name__, + "ARC4 has been moved to " + "cryptography.hazmat.decrepit.ciphers.algorithms.ARC4 and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.algorithms in 48.0.0.", + utils.DeprecatedIn43, + name="ARC4", +) + + +utils.deprecated( + TripleDES, + __name__, + "TripleDES has been moved to " + "cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.algorithms in 48.0.0.", + utils.DeprecatedIn43, + name="TripleDES", +) + + +class ChaCha20(CipherAlgorithm): + name = "ChaCha20" + key_sizes = frozenset([256]) + + def __init__(self, key: utils.Buffer, nonce: utils.Buffer): + self.key = _verify_key_size(self, key) + utils._check_byteslike("nonce", nonce) + + if len(nonce) != 16: + raise ValueError("nonce must be 128-bits (16 bytes)") + + self._nonce = nonce + + @property + def nonce(self) -> utils.Buffer: + return self._nonce + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class SM4(BlockCipherAlgorithm): + name = "SM4" + block_size = 128 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 diff --git a/lib/cryptography/hazmat/primitives/ciphers/base.py b/lib/cryptography/hazmat/primitives/ciphers/base.py new file mode 100644 index 0000000..24fceea --- /dev/null +++ b/lib/cryptography/hazmat/primitives/ciphers/base.py @@ -0,0 +1,146 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import typing + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives._cipheralgorithm import CipherAlgorithm +from cryptography.hazmat.primitives.ciphers import modes +from cryptography.utils import Buffer + + +class CipherContext(metaclass=abc.ABCMeta): + @abc.abstractmethod + def update(self, data: Buffer) -> bytes: + """ + Processes the provided bytes through the cipher and returns the results + as bytes. + """ + + @abc.abstractmethod + def update_into(self, data: Buffer, buf: Buffer) -> int: + """ + Processes the provided bytes and writes the resulting data into the + provided buffer. Returns the number of bytes written. + """ + + @abc.abstractmethod + def finalize(self) -> bytes: + """ + Returns the results of processing the final block as bytes. + """ + + @abc.abstractmethod + def reset_nonce(self, nonce: bytes) -> None: + """ + Resets the nonce for the cipher context to the provided value. + Raises an exception if it does not support reset or if the + provided nonce does not have a valid length. + """ + + +class AEADCipherContext(CipherContext, metaclass=abc.ABCMeta): + @abc.abstractmethod + def authenticate_additional_data(self, data: Buffer) -> None: + """ + Authenticates the provided bytes. + """ + + +class AEADDecryptionContext(AEADCipherContext, metaclass=abc.ABCMeta): + @abc.abstractmethod + def finalize_with_tag(self, tag: bytes) -> bytes: + """ + Returns the results of processing the final block as bytes and allows + delayed passing of the authentication tag. + """ + + +class AEADEncryptionContext(AEADCipherContext, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def tag(self) -> bytes: + """ + Returns tag bytes. This is only available after encryption is + finalized. + """ + + +Mode = typing.TypeVar( + "Mode", bound=typing.Optional[modes.Mode], covariant=True +) + + +class Cipher(typing.Generic[Mode]): + def __init__( + self, + algorithm: CipherAlgorithm, + mode: Mode, + backend: typing.Any = None, + ) -> None: + if not isinstance(algorithm, CipherAlgorithm): + raise TypeError("Expected interface of CipherAlgorithm.") + + if mode is not None: + # mypy needs this assert to narrow the type from our generic + # type. Maybe it won't some time in the future. + assert isinstance(mode, modes.Mode) + mode.validate_for_algorithm(algorithm) + + self.algorithm = algorithm + self.mode = mode + + @typing.overload + def encryptor( + self: Cipher[modes.ModeWithAuthenticationTag], + ) -> AEADEncryptionContext: ... + + @typing.overload + def encryptor( + self: _CIPHER_TYPE, + ) -> CipherContext: ... + + def encryptor(self): + if isinstance(self.mode, modes.ModeWithAuthenticationTag): + if self.mode.tag is not None: + raise ValueError( + "Authentication tag must be None when encrypting." + ) + + return rust_openssl.ciphers.create_encryption_ctx( + self.algorithm, self.mode + ) + + @typing.overload + def decryptor( + self: Cipher[modes.ModeWithAuthenticationTag], + ) -> AEADDecryptionContext: ... + + @typing.overload + def decryptor( + self: _CIPHER_TYPE, + ) -> CipherContext: ... + + def decryptor(self): + return rust_openssl.ciphers.create_decryption_ctx( + self.algorithm, self.mode + ) + + +_CIPHER_TYPE = Cipher[ + typing.Union[ + modes.ModeWithNonce, + modes.ModeWithTweak, + modes.ECB, + modes.ModeWithInitializationVector, + None, + ] +] + +CipherContext.register(rust_openssl.ciphers.CipherContext) +AEADEncryptionContext.register(rust_openssl.ciphers.AEADEncryptionContext) +AEADDecryptionContext.register(rust_openssl.ciphers.AEADDecryptionContext) diff --git a/lib/cryptography/hazmat/primitives/ciphers/modes.py b/lib/cryptography/hazmat/primitives/ciphers/modes.py new file mode 100644 index 0000000..36c555c --- /dev/null +++ b/lib/cryptography/hazmat/primitives/ciphers/modes.py @@ -0,0 +1,268 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) +from cryptography.hazmat.primitives.ciphers import algorithms + + +class Mode(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this mode (e.g. "ECB", "CBC"). + """ + + @abc.abstractmethod + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + """ + Checks that all the necessary invariants of this (mode, algorithm) + combination are met. + """ + + +class ModeWithInitializationVector(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def initialization_vector(self) -> utils.Buffer: + """ + The value of the initialization vector for this mode as bytes. + """ + + +class ModeWithTweak(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def tweak(self) -> utils.Buffer: + """ + The value of the tweak for this mode as bytes. + """ + + +class ModeWithNonce(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def nonce(self) -> utils.Buffer: + """ + The value of the nonce for this mode as bytes. + """ + + +class ModeWithAuthenticationTag(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def tag(self) -> bytes | None: + """ + The value of the tag supplied to the constructor of this mode. + """ + + +def _check_aes_key_length(self: Mode, algorithm: CipherAlgorithm) -> None: + if algorithm.key_size > 256 and algorithm.name == "AES": + raise ValueError( + "Only 128, 192, and 256 bit keys are allowed for this AES mode" + ) + + +def _check_iv_length( + self: ModeWithInitializationVector, algorithm: BlockCipherAlgorithm +) -> None: + iv_len = len(self.initialization_vector) + if iv_len * 8 != algorithm.block_size: + raise ValueError(f"Invalid IV size ({iv_len}) for {self.name}.") + + +def _check_nonce_length( + nonce: utils.Buffer, name: str, algorithm: CipherAlgorithm +) -> None: + if not isinstance(algorithm, BlockCipherAlgorithm): + raise UnsupportedAlgorithm( + f"{name} requires a block cipher algorithm", + _Reasons.UNSUPPORTED_CIPHER, + ) + if len(nonce) * 8 != algorithm.block_size: + raise ValueError(f"Invalid nonce size ({len(nonce)}) for {name}.") + + +def _check_iv_and_key_length( + self: ModeWithInitializationVector, algorithm: CipherAlgorithm +) -> None: + if not isinstance(algorithm, BlockCipherAlgorithm): + raise UnsupportedAlgorithm( + f"{self} requires a block cipher algorithm", + _Reasons.UNSUPPORTED_CIPHER, + ) + _check_aes_key_length(self, algorithm) + _check_iv_length(self, algorithm) + + +class CBC(ModeWithInitializationVector): + name = "CBC" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class XTS(ModeWithTweak): + name = "XTS" + + def __init__(self, tweak: utils.Buffer): + utils._check_byteslike("tweak", tweak) + + if len(tweak) != 16: + raise ValueError("tweak must be 128-bits (16 bytes)") + + self._tweak = tweak + + @property + def tweak(self) -> utils.Buffer: + return self._tweak + + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + if isinstance(algorithm, (algorithms.AES128, algorithms.AES256)): + raise TypeError( + "The AES128 and AES256 classes do not support XTS, please use " + "the standard AES class instead." + ) + + if algorithm.key_size not in (256, 512): + raise ValueError( + "The XTS specification requires a 256-bit key for AES-128-XTS" + " and 512-bit key for AES-256-XTS" + ) + + +class ECB(Mode): + name = "ECB" + + validate_for_algorithm = _check_aes_key_length + + +class OFB(ModeWithInitializationVector): + name = "OFB" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class CFB(ModeWithInitializationVector): + name = "CFB" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class CFB8(ModeWithInitializationVector): + name = "CFB8" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class CTR(ModeWithNonce): + name = "CTR" + + def __init__(self, nonce: utils.Buffer): + utils._check_byteslike("nonce", nonce) + self._nonce = nonce + + @property + def nonce(self) -> utils.Buffer: + return self._nonce + + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + _check_aes_key_length(self, algorithm) + _check_nonce_length(self.nonce, self.name, algorithm) + + +class GCM(ModeWithInitializationVector, ModeWithAuthenticationTag): + name = "GCM" + _MAX_ENCRYPTED_BYTES = (2**39 - 256) // 8 + _MAX_AAD_BYTES = (2**64) // 8 + + def __init__( + self, + initialization_vector: utils.Buffer, + tag: bytes | None = None, + min_tag_length: int = 16, + ): + # OpenSSL 3.0.0 constrains GCM IVs to [64, 1024] bits inclusive + # This is a sane limit anyway so we'll enforce it here. + utils._check_byteslike("initialization_vector", initialization_vector) + if len(initialization_vector) < 8 or len(initialization_vector) > 128: + raise ValueError( + "initialization_vector must be between 8 and 128 bytes (64 " + "and 1024 bits)." + ) + self._initialization_vector = initialization_vector + if tag is not None: + utils._check_bytes("tag", tag) + if min_tag_length < 4: + raise ValueError("min_tag_length must be >= 4") + if len(tag) < min_tag_length: + raise ValueError( + f"Authentication tag must be {min_tag_length} bytes or " + "longer." + ) + self._tag = tag + self._min_tag_length = min_tag_length + + @property + def tag(self) -> bytes | None: + return self._tag + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + _check_aes_key_length(self, algorithm) + if not isinstance(algorithm, BlockCipherAlgorithm): + raise UnsupportedAlgorithm( + "GCM requires a block cipher algorithm", + _Reasons.UNSUPPORTED_CIPHER, + ) + block_size_bytes = algorithm.block_size // 8 + if self._tag is not None and len(self._tag) > block_size_bytes: + raise ValueError( + f"Authentication tag cannot be more than {block_size_bytes} " + "bytes." + ) diff --git a/lib/cryptography/hazmat/primitives/cmac.py b/lib/cryptography/hazmat/primitives/cmac.py new file mode 100644 index 0000000..2c67ce2 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/cmac.py @@ -0,0 +1,10 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = ["CMAC"] +CMAC = rust_openssl.cmac.CMAC diff --git a/lib/cryptography/hazmat/primitives/constant_time.py b/lib/cryptography/hazmat/primitives/constant_time.py new file mode 100644 index 0000000..3975c71 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/constant_time.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import hmac + + +def bytes_eq(a: bytes, b: bytes) -> bool: + if not isinstance(a, bytes) or not isinstance(b, bytes): + raise TypeError("a and b must be bytes.") + + return hmac.compare_digest(a, b) diff --git a/lib/cryptography/hazmat/primitives/hashes.py b/lib/cryptography/hazmat/primitives/hashes.py new file mode 100644 index 0000000..4b55ec3 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/hashes.py @@ -0,0 +1,246 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.utils import Buffer + +__all__ = [ + "MD5", + "SHA1", + "SHA3_224", + "SHA3_256", + "SHA3_384", + "SHA3_512", + "SHA224", + "SHA256", + "SHA384", + "SHA512", + "SHA512_224", + "SHA512_256", + "SHAKE128", + "SHAKE256", + "SM3", + "BLAKE2b", + "BLAKE2s", + "ExtendableOutputFunction", + "Hash", + "HashAlgorithm", + "HashContext", + "XOFHash", +] + + +class HashAlgorithm(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this algorithm (e.g. "sha256", "md5"). + """ + + @property + @abc.abstractmethod + def digest_size(self) -> int: + """ + The size of the resulting digest in bytes. + """ + + @property + @abc.abstractmethod + def block_size(self) -> int | None: + """ + The internal block size of the hash function, or None if the hash + function does not use blocks internally (e.g. SHA3). + """ + + +class HashContext(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def algorithm(self) -> HashAlgorithm: + """ + A HashAlgorithm that will be used by this context. + """ + + @abc.abstractmethod + def update(self, data: Buffer) -> None: + """ + Processes the provided bytes through the hash. + """ + + @abc.abstractmethod + def finalize(self) -> bytes: + """ + Finalizes the hash context and returns the hash digest as bytes. + """ + + @abc.abstractmethod + def copy(self) -> HashContext: + """ + Return a HashContext that is a copy of the current context. + """ + + +Hash = rust_openssl.hashes.Hash +HashContext.register(Hash) + +XOFHash = rust_openssl.hashes.XOFHash + + +class ExtendableOutputFunction(metaclass=abc.ABCMeta): + """ + An interface for extendable output functions. + """ + + +class SHA1(HashAlgorithm): + name = "sha1" + digest_size = 20 + block_size = 64 + + +class SHA512_224(HashAlgorithm): # noqa: N801 + name = "sha512-224" + digest_size = 28 + block_size = 128 + + +class SHA512_256(HashAlgorithm): # noqa: N801 + name = "sha512-256" + digest_size = 32 + block_size = 128 + + +class SHA224(HashAlgorithm): + name = "sha224" + digest_size = 28 + block_size = 64 + + +class SHA256(HashAlgorithm): + name = "sha256" + digest_size = 32 + block_size = 64 + + +class SHA384(HashAlgorithm): + name = "sha384" + digest_size = 48 + block_size = 128 + + +class SHA512(HashAlgorithm): + name = "sha512" + digest_size = 64 + block_size = 128 + + +class SHA3_224(HashAlgorithm): # noqa: N801 + name = "sha3-224" + digest_size = 28 + block_size = None + + +class SHA3_256(HashAlgorithm): # noqa: N801 + name = "sha3-256" + digest_size = 32 + block_size = None + + +class SHA3_384(HashAlgorithm): # noqa: N801 + name = "sha3-384" + digest_size = 48 + block_size = None + + +class SHA3_512(HashAlgorithm): # noqa: N801 + name = "sha3-512" + digest_size = 64 + block_size = None + + +class SHAKE128(HashAlgorithm, ExtendableOutputFunction): + name = "shake128" + block_size = None + + def __init__(self, digest_size: int): + if not isinstance(digest_size, int): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + +class SHAKE256(HashAlgorithm, ExtendableOutputFunction): + name = "shake256" + block_size = None + + def __init__(self, digest_size: int): + if not isinstance(digest_size, int): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + +class MD5(HashAlgorithm): + name = "md5" + digest_size = 16 + block_size = 64 + + +class BLAKE2b(HashAlgorithm): + name = "blake2b" + _max_digest_size = 64 + _min_digest_size = 1 + block_size = 128 + + def __init__(self, digest_size: int): + if digest_size != 64: + raise ValueError("Digest size must be 64") + + self._digest_size = digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + +class BLAKE2s(HashAlgorithm): + name = "blake2s" + block_size = 64 + _max_digest_size = 32 + _min_digest_size = 1 + + def __init__(self, digest_size: int): + if digest_size != 32: + raise ValueError("Digest size must be 32") + + self._digest_size = digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + +class SM3(HashAlgorithm): + name = "sm3" + digest_size = 32 + block_size = 64 diff --git a/lib/cryptography/hazmat/primitives/hmac.py b/lib/cryptography/hazmat/primitives/hmac.py new file mode 100644 index 0000000..a9442d5 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/hmac.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes + +__all__ = ["HMAC"] + +HMAC = rust_openssl.hmac.HMAC +hashes.HashContext.register(HMAC) diff --git a/lib/cryptography/hazmat/primitives/kdf/__init__.py b/lib/cryptography/hazmat/primitives/kdf/__init__.py new file mode 100644 index 0000000..79bb459 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/__init__.py @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + + +class KeyDerivationFunction(metaclass=abc.ABCMeta): + @abc.abstractmethod + def derive(self, key_material: bytes) -> bytes: + """ + Deterministically generates and returns a new key based on the existing + key material. + """ + + @abc.abstractmethod + def verify(self, key_material: bytes, expected_key: bytes) -> None: + """ + Checks whether the key generated by the key material matches the + expected derived key. Raises an exception if they do not match. + """ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..227880a Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/argon2.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/argon2.cpython-314.pyc new file mode 100644 index 0000000..5685144 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/argon2.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-314.pyc new file mode 100644 index 0000000..e731374 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-314.pyc new file mode 100644 index 0000000..7ef4046 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-314.pyc new file mode 100644 index 0000000..0bf5f7b Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-314.pyc new file mode 100644 index 0000000..5323cb5 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-314.pyc new file mode 100644 index 0000000..9758c68 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-314.pyc b/lib/cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-314.pyc new file mode 100644 index 0000000..a709910 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/kdf/argon2.py b/lib/cryptography/hazmat/primitives/kdf/argon2.py new file mode 100644 index 0000000..405fc8d --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/argon2.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +Argon2id = rust_openssl.kdf.Argon2id +KeyDerivationFunction.register(Argon2id) + +__all__ = ["Argon2id"] diff --git a/lib/cryptography/hazmat/primitives/kdf/concatkdf.py b/lib/cryptography/hazmat/primitives/kdf/concatkdf.py new file mode 100644 index 0000000..1b92841 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/concatkdf.py @@ -0,0 +1,125 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing +from collections.abc import Callable + +from cryptography import utils +from cryptography.exceptions import AlreadyFinalized, InvalidKey +from cryptography.hazmat.primitives import constant_time, hashes, hmac +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +def _int_to_u32be(n: int) -> bytes: + return n.to_bytes(length=4, byteorder="big") + + +def _common_args_checks( + algorithm: hashes.HashAlgorithm, + length: int, + otherinfo: bytes | None, +) -> None: + max_length = algorithm.digest_size * (2**32 - 1) + if length > max_length: + raise ValueError(f"Cannot derive keys larger than {max_length} bits.") + if otherinfo is not None: + utils._check_bytes("otherinfo", otherinfo) + + +def _concatkdf_derive( + key_material: utils.Buffer, + length: int, + auxfn: Callable[[], hashes.HashContext], + otherinfo: bytes, +) -> bytes: + utils._check_byteslike("key_material", key_material) + output = [b""] + outlen = 0 + counter = 1 + + while length > outlen: + h = auxfn() + h.update(_int_to_u32be(counter)) + h.update(key_material) + h.update(otherinfo) + output.append(h.finalize()) + outlen += len(output[-1]) + counter += 1 + + return b"".join(output)[:length] + + +class ConcatKDFHash(KeyDerivationFunction): + def __init__( + self, + algorithm: hashes.HashAlgorithm, + length: int, + otherinfo: bytes | None, + backend: typing.Any = None, + ): + _common_args_checks(algorithm, length, otherinfo) + self._algorithm = algorithm + self._length = length + self._otherinfo: bytes = otherinfo if otherinfo is not None else b"" + + self._used = False + + def _hash(self) -> hashes.Hash: + return hashes.Hash(self._algorithm) + + def derive(self, key_material: utils.Buffer) -> bytes: + if self._used: + raise AlreadyFinalized + self._used = True + return _concatkdf_derive( + key_material, self._length, self._hash, self._otherinfo + ) + + def verify(self, key_material: bytes, expected_key: bytes) -> None: + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey + + +class ConcatKDFHMAC(KeyDerivationFunction): + def __init__( + self, + algorithm: hashes.HashAlgorithm, + length: int, + salt: bytes | None, + otherinfo: bytes | None, + backend: typing.Any = None, + ): + _common_args_checks(algorithm, length, otherinfo) + self._algorithm = algorithm + self._length = length + self._otherinfo: bytes = otherinfo if otherinfo is not None else b"" + + if algorithm.block_size is None: + raise TypeError(f"{algorithm.name} is unsupported for ConcatKDF") + + if salt is None: + salt = b"\x00" * algorithm.block_size + else: + utils._check_bytes("salt", salt) + + self._salt = salt + + self._used = False + + def _hmac(self) -> hmac.HMAC: + return hmac.HMAC(self._salt, self._algorithm) + + def derive(self, key_material: utils.Buffer) -> bytes: + if self._used: + raise AlreadyFinalized + self._used = True + return _concatkdf_derive( + key_material, self._length, self._hmac, self._otherinfo + ) + + def verify(self, key_material: bytes, expected_key: bytes) -> None: + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/lib/cryptography/hazmat/primitives/kdf/hkdf.py b/lib/cryptography/hazmat/primitives/kdf/hkdf.py new file mode 100644 index 0000000..1e162d9 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/hkdf.py @@ -0,0 +1,16 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +HKDF = rust_openssl.kdf.HKDF +HKDFExpand = rust_openssl.kdf.HKDFExpand + +KeyDerivationFunction.register(HKDF) +KeyDerivationFunction.register(HKDFExpand) + +__all__ = ["HKDF", "HKDFExpand"] diff --git a/lib/cryptography/hazmat/primitives/kdf/kbkdf.py b/lib/cryptography/hazmat/primitives/kdf/kbkdf.py new file mode 100644 index 0000000..5b47137 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/kbkdf.py @@ -0,0 +1,303 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing +from collections.abc import Callable + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.primitives import ( + ciphers, + cmac, + constant_time, + hashes, + hmac, +) +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +class Mode(utils.Enum): + CounterMode = "ctr" + + +class CounterLocation(utils.Enum): + BeforeFixed = "before_fixed" + AfterFixed = "after_fixed" + MiddleFixed = "middle_fixed" + + +class _KBKDFDeriver: + def __init__( + self, + prf: Callable, + mode: Mode, + length: int, + rlen: int, + llen: int | None, + location: CounterLocation, + break_location: int | None, + label: bytes | None, + context: bytes | None, + fixed: bytes | None, + ): + assert callable(prf) + + if not isinstance(mode, Mode): + raise TypeError("mode must be of type Mode") + + if not isinstance(location, CounterLocation): + raise TypeError("location must be of type CounterLocation") + + if break_location is None and location is CounterLocation.MiddleFixed: + raise ValueError("Please specify a break_location") + + if ( + break_location is not None + and location != CounterLocation.MiddleFixed + ): + raise ValueError( + "break_location is ignored when location is not" + " CounterLocation.MiddleFixed" + ) + + if break_location is not None and not isinstance(break_location, int): + raise TypeError("break_location must be an integer") + + if break_location is not None and break_location < 0: + raise ValueError("break_location must be a positive integer") + + if (label or context) and fixed: + raise ValueError( + "When supplying fixed data, label and context are ignored." + ) + + if rlen is None or not self._valid_byte_length(rlen): + raise ValueError("rlen must be between 1 and 4") + + if llen is None and fixed is None: + raise ValueError("Please specify an llen") + + if llen is not None and not isinstance(llen, int): + raise TypeError("llen must be an integer") + + if llen == 0: + raise ValueError("llen must be non-zero") + + if label is None: + label = b"" + + if context is None: + context = b"" + + utils._check_bytes("label", label) + utils._check_bytes("context", context) + self._prf = prf + self._mode = mode + self._length = length + self._rlen = rlen + self._llen = llen + self._location = location + self._break_location = break_location + self._label = label + self._context = context + self._used = False + self._fixed_data = fixed + + @staticmethod + def _valid_byte_length(value: int) -> bool: + if not isinstance(value, int): + raise TypeError("value must be of type int") + + value_bin = utils.int_to_bytes(1, value) + return 1 <= len(value_bin) <= 4 + + def derive( + self, key_material: utils.Buffer, prf_output_size: int + ) -> bytes: + if self._used: + raise AlreadyFinalized + + utils._check_byteslike("key_material", key_material) + self._used = True + + # inverse floor division (equivalent to ceiling) + rounds = -(-self._length // prf_output_size) + + output = [b""] + + # For counter mode, the number of iterations shall not be + # larger than 2^r-1, where r <= 32 is the binary length of the counter + # This ensures that the counter values used as an input to the + # PRF will not repeat during a particular call to the KDF function. + r_bin = utils.int_to_bytes(1, self._rlen) + if rounds > pow(2, len(r_bin) * 8) - 1: + raise ValueError("There are too many iterations.") + + fixed = self._generate_fixed_input() + + if self._location == CounterLocation.BeforeFixed: + data_before_ctr = b"" + data_after_ctr = fixed + elif self._location == CounterLocation.AfterFixed: + data_before_ctr = fixed + data_after_ctr = b"" + else: + if isinstance( + self._break_location, int + ) and self._break_location > len(fixed): + raise ValueError("break_location offset > len(fixed)") + data_before_ctr = fixed[: self._break_location] + data_after_ctr = fixed[self._break_location :] + + for i in range(1, rounds + 1): + h = self._prf(key_material) + + counter = utils.int_to_bytes(i, self._rlen) + input_data = data_before_ctr + counter + data_after_ctr + + h.update(input_data) + + output.append(h.finalize()) + + return b"".join(output)[: self._length] + + def _generate_fixed_input(self) -> bytes: + if self._fixed_data and isinstance(self._fixed_data, bytes): + return self._fixed_data + + l_val = utils.int_to_bytes(self._length * 8, self._llen) + + return b"".join([self._label, b"\x00", self._context, l_val]) + + +class KBKDFHMAC(KeyDerivationFunction): + def __init__( + self, + algorithm: hashes.HashAlgorithm, + mode: Mode, + length: int, + rlen: int, + llen: int | None, + location: CounterLocation, + label: bytes | None, + context: bytes | None, + fixed: bytes | None, + backend: typing.Any = None, + *, + break_location: int | None = None, + ): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise UnsupportedAlgorithm( + "Algorithm supplied is not a supported hash algorithm.", + _Reasons.UNSUPPORTED_HASH, + ) + + from cryptography.hazmat.backends.openssl.backend import ( + backend as ossl, + ) + + if not ossl.hmac_supported(algorithm): + raise UnsupportedAlgorithm( + "Algorithm supplied is not a supported hmac algorithm.", + _Reasons.UNSUPPORTED_HASH, + ) + + self._algorithm = algorithm + + self._deriver = _KBKDFDeriver( + self._prf, + mode, + length, + rlen, + llen, + location, + break_location, + label, + context, + fixed, + ) + + def _prf(self, key_material: bytes) -> hmac.HMAC: + return hmac.HMAC(key_material, self._algorithm) + + def derive(self, key_material: utils.Buffer) -> bytes: + return self._deriver.derive(key_material, self._algorithm.digest_size) + + def verify(self, key_material: bytes, expected_key: bytes) -> None: + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey + + +class KBKDFCMAC(KeyDerivationFunction): + def __init__( + self, + algorithm, + mode: Mode, + length: int, + rlen: int, + llen: int | None, + location: CounterLocation, + label: bytes | None, + context: bytes | None, + fixed: bytes | None, + backend: typing.Any = None, + *, + break_location: int | None = None, + ): + if not issubclass( + algorithm, ciphers.BlockCipherAlgorithm + ) or not issubclass(algorithm, ciphers.CipherAlgorithm): + raise UnsupportedAlgorithm( + "Algorithm supplied is not a supported cipher algorithm.", + _Reasons.UNSUPPORTED_CIPHER, + ) + + self._algorithm = algorithm + self._cipher: ciphers.BlockCipherAlgorithm | None = None + + self._deriver = _KBKDFDeriver( + self._prf, + mode, + length, + rlen, + llen, + location, + break_location, + label, + context, + fixed, + ) + + def _prf(self, _: bytes) -> cmac.CMAC: + assert self._cipher is not None + + return cmac.CMAC(self._cipher) + + def derive(self, key_material: utils.Buffer) -> bytes: + self._cipher = self._algorithm(key_material) + + assert self._cipher is not None + + from cryptography.hazmat.backends.openssl.backend import ( + backend as ossl, + ) + + if not ossl.cmac_algorithm_supported(self._cipher): + raise UnsupportedAlgorithm( + "Algorithm supplied is not a supported cipher algorithm.", + _Reasons.UNSUPPORTED_CIPHER, + ) + + return self._deriver.derive(key_material, self._cipher.block_size // 8) + + def verify(self, key_material: bytes, expected_key: bytes) -> None: + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/lib/cryptography/hazmat/primitives/kdf/pbkdf2.py b/lib/cryptography/hazmat/primitives/kdf/pbkdf2.py new file mode 100644 index 0000000..d539f13 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/pbkdf2.py @@ -0,0 +1,62 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import constant_time, hashes +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +class PBKDF2HMAC(KeyDerivationFunction): + def __init__( + self, + algorithm: hashes.HashAlgorithm, + length: int, + salt: bytes, + iterations: int, + backend: typing.Any = None, + ): + from cryptography.hazmat.backends.openssl.backend import ( + backend as ossl, + ) + + if not ossl.pbkdf2_hmac_supported(algorithm): + raise UnsupportedAlgorithm( + f"{algorithm.name} is not supported for PBKDF2.", + _Reasons.UNSUPPORTED_HASH, + ) + self._used = False + self._algorithm = algorithm + self._length = length + utils._check_bytes("salt", salt) + self._salt = salt + self._iterations = iterations + + def derive(self, key_material: utils.Buffer) -> bytes: + if self._used: + raise AlreadyFinalized("PBKDF2 instances can only be used once.") + self._used = True + + return rust_openssl.kdf.derive_pbkdf2_hmac( + key_material, + self._algorithm, + self._salt, + self._iterations, + self._length, + ) + + def verify(self, key_material: bytes, expected_key: bytes) -> None: + derived_key = self.derive(key_material) + if not constant_time.bytes_eq(derived_key, expected_key): + raise InvalidKey("Keys do not match.") diff --git a/lib/cryptography/hazmat/primitives/kdf/scrypt.py b/lib/cryptography/hazmat/primitives/kdf/scrypt.py new file mode 100644 index 0000000..f791cee --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/scrypt.py @@ -0,0 +1,19 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import sys + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +# This is used by the scrypt tests to skip tests that require more memory +# than the MEM_LIMIT +_MEM_LIMIT = sys.maxsize // 2 + +Scrypt = rust_openssl.kdf.Scrypt +KeyDerivationFunction.register(Scrypt) + +__all__ = ["Scrypt"] diff --git a/lib/cryptography/hazmat/primitives/kdf/x963kdf.py b/lib/cryptography/hazmat/primitives/kdf/x963kdf.py new file mode 100644 index 0000000..63870cd --- /dev/null +++ b/lib/cryptography/hazmat/primitives/kdf/x963kdf.py @@ -0,0 +1,61 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography import utils +from cryptography.exceptions import AlreadyFinalized, InvalidKey +from cryptography.hazmat.primitives import constant_time, hashes +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +def _int_to_u32be(n: int) -> bytes: + return n.to_bytes(length=4, byteorder="big") + + +class X963KDF(KeyDerivationFunction): + def __init__( + self, + algorithm: hashes.HashAlgorithm, + length: int, + sharedinfo: bytes | None, + backend: typing.Any = None, + ): + max_len = algorithm.digest_size * (2**32 - 1) + if length > max_len: + raise ValueError(f"Cannot derive keys larger than {max_len} bits.") + if sharedinfo is not None: + utils._check_bytes("sharedinfo", sharedinfo) + + self._algorithm = algorithm + self._length = length + self._sharedinfo = sharedinfo + self._used = False + + def derive(self, key_material: utils.Buffer) -> bytes: + if self._used: + raise AlreadyFinalized + self._used = True + utils._check_byteslike("key_material", key_material) + output = [b""] + outlen = 0 + counter = 1 + + while self._length > outlen: + h = hashes.Hash(self._algorithm) + h.update(key_material) + h.update(_int_to_u32be(counter)) + if self._sharedinfo is not None: + h.update(self._sharedinfo) + output.append(h.finalize()) + outlen += len(output[-1]) + counter += 1 + + return b"".join(output)[: self._length] + + def verify(self, key_material: bytes, expected_key: bytes) -> None: + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/lib/cryptography/hazmat/primitives/keywrap.py b/lib/cryptography/hazmat/primitives/keywrap.py new file mode 100644 index 0000000..b93d87d --- /dev/null +++ b/lib/cryptography/hazmat/primitives/keywrap.py @@ -0,0 +1,177 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives.ciphers.modes import ECB +from cryptography.hazmat.primitives.constant_time import bytes_eq + + +def _wrap_core( + wrapping_key: bytes, + a: bytes, + r: list[bytes], +) -> bytes: + # RFC 3394 Key Wrap - 2.2.1 (index method) + encryptor = Cipher(AES(wrapping_key), ECB()).encryptor() + n = len(r) + for j in range(6): + for i in range(n): + # every encryption operation is a discrete 16 byte chunk (because + # AES has a 128-bit block size) and since we're using ECB it is + # safe to reuse the encryptor for the entire operation + b = encryptor.update(a + r[i]) + a = ( + int.from_bytes(b[:8], byteorder="big") ^ ((n * j) + i + 1) + ).to_bytes(length=8, byteorder="big") + r[i] = b[-8:] + + assert encryptor.finalize() == b"" + + return a + b"".join(r) + + +def aes_key_wrap( + wrapping_key: bytes, + key_to_wrap: bytes, + backend: typing.Any = None, +) -> bytes: + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + if len(key_to_wrap) < 16: + raise ValueError("The key to wrap must be at least 16 bytes") + + if len(key_to_wrap) % 8 != 0: + raise ValueError("The key to wrap must be a multiple of 8 bytes") + + a = b"\xa6\xa6\xa6\xa6\xa6\xa6\xa6\xa6" + r = [key_to_wrap[i : i + 8] for i in range(0, len(key_to_wrap), 8)] + return _wrap_core(wrapping_key, a, r) + + +def _unwrap_core( + wrapping_key: bytes, + a: bytes, + r: list[bytes], +) -> tuple[bytes, list[bytes]]: + # Implement RFC 3394 Key Unwrap - 2.2.2 (index method) + decryptor = Cipher(AES(wrapping_key), ECB()).decryptor() + n = len(r) + for j in reversed(range(6)): + for i in reversed(range(n)): + atr = ( + int.from_bytes(a, byteorder="big") ^ ((n * j) + i + 1) + ).to_bytes(length=8, byteorder="big") + r[i] + # every decryption operation is a discrete 16 byte chunk so + # it is safe to reuse the decryptor for the entire operation + b = decryptor.update(atr) + a = b[:8] + r[i] = b[-8:] + + assert decryptor.finalize() == b"" + return a, r + + +def aes_key_wrap_with_padding( + wrapping_key: bytes, + key_to_wrap: bytes, + backend: typing.Any = None, +) -> bytes: + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + aiv = b"\xa6\x59\x59\xa6" + len(key_to_wrap).to_bytes( + length=4, byteorder="big" + ) + # pad the key to wrap if necessary + pad = (8 - (len(key_to_wrap) % 8)) % 8 + key_to_wrap = key_to_wrap + b"\x00" * pad + if len(key_to_wrap) == 8: + # RFC 5649 - 4.1 - exactly 8 octets after padding + encryptor = Cipher(AES(wrapping_key), ECB()).encryptor() + b = encryptor.update(aiv + key_to_wrap) + assert encryptor.finalize() == b"" + return b + else: + r = [key_to_wrap[i : i + 8] for i in range(0, len(key_to_wrap), 8)] + return _wrap_core(wrapping_key, aiv, r) + + +def aes_key_unwrap_with_padding( + wrapping_key: bytes, + wrapped_key: bytes, + backend: typing.Any = None, +) -> bytes: + if len(wrapped_key) < 16: + raise InvalidUnwrap("Must be at least 16 bytes") + + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + if len(wrapped_key) == 16: + # RFC 5649 - 4.2 - exactly two 64-bit blocks + decryptor = Cipher(AES(wrapping_key), ECB()).decryptor() + out = decryptor.update(wrapped_key) + assert decryptor.finalize() == b"" + a = out[:8] + data = out[8:] + n = 1 + else: + r = [wrapped_key[i : i + 8] for i in range(0, len(wrapped_key), 8)] + encrypted_aiv = r.pop(0) + n = len(r) + a, r = _unwrap_core(wrapping_key, encrypted_aiv, r) + data = b"".join(r) + + # 1) Check that MSB(32,A) = A65959A6. + # 2) Check that 8*(n-1) < LSB(32,A) <= 8*n. If so, let + # MLI = LSB(32,A). + # 3) Let b = (8*n)-MLI, and then check that the rightmost b octets of + # the output data are zero. + mli = int.from_bytes(a[4:], byteorder="big") + b = (8 * n) - mli + if ( + not bytes_eq(a[:4], b"\xa6\x59\x59\xa6") + or not 8 * (n - 1) < mli <= 8 * n + or (b != 0 and not bytes_eq(data[-b:], b"\x00" * b)) + ): + raise InvalidUnwrap() + + if b == 0: + return data + else: + return data[:-b] + + +def aes_key_unwrap( + wrapping_key: bytes, + wrapped_key: bytes, + backend: typing.Any = None, +) -> bytes: + if len(wrapped_key) < 24: + raise InvalidUnwrap("Must be at least 24 bytes") + + if len(wrapped_key) % 8 != 0: + raise InvalidUnwrap("The wrapped key must be a multiple of 8 bytes") + + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + aiv = b"\xa6\xa6\xa6\xa6\xa6\xa6\xa6\xa6" + r = [wrapped_key[i : i + 8] for i in range(0, len(wrapped_key), 8)] + a = r.pop(0) + a, r = _unwrap_core(wrapping_key, a, r) + if not bytes_eq(a, aiv): + raise InvalidUnwrap() + + return b"".join(r) + + +class InvalidUnwrap(Exception): + pass diff --git a/lib/cryptography/hazmat/primitives/padding.py b/lib/cryptography/hazmat/primitives/padding.py new file mode 100644 index 0000000..f9cd1f1 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/padding.py @@ -0,0 +1,69 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.hazmat.bindings._rust import ( + ANSIX923PaddingContext, + ANSIX923UnpaddingContext, + PKCS7PaddingContext, + PKCS7UnpaddingContext, +) + + +class PaddingContext(metaclass=abc.ABCMeta): + @abc.abstractmethod + def update(self, data: utils.Buffer) -> bytes: + """ + Pads the provided bytes and returns any available data as bytes. + """ + + @abc.abstractmethod + def finalize(self) -> bytes: + """ + Finalize the padding, returns bytes. + """ + + +def _byte_padding_check(block_size: int) -> None: + if not (0 <= block_size <= 2040): + raise ValueError("block_size must be in range(0, 2041).") + + if block_size % 8 != 0: + raise ValueError("block_size must be a multiple of 8.") + + +class PKCS7: + def __init__(self, block_size: int): + _byte_padding_check(block_size) + self.block_size = block_size + + def padder(self) -> PaddingContext: + return PKCS7PaddingContext(self.block_size) + + def unpadder(self) -> PaddingContext: + return PKCS7UnpaddingContext(self.block_size) + + +PaddingContext.register(PKCS7PaddingContext) +PaddingContext.register(PKCS7UnpaddingContext) + + +class ANSIX923: + def __init__(self, block_size: int): + _byte_padding_check(block_size) + self.block_size = block_size + + def padder(self) -> PaddingContext: + return ANSIX923PaddingContext(self.block_size) + + def unpadder(self) -> PaddingContext: + return ANSIX923UnpaddingContext(self.block_size) + + +PaddingContext.register(ANSIX923PaddingContext) +PaddingContext.register(ANSIX923UnpaddingContext) diff --git a/lib/cryptography/hazmat/primitives/poly1305.py b/lib/cryptography/hazmat/primitives/poly1305.py new file mode 100644 index 0000000..7f5a77a --- /dev/null +++ b/lib/cryptography/hazmat/primitives/poly1305.py @@ -0,0 +1,11 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = ["Poly1305"] + +Poly1305 = rust_openssl.poly1305.Poly1305 diff --git a/lib/cryptography/hazmat/primitives/serialization/__init__.py b/lib/cryptography/hazmat/primitives/serialization/__init__.py new file mode 100644 index 0000000..62283cc --- /dev/null +++ b/lib/cryptography/hazmat/primitives/serialization/__init__.py @@ -0,0 +1,65 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.primitives._serialization import ( + BestAvailableEncryption, + Encoding, + KeySerializationEncryption, + NoEncryption, + ParameterFormat, + PrivateFormat, + PublicFormat, + _KeySerializationEncryption, +) +from cryptography.hazmat.primitives.serialization.base import ( + load_der_parameters, + load_der_private_key, + load_der_public_key, + load_pem_parameters, + load_pem_private_key, + load_pem_public_key, +) +from cryptography.hazmat.primitives.serialization.ssh import ( + SSHCertificate, + SSHCertificateBuilder, + SSHCertificateType, + SSHCertPrivateKeyTypes, + SSHCertPublicKeyTypes, + SSHPrivateKeyTypes, + SSHPublicKeyTypes, + load_ssh_private_key, + load_ssh_public_identity, + load_ssh_public_key, + ssh_key_fingerprint, +) + +__all__ = [ + "BestAvailableEncryption", + "Encoding", + "KeySerializationEncryption", + "NoEncryption", + "ParameterFormat", + "PrivateFormat", + "PublicFormat", + "SSHCertPrivateKeyTypes", + "SSHCertPublicKeyTypes", + "SSHCertificate", + "SSHCertificateBuilder", + "SSHCertificateType", + "SSHPrivateKeyTypes", + "SSHPublicKeyTypes", + "_KeySerializationEncryption", + "load_der_parameters", + "load_der_private_key", + "load_der_public_key", + "load_pem_parameters", + "load_pem_private_key", + "load_pem_public_key", + "load_ssh_private_key", + "load_ssh_public_identity", + "load_ssh_public_key", + "ssh_key_fingerprint", +] diff --git a/lib/cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..6dc92e1 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-314.pyc b/lib/cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..8a95887 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-314.pyc b/lib/cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-314.pyc new file mode 100644 index 0000000..9da29e0 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-314.pyc b/lib/cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-314.pyc new file mode 100644 index 0000000..5476b84 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-314.pyc b/lib/cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-314.pyc new file mode 100644 index 0000000..8983e9c Binary files /dev/null and b/lib/cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/serialization/base.py b/lib/cryptography/hazmat/primitives/serialization/base.py new file mode 100644 index 0000000..e7c998b --- /dev/null +++ b/lib/cryptography/hazmat/primitives/serialization/base.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +load_pem_private_key = rust_openssl.keys.load_pem_private_key +load_der_private_key = rust_openssl.keys.load_der_private_key + +load_pem_public_key = rust_openssl.keys.load_pem_public_key +load_der_public_key = rust_openssl.keys.load_der_public_key + +load_pem_parameters = rust_openssl.dh.from_pem_parameters +load_der_parameters = rust_openssl.dh.from_der_parameters diff --git a/lib/cryptography/hazmat/primitives/serialization/pkcs12.py b/lib/cryptography/hazmat/primitives/serialization/pkcs12.py new file mode 100644 index 0000000..58884ff --- /dev/null +++ b/lib/cryptography/hazmat/primitives/serialization/pkcs12.py @@ -0,0 +1,176 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing +from collections.abc import Iterable + +from cryptography import x509 +from cryptography.hazmat.bindings._rust import pkcs12 as rust_pkcs12 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives._serialization import PBES as PBES +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed448, + ed25519, + rsa, +) +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes + +__all__ = [ + "PBES", + "PKCS12Certificate", + "PKCS12KeyAndCertificates", + "PKCS12PrivateKeyTypes", + "load_key_and_certificates", + "load_pkcs12", + "serialize_java_truststore", + "serialize_key_and_certificates", +] + +PKCS12PrivateKeyTypes = typing.Union[ + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, +] + + +PKCS12Certificate = rust_pkcs12.PKCS12Certificate + + +class PKCS12KeyAndCertificates: + def __init__( + self, + key: PrivateKeyTypes | None, + cert: PKCS12Certificate | None, + additional_certs: list[PKCS12Certificate], + ): + if key is not None and not isinstance( + key, + ( + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + ), + ): + raise TypeError( + "Key must be RSA, DSA, EllipticCurve, ED25519, or ED448" + " private key, or None." + ) + if cert is not None and not isinstance(cert, PKCS12Certificate): + raise TypeError("cert must be a PKCS12Certificate object or None") + if not all( + isinstance(add_cert, PKCS12Certificate) + for add_cert in additional_certs + ): + raise TypeError( + "all values in additional_certs must be PKCS12Certificate" + " objects" + ) + self._key = key + self._cert = cert + self._additional_certs = additional_certs + + @property + def key(self) -> PrivateKeyTypes | None: + return self._key + + @property + def cert(self) -> PKCS12Certificate | None: + return self._cert + + @property + def additional_certs(self) -> list[PKCS12Certificate]: + return self._additional_certs + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PKCS12KeyAndCertificates): + return NotImplemented + + return ( + self.key == other.key + and self.cert == other.cert + and self.additional_certs == other.additional_certs + ) + + def __hash__(self) -> int: + return hash((self.key, self.cert, tuple(self.additional_certs))) + + def __repr__(self) -> str: + fmt = ( + "" + ) + return fmt.format(self.key, self.cert, self.additional_certs) + + +load_key_and_certificates = rust_pkcs12.load_key_and_certificates +load_pkcs12 = rust_pkcs12.load_pkcs12 + + +_PKCS12CATypes = typing.Union[ + x509.Certificate, + PKCS12Certificate, +] + + +def serialize_java_truststore( + certs: Iterable[PKCS12Certificate], + encryption_algorithm: serialization.KeySerializationEncryption, +) -> bytes: + if not certs: + raise ValueError("You must supply at least one cert") + + if not isinstance( + encryption_algorithm, serialization.KeySerializationEncryption + ): + raise TypeError( + "Key encryption algorithm must be a " + "KeySerializationEncryption instance" + ) + + return rust_pkcs12.serialize_java_truststore(certs, encryption_algorithm) + + +def serialize_key_and_certificates( + name: bytes | None, + key: PKCS12PrivateKeyTypes | None, + cert: x509.Certificate | None, + cas: Iterable[_PKCS12CATypes] | None, + encryption_algorithm: serialization.KeySerializationEncryption, +) -> bytes: + if key is not None and not isinstance( + key, + ( + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + ), + ): + raise TypeError( + "Key must be RSA, DSA, EllipticCurve, ED25519, or ED448" + " private key, or None." + ) + + if not isinstance( + encryption_algorithm, serialization.KeySerializationEncryption + ): + raise TypeError( + "Key encryption algorithm must be a " + "KeySerializationEncryption instance" + ) + + if key is None and cert is None and not cas: + raise ValueError("You must supply at least one of key, cert, or cas") + + return rust_pkcs12.serialize_key_and_certificates( + name, key, cert, cas, encryption_algorithm + ) diff --git a/lib/cryptography/hazmat/primitives/serialization/pkcs7.py b/lib/cryptography/hazmat/primitives/serialization/pkcs7.py new file mode 100644 index 0000000..456dc5b --- /dev/null +++ b/lib/cryptography/hazmat/primitives/serialization/pkcs7.py @@ -0,0 +1,411 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import email.base64mime +import email.generator +import email.message +import email.policy +import io +import typing +from collections.abc import Iterable + +from cryptography import utils, x509 +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import pkcs7 as rust_pkcs7 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec, padding, rsa +from cryptography.hazmat.primitives.ciphers import ( + algorithms, +) +from cryptography.utils import _check_byteslike + +load_pem_pkcs7_certificates = rust_pkcs7.load_pem_pkcs7_certificates + +load_der_pkcs7_certificates = rust_pkcs7.load_der_pkcs7_certificates + +serialize_certificates = rust_pkcs7.serialize_certificates + +PKCS7HashTypes = typing.Union[ + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, +] + +PKCS7PrivateKeyTypes = typing.Union[ + rsa.RSAPrivateKey, ec.EllipticCurvePrivateKey +] + +ContentEncryptionAlgorithm = typing.Union[ + typing.Type[algorithms.AES128], typing.Type[algorithms.AES256] +] + + +class PKCS7Options(utils.Enum): + Text = "Add text/plain MIME type" + Binary = "Don't translate input data into canonical MIME format" + DetachedSignature = "Don't embed data in the PKCS7 structure" + NoCapabilities = "Don't embed SMIME capabilities" + NoAttributes = "Don't embed authenticatedAttributes" + NoCerts = "Don't embed signer certificate" + + +class PKCS7SignatureBuilder: + def __init__( + self, + data: utils.Buffer | None = None, + signers: list[ + tuple[ + x509.Certificate, + PKCS7PrivateKeyTypes, + PKCS7HashTypes, + padding.PSS | padding.PKCS1v15 | None, + ] + ] = [], + additional_certs: list[x509.Certificate] = [], + ): + self._data = data + self._signers = signers + self._additional_certs = additional_certs + + def set_data(self, data: utils.Buffer) -> PKCS7SignatureBuilder: + _check_byteslike("data", data) + if self._data is not None: + raise ValueError("data may only be set once") + + return PKCS7SignatureBuilder(data, self._signers) + + def add_signer( + self, + certificate: x509.Certificate, + private_key: PKCS7PrivateKeyTypes, + hash_algorithm: PKCS7HashTypes, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ) -> PKCS7SignatureBuilder: + if not isinstance( + hash_algorithm, + ( + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + ), + ): + raise TypeError( + "hash_algorithm must be one of hashes.SHA224, " + "SHA256, SHA384, or SHA512" + ) + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + if not isinstance( + private_key, (rsa.RSAPrivateKey, ec.EllipticCurvePrivateKey) + ): + raise TypeError("Only RSA & EC keys are supported at this time.") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + return PKCS7SignatureBuilder( + self._data, + [ + *self._signers, + (certificate, private_key, hash_algorithm, rsa_padding), + ], + ) + + def add_certificate( + self, certificate: x509.Certificate + ) -> PKCS7SignatureBuilder: + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + return PKCS7SignatureBuilder( + self._data, self._signers, [*self._additional_certs, certificate] + ) + + def sign( + self, + encoding: serialization.Encoding, + options: Iterable[PKCS7Options], + backend: typing.Any = None, + ) -> bytes: + if len(self._signers) == 0: + raise ValueError("Must have at least one signer") + if self._data is None: + raise ValueError("You must add data to sign") + options = list(options) + if not all(isinstance(x, PKCS7Options) for x in options): + raise ValueError("options must be from the PKCS7Options enum") + if encoding not in ( + serialization.Encoding.PEM, + serialization.Encoding.DER, + serialization.Encoding.SMIME, + ): + raise ValueError( + "Must be PEM, DER, or SMIME from the Encoding enum" + ) + + # Text is a meaningless option unless it is accompanied by + # DetachedSignature + if ( + PKCS7Options.Text in options + and PKCS7Options.DetachedSignature not in options + ): + raise ValueError( + "When passing the Text option you must also pass " + "DetachedSignature" + ) + + if PKCS7Options.Text in options and encoding in ( + serialization.Encoding.DER, + serialization.Encoding.PEM, + ): + raise ValueError( + "The Text option is only available for SMIME serialization" + ) + + # No attributes implies no capabilities so we'll error if you try to + # pass both. + if ( + PKCS7Options.NoAttributes in options + and PKCS7Options.NoCapabilities in options + ): + raise ValueError( + "NoAttributes is a superset of NoCapabilities. Do not pass " + "both values." + ) + + return rust_pkcs7.sign_and_serialize(self, encoding, options) + + +class PKCS7EnvelopeBuilder: + def __init__( + self, + *, + _data: bytes | None = None, + _recipients: list[x509.Certificate] | None = None, + _content_encryption_algorithm: ContentEncryptionAlgorithm + | None = None, + ): + from cryptography.hazmat.backends.openssl.backend import ( + backend as ossl, + ) + + if not ossl.rsa_encryption_supported(padding=padding.PKCS1v15()): + raise UnsupportedAlgorithm( + "RSA with PKCS1 v1.5 padding is not supported by this version" + " of OpenSSL.", + _Reasons.UNSUPPORTED_PADDING, + ) + self._data = _data + self._recipients = _recipients if _recipients is not None else [] + self._content_encryption_algorithm = _content_encryption_algorithm + + def set_data(self, data: bytes) -> PKCS7EnvelopeBuilder: + _check_byteslike("data", data) + if self._data is not None: + raise ValueError("data may only be set once") + + return PKCS7EnvelopeBuilder( + _data=data, + _recipients=self._recipients, + _content_encryption_algorithm=self._content_encryption_algorithm, + ) + + def add_recipient( + self, + certificate: x509.Certificate, + ) -> PKCS7EnvelopeBuilder: + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + if not isinstance(certificate.public_key(), rsa.RSAPublicKey): + raise TypeError("Only RSA keys are supported at this time.") + + return PKCS7EnvelopeBuilder( + _data=self._data, + _recipients=[ + *self._recipients, + certificate, + ], + _content_encryption_algorithm=self._content_encryption_algorithm, + ) + + def set_content_encryption_algorithm( + self, content_encryption_algorithm: ContentEncryptionAlgorithm + ) -> PKCS7EnvelopeBuilder: + if self._content_encryption_algorithm is not None: + raise ValueError("Content encryption algo may only be set once") + if content_encryption_algorithm not in { + algorithms.AES128, + algorithms.AES256, + }: + raise TypeError("Only AES128 and AES256 are supported") + + return PKCS7EnvelopeBuilder( + _data=self._data, + _recipients=self._recipients, + _content_encryption_algorithm=content_encryption_algorithm, + ) + + def encrypt( + self, + encoding: serialization.Encoding, + options: Iterable[PKCS7Options], + ) -> bytes: + if len(self._recipients) == 0: + raise ValueError("Must have at least one recipient") + if self._data is None: + raise ValueError("You must add data to encrypt") + + # The default content encryption algorithm is AES-128, which the S/MIME + # v3.2 RFC specifies as MUST support (https://datatracker.ietf.org/doc/html/rfc5751#section-2.7) + content_encryption_algorithm = ( + self._content_encryption_algorithm or algorithms.AES128 + ) + + options = list(options) + if not all(isinstance(x, PKCS7Options) for x in options): + raise ValueError("options must be from the PKCS7Options enum") + if encoding not in ( + serialization.Encoding.PEM, + serialization.Encoding.DER, + serialization.Encoding.SMIME, + ): + raise ValueError( + "Must be PEM, DER, or SMIME from the Encoding enum" + ) + + # Only allow options that make sense for encryption + if any( + opt not in [PKCS7Options.Text, PKCS7Options.Binary] + for opt in options + ): + raise ValueError( + "Only the following options are supported for encryption: " + "Text, Binary" + ) + elif PKCS7Options.Text in options and PKCS7Options.Binary in options: + # OpenSSL accepts both options at the same time, but ignores Text. + # We fail defensively to avoid unexpected outputs. + raise ValueError( + "Cannot use Binary and Text options at the same time" + ) + + return rust_pkcs7.encrypt_and_serialize( + self, content_encryption_algorithm, encoding, options + ) + + +pkcs7_decrypt_der = rust_pkcs7.decrypt_der +pkcs7_decrypt_pem = rust_pkcs7.decrypt_pem +pkcs7_decrypt_smime = rust_pkcs7.decrypt_smime + + +def _smime_signed_encode( + data: bytes, signature: bytes, micalg: str, text_mode: bool +) -> bytes: + # This function works pretty hard to replicate what OpenSSL does + # precisely. For good and for ill. + + m = email.message.Message() + m.add_header("MIME-Version", "1.0") + m.add_header( + "Content-Type", + "multipart/signed", + protocol="application/x-pkcs7-signature", + micalg=micalg, + ) + + m.preamble = "This is an S/MIME signed message\n" + + msg_part = OpenSSLMimePart() + msg_part.set_payload(data) + if text_mode: + msg_part.add_header("Content-Type", "text/plain") + m.attach(msg_part) + + sig_part = email.message.MIMEPart() + sig_part.add_header( + "Content-Type", "application/x-pkcs7-signature", name="smime.p7s" + ) + sig_part.add_header("Content-Transfer-Encoding", "base64") + sig_part.add_header( + "Content-Disposition", "attachment", filename="smime.p7s" + ) + sig_part.set_payload( + email.base64mime.body_encode(signature, maxlinelen=65) + ) + del sig_part["MIME-Version"] + m.attach(sig_part) + + fp = io.BytesIO() + g = email.generator.BytesGenerator( + fp, + maxheaderlen=0, + mangle_from_=False, + policy=m.policy.clone(linesep="\r\n"), + ) + g.flatten(m) + return fp.getvalue() + + +def _smime_enveloped_encode(data: bytes) -> bytes: + m = email.message.Message() + m.add_header("MIME-Version", "1.0") + m.add_header("Content-Disposition", "attachment", filename="smime.p7m") + m.add_header( + "Content-Type", + "application/pkcs7-mime", + smime_type="enveloped-data", + name="smime.p7m", + ) + m.add_header("Content-Transfer-Encoding", "base64") + + m.set_payload(email.base64mime.body_encode(data, maxlinelen=65)) + + return m.as_bytes(policy=m.policy.clone(linesep="\n", max_line_length=0)) + + +def _smime_enveloped_decode(data: bytes) -> bytes: + m = email.message_from_bytes(data) + if m.get_content_type() not in { + "application/x-pkcs7-mime", + "application/pkcs7-mime", + }: + raise ValueError("Not an S/MIME enveloped message") + return bytes(m.get_payload(decode=True)) + + +def _smime_remove_text_headers(data: bytes) -> bytes: + m = email.message_from_bytes(data) + # Using get() instead of get_content_type() since it has None as default, + # where the latter has "text/plain". Both methods are case-insensitive. + content_type = m.get("content-type") + if content_type is None: + raise ValueError( + "Decrypted MIME data has no 'Content-Type' header. " + "Please remove the 'Text' option to parse it manually." + ) + if "text/plain" not in content_type: + raise ValueError( + f"Decrypted MIME data content type is '{content_type}', not " + "'text/plain'. Remove the 'Text' option to parse it manually." + ) + return bytes(m.get_payload(decode=True)) + + +class OpenSSLMimePart(email.message.MIMEPart): + # A MIMEPart subclass that replicates OpenSSL's behavior of not including + # a newline if there are no headers. + def _write_headers(self, generator) -> None: + if list(self.raw_items()): + generator._write_headers(self) diff --git a/lib/cryptography/hazmat/primitives/serialization/ssh.py b/lib/cryptography/hazmat/primitives/serialization/ssh.py new file mode 100644 index 0000000..cb10cf8 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/serialization/ssh.py @@ -0,0 +1,1619 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import binascii +import enum +import os +import re +import typing +import warnings +from base64 import encodebytes as _base64_encode +from dataclasses import dataclass + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed25519, + padding, + rsa, +) +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils +from cryptography.hazmat.primitives.ciphers import ( + AEADDecryptionContext, + Cipher, + algorithms, + modes, +) +from cryptography.hazmat.primitives.serialization import ( + Encoding, + KeySerializationEncryption, + NoEncryption, + PrivateFormat, + PublicFormat, + _KeySerializationEncryption, +) + +try: + from bcrypt import kdf as _bcrypt_kdf + + _bcrypt_supported = True +except ImportError: + _bcrypt_supported = False + + def _bcrypt_kdf( + password: bytes, + salt: bytes, + desired_key_bytes: int, + rounds: int, + ignore_few_rounds: bool = False, + ) -> bytes: + raise UnsupportedAlgorithm("Need bcrypt module") + + +_SSH_ED25519 = b"ssh-ed25519" +_SSH_RSA = b"ssh-rsa" +_SSH_DSA = b"ssh-dss" +_ECDSA_NISTP256 = b"ecdsa-sha2-nistp256" +_ECDSA_NISTP384 = b"ecdsa-sha2-nistp384" +_ECDSA_NISTP521 = b"ecdsa-sha2-nistp521" +_CERT_SUFFIX = b"-cert-v01@openssh.com" + +# U2F application string suffixed pubkey +_SK_SSH_ED25519 = b"sk-ssh-ed25519@openssh.com" +_SK_SSH_ECDSA_NISTP256 = b"sk-ecdsa-sha2-nistp256@openssh.com" + +# These are not key types, only algorithms, so they cannot appear +# as a public key type +_SSH_RSA_SHA256 = b"rsa-sha2-256" +_SSH_RSA_SHA512 = b"rsa-sha2-512" + +_SSH_PUBKEY_RC = re.compile(rb"\A(\S+)[ \t]+(\S+)") +_SK_MAGIC = b"openssh-key-v1\0" +_SK_START = b"-----BEGIN OPENSSH PRIVATE KEY-----" +_SK_END = b"-----END OPENSSH PRIVATE KEY-----" +_BCRYPT = b"bcrypt" +_NONE = b"none" +_DEFAULT_CIPHER = b"aes256-ctr" +_DEFAULT_ROUNDS = 16 + +# re is only way to work on bytes-like data +_PEM_RC = re.compile(_SK_START + b"(.*?)" + _SK_END, re.DOTALL) + +# padding for max blocksize +_PADDING = memoryview(bytearray(range(1, 1 + 16))) + + +@dataclass +class _SSHCipher: + alg: type[algorithms.AES] + key_len: int + mode: type[modes.CTR] | type[modes.CBC] | type[modes.GCM] + block_len: int + iv_len: int + tag_len: int | None + is_aead: bool + + +# ciphers that are actually used in key wrapping +_SSH_CIPHERS: dict[bytes, _SSHCipher] = { + b"aes256-ctr": _SSHCipher( + alg=algorithms.AES, + key_len=32, + mode=modes.CTR, + block_len=16, + iv_len=16, + tag_len=None, + is_aead=False, + ), + b"aes256-cbc": _SSHCipher( + alg=algorithms.AES, + key_len=32, + mode=modes.CBC, + block_len=16, + iv_len=16, + tag_len=None, + is_aead=False, + ), + b"aes256-gcm@openssh.com": _SSHCipher( + alg=algorithms.AES, + key_len=32, + mode=modes.GCM, + block_len=16, + iv_len=12, + tag_len=16, + is_aead=True, + ), +} + +# map local curve name to key type +_ECDSA_KEY_TYPE = { + "secp256r1": _ECDSA_NISTP256, + "secp384r1": _ECDSA_NISTP384, + "secp521r1": _ECDSA_NISTP521, +} + + +def _get_ssh_key_type(key: SSHPrivateKeyTypes | SSHPublicKeyTypes) -> bytes: + if isinstance(key, ec.EllipticCurvePrivateKey): + key_type = _ecdsa_key_type(key.public_key()) + elif isinstance(key, ec.EllipticCurvePublicKey): + key_type = _ecdsa_key_type(key) + elif isinstance(key, (rsa.RSAPrivateKey, rsa.RSAPublicKey)): + key_type = _SSH_RSA + elif isinstance(key, (dsa.DSAPrivateKey, dsa.DSAPublicKey)): + key_type = _SSH_DSA + elif isinstance( + key, (ed25519.Ed25519PrivateKey, ed25519.Ed25519PublicKey) + ): + key_type = _SSH_ED25519 + else: + raise ValueError("Unsupported key type") + + return key_type + + +def _ecdsa_key_type(public_key: ec.EllipticCurvePublicKey) -> bytes: + """Return SSH key_type and curve_name for private key.""" + curve = public_key.curve + if curve.name not in _ECDSA_KEY_TYPE: + raise ValueError( + f"Unsupported curve for ssh private key: {curve.name!r}" + ) + return _ECDSA_KEY_TYPE[curve.name] + + +def _ssh_pem_encode( + data: utils.Buffer, + prefix: bytes = _SK_START + b"\n", + suffix: bytes = _SK_END + b"\n", +) -> bytes: + return b"".join([prefix, _base64_encode(data), suffix]) + + +def _check_block_size(data: utils.Buffer, block_len: int) -> None: + """Require data to be full blocks""" + if not data or len(data) % block_len != 0: + raise ValueError("Corrupt data: missing padding") + + +def _check_empty(data: utils.Buffer) -> None: + """All data should have been parsed.""" + if data: + raise ValueError("Corrupt data: unparsed data") + + +def _init_cipher( + ciphername: bytes, + password: bytes | None, + salt: bytes, + rounds: int, +) -> Cipher[modes.CBC | modes.CTR | modes.GCM]: + """Generate key + iv and return cipher.""" + if not password: + raise TypeError( + "Key is password-protected, but password was not provided." + ) + + ciph = _SSH_CIPHERS[ciphername] + seed = _bcrypt_kdf( + password, salt, ciph.key_len + ciph.iv_len, rounds, True + ) + return Cipher( + ciph.alg(seed[: ciph.key_len]), + ciph.mode(seed[ciph.key_len :]), + ) + + +def _get_u32(data: memoryview) -> tuple[int, memoryview]: + """Uint32""" + if len(data) < 4: + raise ValueError("Invalid data") + return int.from_bytes(data[:4], byteorder="big"), data[4:] + + +def _get_u64(data: memoryview) -> tuple[int, memoryview]: + """Uint64""" + if len(data) < 8: + raise ValueError("Invalid data") + return int.from_bytes(data[:8], byteorder="big"), data[8:] + + +def _get_sshstr(data: memoryview) -> tuple[memoryview, memoryview]: + """Bytes with u32 length prefix""" + n, data = _get_u32(data) + if n > len(data): + raise ValueError("Invalid data") + return data[:n], data[n:] + + +def _get_mpint(data: memoryview) -> tuple[int, memoryview]: + """Big integer.""" + val, data = _get_sshstr(data) + if val and val[0] > 0x7F: + raise ValueError("Invalid data") + return int.from_bytes(val, "big"), data + + +def _to_mpint(val: int) -> bytes: + """Storage format for signed bigint.""" + if val < 0: + raise ValueError("negative mpint not allowed") + if not val: + return b"" + nbytes = (val.bit_length() + 8) // 8 + return utils.int_to_bytes(val, nbytes) + + +class _FragList: + """Build recursive structure without data copy.""" + + flist: list[utils.Buffer] + + def __init__(self, init: list[utils.Buffer] | None = None) -> None: + self.flist = [] + if init: + self.flist.extend(init) + + def put_raw(self, val: utils.Buffer) -> None: + """Add plain bytes""" + self.flist.append(val) + + def put_u32(self, val: int) -> None: + """Big-endian uint32""" + self.flist.append(val.to_bytes(length=4, byteorder="big")) + + def put_u64(self, val: int) -> None: + """Big-endian uint64""" + self.flist.append(val.to_bytes(length=8, byteorder="big")) + + def put_sshstr(self, val: bytes | _FragList) -> None: + """Bytes prefixed with u32 length""" + if isinstance(val, (bytes, memoryview, bytearray)): + self.put_u32(len(val)) + self.flist.append(val) + else: + self.put_u32(val.size()) + self.flist.extend(val.flist) + + def put_mpint(self, val: int) -> None: + """Big-endian bigint prefixed with u32 length""" + self.put_sshstr(_to_mpint(val)) + + def size(self) -> int: + """Current number of bytes""" + return sum(map(len, self.flist)) + + def render(self, dstbuf: memoryview, pos: int = 0) -> int: + """Write into bytearray""" + for frag in self.flist: + flen = len(frag) + start, pos = pos, pos + flen + dstbuf[start:pos] = frag + return pos + + def tobytes(self) -> bytes: + """Return as bytes""" + buf = memoryview(bytearray(self.size())) + self.render(buf) + return buf.tobytes() + + +class _SSHFormatRSA: + """Format for RSA keys. + + Public: + mpint e, n + Private: + mpint n, e, d, iqmp, p, q + """ + + def get_public( + self, data: memoryview + ) -> tuple[tuple[int, int], memoryview]: + """RSA public fields""" + e, data = _get_mpint(data) + n, data = _get_mpint(data) + return (e, n), data + + def load_public( + self, data: memoryview + ) -> tuple[rsa.RSAPublicKey, memoryview]: + """Make RSA public key from data.""" + (e, n), data = self.get_public(data) + public_numbers = rsa.RSAPublicNumbers(e, n) + public_key = public_numbers.public_key() + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[rsa.RSAPrivateKey, memoryview]: + """Make RSA private key from data.""" + n, data = _get_mpint(data) + e, data = _get_mpint(data) + d, data = _get_mpint(data) + iqmp, data = _get_mpint(data) + p, data = _get_mpint(data) + q, data = _get_mpint(data) + + if (e, n) != pubfields: + raise ValueError("Corrupt data: rsa field mismatch") + dmp1 = rsa.rsa_crt_dmp1(d, p) + dmq1 = rsa.rsa_crt_dmq1(d, q) + public_numbers = rsa.RSAPublicNumbers(e, n) + private_numbers = rsa.RSAPrivateNumbers( + p, q, d, dmp1, dmq1, iqmp, public_numbers + ) + private_key = private_numbers.private_key( + unsafe_skip_rsa_key_validation=unsafe_skip_rsa_key_validation + ) + return private_key, data + + def encode_public( + self, public_key: rsa.RSAPublicKey, f_pub: _FragList + ) -> None: + """Write RSA public key""" + pubn = public_key.public_numbers() + f_pub.put_mpint(pubn.e) + f_pub.put_mpint(pubn.n) + + def encode_private( + self, private_key: rsa.RSAPrivateKey, f_priv: _FragList + ) -> None: + """Write RSA private key""" + private_numbers = private_key.private_numbers() + public_numbers = private_numbers.public_numbers + + f_priv.put_mpint(public_numbers.n) + f_priv.put_mpint(public_numbers.e) + + f_priv.put_mpint(private_numbers.d) + f_priv.put_mpint(private_numbers.iqmp) + f_priv.put_mpint(private_numbers.p) + f_priv.put_mpint(private_numbers.q) + + +class _SSHFormatDSA: + """Format for DSA keys. + + Public: + mpint p, q, g, y + Private: + mpint p, q, g, y, x + """ + + def get_public(self, data: memoryview) -> tuple[tuple, memoryview]: + """DSA public fields""" + p, data = _get_mpint(data) + q, data = _get_mpint(data) + g, data = _get_mpint(data) + y, data = _get_mpint(data) + return (p, q, g, y), data + + def load_public( + self, data: memoryview + ) -> tuple[dsa.DSAPublicKey, memoryview]: + """Make DSA public key from data.""" + (p, q, g, y), data = self.get_public(data) + parameter_numbers = dsa.DSAParameterNumbers(p, q, g) + public_numbers = dsa.DSAPublicNumbers(y, parameter_numbers) + self._validate(public_numbers) + public_key = public_numbers.public_key() + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[dsa.DSAPrivateKey, memoryview]: + """Make DSA private key from data.""" + (p, q, g, y), data = self.get_public(data) + x, data = _get_mpint(data) + + if (p, q, g, y) != pubfields: + raise ValueError("Corrupt data: dsa field mismatch") + parameter_numbers = dsa.DSAParameterNumbers(p, q, g) + public_numbers = dsa.DSAPublicNumbers(y, parameter_numbers) + self._validate(public_numbers) + private_numbers = dsa.DSAPrivateNumbers(x, public_numbers) + private_key = private_numbers.private_key() + return private_key, data + + def encode_public( + self, public_key: dsa.DSAPublicKey, f_pub: _FragList + ) -> None: + """Write DSA public key""" + public_numbers = public_key.public_numbers() + parameter_numbers = public_numbers.parameter_numbers + self._validate(public_numbers) + + f_pub.put_mpint(parameter_numbers.p) + f_pub.put_mpint(parameter_numbers.q) + f_pub.put_mpint(parameter_numbers.g) + f_pub.put_mpint(public_numbers.y) + + def encode_private( + self, private_key: dsa.DSAPrivateKey, f_priv: _FragList + ) -> None: + """Write DSA private key""" + self.encode_public(private_key.public_key(), f_priv) + f_priv.put_mpint(private_key.private_numbers().x) + + def _validate(self, public_numbers: dsa.DSAPublicNumbers) -> None: + parameter_numbers = public_numbers.parameter_numbers + if parameter_numbers.p.bit_length() != 1024: + raise ValueError("SSH supports only 1024 bit DSA keys") + + +class _SSHFormatECDSA: + """Format for ECDSA keys. + + Public: + str curve + bytes point + Private: + str curve + bytes point + mpint secret + """ + + def __init__(self, ssh_curve_name: bytes, curve: ec.EllipticCurve): + self.ssh_curve_name = ssh_curve_name + self.curve = curve + + def get_public( + self, data: memoryview + ) -> tuple[tuple[memoryview, memoryview], memoryview]: + """ECDSA public fields""" + curve, data = _get_sshstr(data) + point, data = _get_sshstr(data) + if curve != self.ssh_curve_name: + raise ValueError("Curve name mismatch") + if point[0] != 4: + raise NotImplementedError("Need uncompressed point") + return (curve, point), data + + def load_public( + self, data: memoryview + ) -> tuple[ec.EllipticCurvePublicKey, memoryview]: + """Make ECDSA public key from data.""" + (_, point), data = self.get_public(data) + public_key = ec.EllipticCurvePublicKey.from_encoded_point( + self.curve, point.tobytes() + ) + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[ec.EllipticCurvePrivateKey, memoryview]: + """Make ECDSA private key from data.""" + (curve_name, point), data = self.get_public(data) + secret, data = _get_mpint(data) + + if (curve_name, point) != pubfields: + raise ValueError("Corrupt data: ecdsa field mismatch") + private_key = ec.derive_private_key(secret, self.curve) + return private_key, data + + def encode_public( + self, public_key: ec.EllipticCurvePublicKey, f_pub: _FragList + ) -> None: + """Write ECDSA public key""" + point = public_key.public_bytes( + Encoding.X962, PublicFormat.UncompressedPoint + ) + f_pub.put_sshstr(self.ssh_curve_name) + f_pub.put_sshstr(point) + + def encode_private( + self, private_key: ec.EllipticCurvePrivateKey, f_priv: _FragList + ) -> None: + """Write ECDSA private key""" + public_key = private_key.public_key() + private_numbers = private_key.private_numbers() + + self.encode_public(public_key, f_priv) + f_priv.put_mpint(private_numbers.private_value) + + +class _SSHFormatEd25519: + """Format for Ed25519 keys. + + Public: + bytes point + Private: + bytes point + bytes secret_and_point + """ + + def get_public( + self, data: memoryview + ) -> tuple[tuple[memoryview], memoryview]: + """Ed25519 public fields""" + point, data = _get_sshstr(data) + return (point,), data + + def load_public( + self, data: memoryview + ) -> tuple[ed25519.Ed25519PublicKey, memoryview]: + """Make Ed25519 public key from data.""" + (point,), data = self.get_public(data) + public_key = ed25519.Ed25519PublicKey.from_public_bytes( + point.tobytes() + ) + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[ed25519.Ed25519PrivateKey, memoryview]: + """Make Ed25519 private key from data.""" + (point,), data = self.get_public(data) + keypair, data = _get_sshstr(data) + + secret = keypair[:32] + point2 = keypair[32:] + if point != point2 or (point,) != pubfields: + raise ValueError("Corrupt data: ed25519 field mismatch") + private_key = ed25519.Ed25519PrivateKey.from_private_bytes(secret) + return private_key, data + + def encode_public( + self, public_key: ed25519.Ed25519PublicKey, f_pub: _FragList + ) -> None: + """Write Ed25519 public key""" + raw_public_key = public_key.public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + f_pub.put_sshstr(raw_public_key) + + def encode_private( + self, private_key: ed25519.Ed25519PrivateKey, f_priv: _FragList + ) -> None: + """Write Ed25519 private key""" + public_key = private_key.public_key() + raw_private_key = private_key.private_bytes( + Encoding.Raw, PrivateFormat.Raw, NoEncryption() + ) + raw_public_key = public_key.public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + f_keypair = _FragList([raw_private_key, raw_public_key]) + + self.encode_public(public_key, f_priv) + f_priv.put_sshstr(f_keypair) + + +def load_application(data) -> tuple[memoryview, memoryview]: + """ + U2F application strings + """ + application, data = _get_sshstr(data) + if not application.tobytes().startswith(b"ssh:"): + raise ValueError( + "U2F application string does not start with b'ssh:' " + f"({application})" + ) + return application, data + + +class _SSHFormatSKEd25519: + """ + The format of a sk-ssh-ed25519@openssh.com public key is: + + string "sk-ssh-ed25519@openssh.com" + string public key + string application (user-specified, but typically "ssh:") + """ + + def load_public( + self, data: memoryview + ) -> tuple[ed25519.Ed25519PublicKey, memoryview]: + """Make Ed25519 public key from data.""" + public_key, data = _lookup_kformat(_SSH_ED25519).load_public(data) + _, data = load_application(data) + return public_key, data + + def get_public(self, data: memoryview) -> typing.NoReturn: + # Confusingly `get_public` is an entry point used by private key + # loading. + raise UnsupportedAlgorithm( + "sk-ssh-ed25519 private keys cannot be loaded" + ) + + +class _SSHFormatSKECDSA: + """ + The format of a sk-ecdsa-sha2-nistp256@openssh.com public key is: + + string "sk-ecdsa-sha2-nistp256@openssh.com" + string curve name + ec_point Q + string application (user-specified, but typically "ssh:") + """ + + def load_public( + self, data: memoryview + ) -> tuple[ec.EllipticCurvePublicKey, memoryview]: + """Make ECDSA public key from data.""" + public_key, data = _lookup_kformat(_ECDSA_NISTP256).load_public(data) + _, data = load_application(data) + return public_key, data + + def get_public(self, data: memoryview) -> typing.NoReturn: + # Confusingly `get_public` is an entry point used by private key + # loading. + raise UnsupportedAlgorithm( + "sk-ecdsa-sha2-nistp256 private keys cannot be loaded" + ) + + +_KEY_FORMATS = { + _SSH_RSA: _SSHFormatRSA(), + _SSH_DSA: _SSHFormatDSA(), + _SSH_ED25519: _SSHFormatEd25519(), + _ECDSA_NISTP256: _SSHFormatECDSA(b"nistp256", ec.SECP256R1()), + _ECDSA_NISTP384: _SSHFormatECDSA(b"nistp384", ec.SECP384R1()), + _ECDSA_NISTP521: _SSHFormatECDSA(b"nistp521", ec.SECP521R1()), + _SK_SSH_ED25519: _SSHFormatSKEd25519(), + _SK_SSH_ECDSA_NISTP256: _SSHFormatSKECDSA(), +} + + +def _lookup_kformat(key_type: utils.Buffer): + """Return valid format or throw error""" + if not isinstance(key_type, bytes): + key_type = memoryview(key_type).tobytes() + if key_type in _KEY_FORMATS: + return _KEY_FORMATS[key_type] + raise UnsupportedAlgorithm(f"Unsupported key type: {key_type!r}") + + +SSHPrivateKeyTypes = typing.Union[ + ec.EllipticCurvePrivateKey, + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ed25519.Ed25519PrivateKey, +] + + +def load_ssh_private_key( + data: utils.Buffer, + password: bytes | None, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, +) -> SSHPrivateKeyTypes: + """Load private key from OpenSSH custom encoding.""" + utils._check_byteslike("data", data) + if password is not None: + utils._check_bytes("password", password) + + m = _PEM_RC.search(data) + if not m: + raise ValueError("Not OpenSSH private key format") + p1 = m.start(1) + p2 = m.end(1) + data = binascii.a2b_base64(memoryview(data)[p1:p2]) + if not data.startswith(_SK_MAGIC): + raise ValueError("Not OpenSSH private key format") + data = memoryview(data)[len(_SK_MAGIC) :] + + # parse header + ciphername, data = _get_sshstr(data) + kdfname, data = _get_sshstr(data) + kdfoptions, data = _get_sshstr(data) + nkeys, data = _get_u32(data) + if nkeys != 1: + raise ValueError("Only one key supported") + + # load public key data + pubdata, data = _get_sshstr(data) + pub_key_type, pubdata = _get_sshstr(pubdata) + kformat = _lookup_kformat(pub_key_type) + pubfields, pubdata = kformat.get_public(pubdata) + _check_empty(pubdata) + + if ciphername != _NONE or kdfname != _NONE: + ciphername_bytes = ciphername.tobytes() + if ciphername_bytes not in _SSH_CIPHERS: + raise UnsupportedAlgorithm( + f"Unsupported cipher: {ciphername_bytes!r}" + ) + if kdfname != _BCRYPT: + raise UnsupportedAlgorithm(f"Unsupported KDF: {kdfname!r}") + blklen = _SSH_CIPHERS[ciphername_bytes].block_len + tag_len = _SSH_CIPHERS[ciphername_bytes].tag_len + # load secret data + edata, data = _get_sshstr(data) + # see https://bugzilla.mindrot.org/show_bug.cgi?id=3553 for + # information about how OpenSSH handles AEAD tags + if _SSH_CIPHERS[ciphername_bytes].is_aead: + tag = bytes(data) + if len(tag) != tag_len: + raise ValueError("Corrupt data: invalid tag length for cipher") + else: + _check_empty(data) + _check_block_size(edata, blklen) + salt, kbuf = _get_sshstr(kdfoptions) + rounds, kbuf = _get_u32(kbuf) + _check_empty(kbuf) + ciph = _init_cipher(ciphername_bytes, password, salt.tobytes(), rounds) + dec = ciph.decryptor() + edata = memoryview(dec.update(edata)) + if _SSH_CIPHERS[ciphername_bytes].is_aead: + assert isinstance(dec, AEADDecryptionContext) + _check_empty(dec.finalize_with_tag(tag)) + else: + # _check_block_size requires data to be a full block so there + # should be no output from finalize + _check_empty(dec.finalize()) + else: + if password: + raise TypeError( + "Password was given but private key is not encrypted." + ) + # load secret data + edata, data = _get_sshstr(data) + _check_empty(data) + blklen = 8 + _check_block_size(edata, blklen) + ck1, edata = _get_u32(edata) + ck2, edata = _get_u32(edata) + if ck1 != ck2: + raise ValueError("Corrupt data: broken checksum") + + # load per-key struct + key_type, edata = _get_sshstr(edata) + if key_type != pub_key_type: + raise ValueError("Corrupt data: key type mismatch") + private_key, edata = kformat.load_private( + edata, + pubfields, + unsafe_skip_rsa_key_validation=unsafe_skip_rsa_key_validation, + ) + # We don't use the comment + _, edata = _get_sshstr(edata) + + # yes, SSH does padding check *after* all other parsing is done. + # need to follow as it writes zero-byte padding too. + if edata != _PADDING[: len(edata)]: + raise ValueError("Corrupt data: invalid padding") + + if isinstance(private_key, dsa.DSAPrivateKey): + warnings.warn( + "SSH DSA keys are deprecated and will be removed in a future " + "release.", + utils.DeprecatedIn40, + stacklevel=2, + ) + + return private_key + + +def _serialize_ssh_private_key( + private_key: SSHPrivateKeyTypes, + password: bytes, + encryption_algorithm: KeySerializationEncryption, +) -> bytes: + """Serialize private key with OpenSSH custom encoding.""" + utils._check_bytes("password", password) + if isinstance(private_key, dsa.DSAPrivateKey): + warnings.warn( + "SSH DSA key support is deprecated and will be " + "removed in a future release", + utils.DeprecatedIn40, + stacklevel=4, + ) + + key_type = _get_ssh_key_type(private_key) + kformat = _lookup_kformat(key_type) + + # setup parameters + f_kdfoptions = _FragList() + if password: + ciphername = _DEFAULT_CIPHER + blklen = _SSH_CIPHERS[ciphername].block_len + kdfname = _BCRYPT + rounds = _DEFAULT_ROUNDS + if ( + isinstance(encryption_algorithm, _KeySerializationEncryption) + and encryption_algorithm._kdf_rounds is not None + ): + rounds = encryption_algorithm._kdf_rounds + salt = os.urandom(16) + f_kdfoptions.put_sshstr(salt) + f_kdfoptions.put_u32(rounds) + ciph = _init_cipher(ciphername, password, salt, rounds) + else: + ciphername = kdfname = _NONE + blklen = 8 + ciph = None + nkeys = 1 + checkval = os.urandom(4) + comment = b"" + + # encode public and private parts together + f_public_key = _FragList() + f_public_key.put_sshstr(key_type) + kformat.encode_public(private_key.public_key(), f_public_key) + + f_secrets = _FragList([checkval, checkval]) + f_secrets.put_sshstr(key_type) + kformat.encode_private(private_key, f_secrets) + f_secrets.put_sshstr(comment) + f_secrets.put_raw(_PADDING[: blklen - (f_secrets.size() % blklen)]) + + # top-level structure + f_main = _FragList() + f_main.put_raw(_SK_MAGIC) + f_main.put_sshstr(ciphername) + f_main.put_sshstr(kdfname) + f_main.put_sshstr(f_kdfoptions) + f_main.put_u32(nkeys) + f_main.put_sshstr(f_public_key) + f_main.put_sshstr(f_secrets) + + # copy result info bytearray + slen = f_secrets.size() + mlen = f_main.size() + buf = memoryview(bytearray(mlen + blklen)) + f_main.render(buf) + ofs = mlen - slen + + # encrypt in-place + if ciph is not None: + ciph.encryptor().update_into(buf[ofs:mlen], buf[ofs:]) + + return _ssh_pem_encode(buf[:mlen]) + + +SSHPublicKeyTypes = typing.Union[ + ec.EllipticCurvePublicKey, + rsa.RSAPublicKey, + dsa.DSAPublicKey, + ed25519.Ed25519PublicKey, +] + +SSHCertPublicKeyTypes = typing.Union[ + ec.EllipticCurvePublicKey, + rsa.RSAPublicKey, + ed25519.Ed25519PublicKey, +] + + +class SSHCertificateType(enum.Enum): + USER = 1 + HOST = 2 + + +class SSHCertificate: + def __init__( + self, + _nonce: memoryview, + _public_key: SSHPublicKeyTypes, + _serial: int, + _cctype: int, + _key_id: memoryview, + _valid_principals: list[bytes], + _valid_after: int, + _valid_before: int, + _critical_options: dict[bytes, bytes], + _extensions: dict[bytes, bytes], + _sig_type: memoryview, + _sig_key: memoryview, + _inner_sig_type: memoryview, + _signature: memoryview, + _tbs_cert_body: memoryview, + _cert_key_type: bytes, + _cert_body: memoryview, + ): + self._nonce = _nonce + self._public_key = _public_key + self._serial = _serial + try: + self._type = SSHCertificateType(_cctype) + except ValueError: + raise ValueError("Invalid certificate type") + self._key_id = _key_id + self._valid_principals = _valid_principals + self._valid_after = _valid_after + self._valid_before = _valid_before + self._critical_options = _critical_options + self._extensions = _extensions + self._sig_type = _sig_type + self._sig_key = _sig_key + self._inner_sig_type = _inner_sig_type + self._signature = _signature + self._cert_key_type = _cert_key_type + self._cert_body = _cert_body + self._tbs_cert_body = _tbs_cert_body + + @property + def nonce(self) -> bytes: + return bytes(self._nonce) + + def public_key(self) -> SSHCertPublicKeyTypes: + # make mypy happy until we remove DSA support entirely and + # the underlying union won't have a disallowed type + return typing.cast(SSHCertPublicKeyTypes, self._public_key) + + @property + def serial(self) -> int: + return self._serial + + @property + def type(self) -> SSHCertificateType: + return self._type + + @property + def key_id(self) -> bytes: + return bytes(self._key_id) + + @property + def valid_principals(self) -> list[bytes]: + return self._valid_principals + + @property + def valid_before(self) -> int: + return self._valid_before + + @property + def valid_after(self) -> int: + return self._valid_after + + @property + def critical_options(self) -> dict[bytes, bytes]: + return self._critical_options + + @property + def extensions(self) -> dict[bytes, bytes]: + return self._extensions + + def signature_key(self) -> SSHCertPublicKeyTypes: + sigformat = _lookup_kformat(self._sig_type) + signature_key, sigkey_rest = sigformat.load_public(self._sig_key) + _check_empty(sigkey_rest) + return signature_key + + def public_bytes(self) -> bytes: + return ( + bytes(self._cert_key_type) + + b" " + + binascii.b2a_base64(bytes(self._cert_body), newline=False) + ) + + def verify_cert_signature(self) -> None: + signature_key = self.signature_key() + if isinstance(signature_key, ed25519.Ed25519PublicKey): + signature_key.verify( + bytes(self._signature), bytes(self._tbs_cert_body) + ) + elif isinstance(signature_key, ec.EllipticCurvePublicKey): + # The signature is encoded as a pair of big-endian integers + r, data = _get_mpint(self._signature) + s, data = _get_mpint(data) + _check_empty(data) + computed_sig = asym_utils.encode_dss_signature(r, s) + hash_alg = _get_ec_hash_alg(signature_key.curve) + signature_key.verify( + computed_sig, bytes(self._tbs_cert_body), ec.ECDSA(hash_alg) + ) + else: + assert isinstance(signature_key, rsa.RSAPublicKey) + if self._inner_sig_type == _SSH_RSA: + hash_alg = hashes.SHA1() + elif self._inner_sig_type == _SSH_RSA_SHA256: + hash_alg = hashes.SHA256() + else: + assert self._inner_sig_type == _SSH_RSA_SHA512 + hash_alg = hashes.SHA512() + signature_key.verify( + bytes(self._signature), + bytes(self._tbs_cert_body), + padding.PKCS1v15(), + hash_alg, + ) + + +def _get_ec_hash_alg(curve: ec.EllipticCurve) -> hashes.HashAlgorithm: + if isinstance(curve, ec.SECP256R1): + return hashes.SHA256() + elif isinstance(curve, ec.SECP384R1): + return hashes.SHA384() + else: + assert isinstance(curve, ec.SECP521R1) + return hashes.SHA512() + + +def _load_ssh_public_identity( + data: utils.Buffer, + _legacy_dsa_allowed=False, +) -> SSHCertificate | SSHPublicKeyTypes: + utils._check_byteslike("data", data) + + m = _SSH_PUBKEY_RC.match(data) + if not m: + raise ValueError("Invalid line format") + key_type = orig_key_type = m.group(1) + key_body = m.group(2) + with_cert = False + if key_type.endswith(_CERT_SUFFIX): + with_cert = True + key_type = key_type[: -len(_CERT_SUFFIX)] + if key_type == _SSH_DSA and not _legacy_dsa_allowed: + raise UnsupportedAlgorithm( + "DSA keys aren't supported in SSH certificates" + ) + kformat = _lookup_kformat(key_type) + + try: + rest = memoryview(binascii.a2b_base64(key_body)) + except (TypeError, binascii.Error): + raise ValueError("Invalid format") + + if with_cert: + cert_body = rest + inner_key_type, rest = _get_sshstr(rest) + if inner_key_type != orig_key_type: + raise ValueError("Invalid key format") + if with_cert: + nonce, rest = _get_sshstr(rest) + public_key, rest = kformat.load_public(rest) + if with_cert: + serial, rest = _get_u64(rest) + cctype, rest = _get_u32(rest) + key_id, rest = _get_sshstr(rest) + principals, rest = _get_sshstr(rest) + valid_principals = [] + while principals: + principal, principals = _get_sshstr(principals) + valid_principals.append(bytes(principal)) + valid_after, rest = _get_u64(rest) + valid_before, rest = _get_u64(rest) + crit_options, rest = _get_sshstr(rest) + critical_options = _parse_exts_opts(crit_options) + exts, rest = _get_sshstr(rest) + extensions = _parse_exts_opts(exts) + # Get the reserved field, which is unused. + _, rest = _get_sshstr(rest) + sig_key_raw, rest = _get_sshstr(rest) + sig_type, sig_key = _get_sshstr(sig_key_raw) + if sig_type == _SSH_DSA and not _legacy_dsa_allowed: + raise UnsupportedAlgorithm( + "DSA signatures aren't supported in SSH certificates" + ) + # Get the entire cert body and subtract the signature + tbs_cert_body = cert_body[: -len(rest)] + signature_raw, rest = _get_sshstr(rest) + _check_empty(rest) + inner_sig_type, sig_rest = _get_sshstr(signature_raw) + # RSA certs can have multiple algorithm types + if ( + sig_type == _SSH_RSA + and inner_sig_type + not in [_SSH_RSA_SHA256, _SSH_RSA_SHA512, _SSH_RSA] + ) or (sig_type != _SSH_RSA and inner_sig_type != sig_type): + raise ValueError("Signature key type does not match") + signature, sig_rest = _get_sshstr(sig_rest) + _check_empty(sig_rest) + return SSHCertificate( + nonce, + public_key, + serial, + cctype, + key_id, + valid_principals, + valid_after, + valid_before, + critical_options, + extensions, + sig_type, + sig_key, + inner_sig_type, + signature, + tbs_cert_body, + orig_key_type, + cert_body, + ) + else: + _check_empty(rest) + return public_key + + +def load_ssh_public_identity( + data: utils.Buffer, +) -> SSHCertificate | SSHPublicKeyTypes: + return _load_ssh_public_identity(data) + + +def _parse_exts_opts(exts_opts: memoryview) -> dict[bytes, bytes]: + result: dict[bytes, bytes] = {} + last_name = None + while exts_opts: + name, exts_opts = _get_sshstr(exts_opts) + bname: bytes = bytes(name) + if bname in result: + raise ValueError("Duplicate name") + if last_name is not None and bname < last_name: + raise ValueError("Fields not lexically sorted") + value, exts_opts = _get_sshstr(exts_opts) + if len(value) > 0: + value, extra = _get_sshstr(value) + if len(extra) > 0: + raise ValueError("Unexpected extra data after value") + result[bname] = bytes(value) + last_name = bname + return result + + +def ssh_key_fingerprint( + key: SSHPublicKeyTypes, + hash_algorithm: hashes.MD5 | hashes.SHA256, +) -> bytes: + if not isinstance(hash_algorithm, (hashes.MD5, hashes.SHA256)): + raise TypeError("hash_algorithm must be either MD5 or SHA256") + + key_type = _get_ssh_key_type(key) + kformat = _lookup_kformat(key_type) + + f_pub = _FragList() + f_pub.put_sshstr(key_type) + kformat.encode_public(key, f_pub) + + ssh_binary_data = f_pub.tobytes() + + # Hash the binary data + hash_obj = hashes.Hash(hash_algorithm) + hash_obj.update(ssh_binary_data) + return hash_obj.finalize() + + +def load_ssh_public_key( + data: utils.Buffer, backend: typing.Any = None +) -> SSHPublicKeyTypes: + cert_or_key = _load_ssh_public_identity(data, _legacy_dsa_allowed=True) + public_key: SSHPublicKeyTypes + if isinstance(cert_or_key, SSHCertificate): + public_key = cert_or_key.public_key() + else: + public_key = cert_or_key + + if isinstance(public_key, dsa.DSAPublicKey): + warnings.warn( + "SSH DSA keys are deprecated and will be removed in a future " + "release.", + utils.DeprecatedIn40, + stacklevel=2, + ) + return public_key + + +def serialize_ssh_public_key(public_key: SSHPublicKeyTypes) -> bytes: + """One-line public key format for OpenSSH""" + if isinstance(public_key, dsa.DSAPublicKey): + warnings.warn( + "SSH DSA key support is deprecated and will be " + "removed in a future release", + utils.DeprecatedIn40, + stacklevel=4, + ) + key_type = _get_ssh_key_type(public_key) + kformat = _lookup_kformat(key_type) + + f_pub = _FragList() + f_pub.put_sshstr(key_type) + kformat.encode_public(public_key, f_pub) + + pub = binascii.b2a_base64(f_pub.tobytes()).strip() + return b"".join([key_type, b" ", pub]) + + +SSHCertPrivateKeyTypes = typing.Union[ + ec.EllipticCurvePrivateKey, + rsa.RSAPrivateKey, + ed25519.Ed25519PrivateKey, +] + + +# This is an undocumented limit enforced in the openssh codebase for sshd and +# ssh-keygen, but it is undefined in the ssh certificates spec. +_SSHKEY_CERT_MAX_PRINCIPALS = 256 + + +class SSHCertificateBuilder: + def __init__( + self, + _public_key: SSHCertPublicKeyTypes | None = None, + _serial: int | None = None, + _type: SSHCertificateType | None = None, + _key_id: bytes | None = None, + _valid_principals: list[bytes] = [], + _valid_for_all_principals: bool = False, + _valid_before: int | None = None, + _valid_after: int | None = None, + _critical_options: list[tuple[bytes, bytes]] = [], + _extensions: list[tuple[bytes, bytes]] = [], + ): + self._public_key = _public_key + self._serial = _serial + self._type = _type + self._key_id = _key_id + self._valid_principals = _valid_principals + self._valid_for_all_principals = _valid_for_all_principals + self._valid_before = _valid_before + self._valid_after = _valid_after + self._critical_options = _critical_options + self._extensions = _extensions + + def public_key( + self, public_key: SSHCertPublicKeyTypes + ) -> SSHCertificateBuilder: + if not isinstance( + public_key, + ( + ec.EllipticCurvePublicKey, + rsa.RSAPublicKey, + ed25519.Ed25519PublicKey, + ), + ): + raise TypeError("Unsupported key type") + if self._public_key is not None: + raise ValueError("public_key already set") + + return SSHCertificateBuilder( + _public_key=public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def serial(self, serial: int) -> SSHCertificateBuilder: + if not isinstance(serial, int): + raise TypeError("serial must be an integer") + if not 0 <= serial < 2**64: + raise ValueError("serial must be between 0 and 2**64") + if self._serial is not None: + raise ValueError("serial already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def type(self, type: SSHCertificateType) -> SSHCertificateBuilder: + if not isinstance(type, SSHCertificateType): + raise TypeError("type must be an SSHCertificateType") + if self._type is not None: + raise ValueError("type already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def key_id(self, key_id: bytes) -> SSHCertificateBuilder: + if not isinstance(key_id, bytes): + raise TypeError("key_id must be bytes") + if self._key_id is not None: + raise ValueError("key_id already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_principals( + self, valid_principals: list[bytes] + ) -> SSHCertificateBuilder: + if self._valid_for_all_principals: + raise ValueError( + "Principals can't be set because the cert is valid " + "for all principals" + ) + if ( + not all(isinstance(x, bytes) for x in valid_principals) + or not valid_principals + ): + raise TypeError( + "principals must be a list of bytes and can't be empty" + ) + if self._valid_principals: + raise ValueError("valid_principals already set") + + if len(valid_principals) > _SSHKEY_CERT_MAX_PRINCIPALS: + raise ValueError( + "Reached or exceeded the maximum number of valid_principals" + ) + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_for_all_principals(self): + if self._valid_principals: + raise ValueError( + "valid_principals already set, can't set " + "valid_for_all_principals" + ) + if self._valid_for_all_principals: + raise ValueError("valid_for_all_principals already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=True, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_before(self, valid_before: int | float) -> SSHCertificateBuilder: + if not isinstance(valid_before, (int, float)): + raise TypeError("valid_before must be an int or float") + valid_before = int(valid_before) + if valid_before < 0 or valid_before >= 2**64: + raise ValueError("valid_before must [0, 2**64)") + if self._valid_before is not None: + raise ValueError("valid_before already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_after(self, valid_after: int | float) -> SSHCertificateBuilder: + if not isinstance(valid_after, (int, float)): + raise TypeError("valid_after must be an int or float") + valid_after = int(valid_after) + if valid_after < 0 or valid_after >= 2**64: + raise ValueError("valid_after must [0, 2**64)") + if self._valid_after is not None: + raise ValueError("valid_after already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def add_critical_option( + self, name: bytes, value: bytes + ) -> SSHCertificateBuilder: + if not isinstance(name, bytes) or not isinstance(value, bytes): + raise TypeError("name and value must be bytes") + # This is O(n**2) + if name in [name for name, _ in self._critical_options]: + raise ValueError("Duplicate critical option name") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=[*self._critical_options, (name, value)], + _extensions=self._extensions, + ) + + def add_extension( + self, name: bytes, value: bytes + ) -> SSHCertificateBuilder: + if not isinstance(name, bytes) or not isinstance(value, bytes): + raise TypeError("name and value must be bytes") + # This is O(n**2) + if name in [name for name, _ in self._extensions]: + raise ValueError("Duplicate extension name") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=[*self._extensions, (name, value)], + ) + + def sign(self, private_key: SSHCertPrivateKeyTypes) -> SSHCertificate: + if not isinstance( + private_key, + ( + ec.EllipticCurvePrivateKey, + rsa.RSAPrivateKey, + ed25519.Ed25519PrivateKey, + ), + ): + raise TypeError("Unsupported private key type") + + if self._public_key is None: + raise ValueError("public_key must be set") + + # Not required + serial = 0 if self._serial is None else self._serial + + if self._type is None: + raise ValueError("type must be set") + + # Not required + key_id = b"" if self._key_id is None else self._key_id + + # A zero length list is valid, but means the certificate + # is valid for any principal of the specified type. We require + # the user to explicitly set valid_for_all_principals to get + # that behavior. + if not self._valid_principals and not self._valid_for_all_principals: + raise ValueError( + "valid_principals must be set if valid_for_all_principals " + "is False" + ) + + if self._valid_before is None: + raise ValueError("valid_before must be set") + + if self._valid_after is None: + raise ValueError("valid_after must be set") + + if self._valid_after > self._valid_before: + raise ValueError("valid_after must be earlier than valid_before") + + # lexically sort our byte strings + self._critical_options.sort(key=lambda x: x[0]) + self._extensions.sort(key=lambda x: x[0]) + + key_type = _get_ssh_key_type(self._public_key) + cert_prefix = key_type + _CERT_SUFFIX + + # Marshal the bytes to be signed + nonce = os.urandom(32) + kformat = _lookup_kformat(key_type) + f = _FragList() + f.put_sshstr(cert_prefix) + f.put_sshstr(nonce) + kformat.encode_public(self._public_key, f) + f.put_u64(serial) + f.put_u32(self._type.value) + f.put_sshstr(key_id) + fprincipals = _FragList() + for p in self._valid_principals: + fprincipals.put_sshstr(p) + f.put_sshstr(fprincipals.tobytes()) + f.put_u64(self._valid_after) + f.put_u64(self._valid_before) + fcrit = _FragList() + for name, value in self._critical_options: + fcrit.put_sshstr(name) + if len(value) > 0: + foptval = _FragList() + foptval.put_sshstr(value) + fcrit.put_sshstr(foptval.tobytes()) + else: + fcrit.put_sshstr(value) + f.put_sshstr(fcrit.tobytes()) + fext = _FragList() + for name, value in self._extensions: + fext.put_sshstr(name) + if len(value) > 0: + fextval = _FragList() + fextval.put_sshstr(value) + fext.put_sshstr(fextval.tobytes()) + else: + fext.put_sshstr(value) + f.put_sshstr(fext.tobytes()) + f.put_sshstr(b"") # RESERVED FIELD + # encode CA public key + ca_type = _get_ssh_key_type(private_key) + caformat = _lookup_kformat(ca_type) + caf = _FragList() + caf.put_sshstr(ca_type) + caformat.encode_public(private_key.public_key(), caf) + f.put_sshstr(caf.tobytes()) + # Sigs according to the rules defined for the CA's public key + # (RFC4253 section 6.6 for ssh-rsa, RFC5656 for ECDSA, + # and RFC8032 for Ed25519). + if isinstance(private_key, ed25519.Ed25519PrivateKey): + signature = private_key.sign(f.tobytes()) + fsig = _FragList() + fsig.put_sshstr(ca_type) + fsig.put_sshstr(signature) + f.put_sshstr(fsig.tobytes()) + elif isinstance(private_key, ec.EllipticCurvePrivateKey): + hash_alg = _get_ec_hash_alg(private_key.curve) + signature = private_key.sign(f.tobytes(), ec.ECDSA(hash_alg)) + r, s = asym_utils.decode_dss_signature(signature) + fsig = _FragList() + fsig.put_sshstr(ca_type) + fsigblob = _FragList() + fsigblob.put_mpint(r) + fsigblob.put_mpint(s) + fsig.put_sshstr(fsigblob.tobytes()) + f.put_sshstr(fsig.tobytes()) + + else: + assert isinstance(private_key, rsa.RSAPrivateKey) + # Just like Golang, we're going to use SHA512 for RSA + # https://cs.opensource.google/go/x/crypto/+/refs/tags/ + # v0.4.0:ssh/certs.go;l=445 + # RFC 8332 defines SHA256 and 512 as options + fsig = _FragList() + fsig.put_sshstr(_SSH_RSA_SHA512) + signature = private_key.sign( + f.tobytes(), padding.PKCS1v15(), hashes.SHA512() + ) + fsig.put_sshstr(signature) + f.put_sshstr(fsig.tobytes()) + + cert_data = binascii.b2a_base64(f.tobytes()).strip() + # load_ssh_public_identity returns a union, but this is + # guaranteed to be an SSHCertificate, so we cast to make + # mypy happy. + return typing.cast( + SSHCertificate, + load_ssh_public_identity(b"".join([cert_prefix, b" ", cert_data])), + ) diff --git a/lib/cryptography/hazmat/primitives/twofactor/__init__.py b/lib/cryptography/hazmat/primitives/twofactor/__init__.py new file mode 100644 index 0000000..c1af423 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/twofactor/__init__.py @@ -0,0 +1,9 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + + +class InvalidToken(Exception): + pass diff --git a/lib/cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..07612e8 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-314.pyc b/lib/cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-314.pyc new file mode 100644 index 0000000..9b28836 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-314.pyc b/lib/cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-314.pyc new file mode 100644 index 0000000..7425b32 Binary files /dev/null and b/lib/cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-314.pyc differ diff --git a/lib/cryptography/hazmat/primitives/twofactor/hotp.py b/lib/cryptography/hazmat/primitives/twofactor/hotp.py new file mode 100644 index 0000000..21fb000 --- /dev/null +++ b/lib/cryptography/hazmat/primitives/twofactor/hotp.py @@ -0,0 +1,101 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import base64 +import typing +from urllib.parse import quote, urlencode + +from cryptography.hazmat.primitives import constant_time, hmac +from cryptography.hazmat.primitives.hashes import SHA1, SHA256, SHA512 +from cryptography.hazmat.primitives.twofactor import InvalidToken +from cryptography.utils import Buffer + +HOTPHashTypes = typing.Union[SHA1, SHA256, SHA512] + + +def _generate_uri( + hotp: HOTP, + type_name: str, + account_name: str, + issuer: str | None, + extra_parameters: list[tuple[str, int]], +) -> str: + parameters = [ + ("digits", hotp._length), + ("secret", base64.b32encode(hotp._key)), + ("algorithm", hotp._algorithm.name.upper()), + ] + + if issuer is not None: + parameters.append(("issuer", issuer)) + + parameters.extend(extra_parameters) + + label = ( + f"{quote(issuer)}:{quote(account_name)}" + if issuer + else quote(account_name) + ) + return f"otpauth://{type_name}/{label}?{urlencode(parameters)}" + + +class HOTP: + def __init__( + self, + key: Buffer, + length: int, + algorithm: HOTPHashTypes, + backend: typing.Any = None, + enforce_key_length: bool = True, + ) -> None: + if len(key) < 16 and enforce_key_length is True: + raise ValueError("Key length has to be at least 128 bits.") + + if not isinstance(length, int): + raise TypeError("Length parameter must be an integer type.") + + if length < 6 or length > 8: + raise ValueError("Length of HOTP has to be between 6 and 8.") + + if not isinstance(algorithm, (SHA1, SHA256, SHA512)): + raise TypeError("Algorithm must be SHA1, SHA256 or SHA512.") + + self._key = key + self._length = length + self._algorithm = algorithm + + def generate(self, counter: int) -> bytes: + if not isinstance(counter, int): + raise TypeError("Counter parameter must be an integer type.") + + truncated_value = self._dynamic_truncate(counter) + hotp = truncated_value % (10**self._length) + return "{0:0{1}}".format(hotp, self._length).encode() + + def verify(self, hotp: bytes, counter: int) -> None: + if not constant_time.bytes_eq(self.generate(counter), hotp): + raise InvalidToken("Supplied HOTP value does not match.") + + def _dynamic_truncate(self, counter: int) -> int: + ctx = hmac.HMAC(self._key, self._algorithm) + + try: + ctx.update(counter.to_bytes(length=8, byteorder="big")) + except OverflowError: + raise ValueError(f"Counter must be between 0 and {2**64 - 1}.") + + hmac_value = ctx.finalize() + + offset = hmac_value[len(hmac_value) - 1] & 0b1111 + p = hmac_value[offset : offset + 4] + return int.from_bytes(p, byteorder="big") & 0x7FFFFFFF + + def get_provisioning_uri( + self, account_name: str, counter: int, issuer: str | None + ) -> str: + return _generate_uri( + self, "hotp", account_name, issuer, [("counter", int(counter))] + ) diff --git a/lib/cryptography/hazmat/primitives/twofactor/totp.py b/lib/cryptography/hazmat/primitives/twofactor/totp.py new file mode 100644 index 0000000..10c725c --- /dev/null +++ b/lib/cryptography/hazmat/primitives/twofactor/totp.py @@ -0,0 +1,56 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.primitives import constant_time +from cryptography.hazmat.primitives.twofactor import InvalidToken +from cryptography.hazmat.primitives.twofactor.hotp import ( + HOTP, + HOTPHashTypes, + _generate_uri, +) +from cryptography.utils import Buffer + + +class TOTP: + def __init__( + self, + key: Buffer, + length: int, + algorithm: HOTPHashTypes, + time_step: int, + backend: typing.Any = None, + enforce_key_length: bool = True, + ): + self._time_step = time_step + self._hotp = HOTP( + key, length, algorithm, enforce_key_length=enforce_key_length + ) + + def generate(self, time: int | float) -> bytes: + if not isinstance(time, (int, float)): + raise TypeError( + "Time parameter must be an integer type or float type." + ) + + counter = int(time / self._time_step) + return self._hotp.generate(counter) + + def verify(self, totp: bytes, time: int) -> None: + if not constant_time.bytes_eq(self.generate(time), totp): + raise InvalidToken("Supplied TOTP value does not match.") + + def get_provisioning_uri( + self, account_name: str, issuer: str | None + ) -> str: + return _generate_uri( + self._hotp, + "totp", + account_name, + issuer, + [("period", int(self._time_step))], + ) diff --git a/lib/cryptography/py.typed b/lib/cryptography/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lib/cryptography/utils.py b/lib/cryptography/utils.py new file mode 100644 index 0000000..3a930fd --- /dev/null +++ b/lib/cryptography/utils.py @@ -0,0 +1,138 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import enum +import sys +import types +import typing +import warnings +from collections.abc import Callable, Sequence + + +# We use a UserWarning subclass, instead of DeprecationWarning, because CPython +# decided deprecation warnings should be invisible by default. +class CryptographyDeprecationWarning(UserWarning): + pass + + +# Several APIs were deprecated with no specific end-of-life date because of the +# ubiquity of their use. They should not be removed until we agree on when that +# cycle ends. +DeprecatedIn36 = CryptographyDeprecationWarning +DeprecatedIn40 = CryptographyDeprecationWarning +DeprecatedIn41 = CryptographyDeprecationWarning +DeprecatedIn42 = CryptographyDeprecationWarning +DeprecatedIn43 = CryptographyDeprecationWarning +DeprecatedIn46 = CryptographyDeprecationWarning + + +# If you're wondering why we don't use `Buffer`, it's because `Buffer` would +# be more accurately named: Bufferable. It means something which has an +# `__buffer__`. Which means you can't actually treat the result as a buffer +# (and do things like take a `len()`). +if sys.version_info >= (3, 9): + Buffer = typing.Union[bytes, bytearray, memoryview] +else: + Buffer = typing.ByteString + + +def _check_bytes(name: str, value: bytes) -> None: + if not isinstance(value, bytes): + raise TypeError(f"{name} must be bytes") + + +def _check_byteslike(name: str, value: Buffer) -> None: + try: + memoryview(value) + except TypeError: + raise TypeError(f"{name} must be bytes-like") + + +def int_to_bytes(integer: int, length: int | None = None) -> bytes: + if length == 0: + raise ValueError("length argument can't be 0") + return integer.to_bytes( + length or (integer.bit_length() + 7) // 8 or 1, "big" + ) + + +class InterfaceNotImplemented(Exception): + pass + + +class _DeprecatedValue: + def __init__(self, value: object, message: str, warning_class): + self.value = value + self.message = message + self.warning_class = warning_class + + +class _ModuleWithDeprecations(types.ModuleType): + def __init__(self, module: types.ModuleType): + super().__init__(module.__name__) + self.__dict__["_module"] = module + + def __getattr__(self, attr: str) -> object: + obj = getattr(self._module, attr) + if isinstance(obj, _DeprecatedValue): + warnings.warn(obj.message, obj.warning_class, stacklevel=2) + obj = obj.value + return obj + + def __setattr__(self, attr: str, value: object) -> None: + setattr(self._module, attr, value) + + def __delattr__(self, attr: str) -> None: + obj = getattr(self._module, attr) + if isinstance(obj, _DeprecatedValue): + warnings.warn(obj.message, obj.warning_class, stacklevel=2) + + delattr(self._module, attr) + + def __dir__(self) -> Sequence[str]: + return ["_module", *dir(self._module)] + + +def deprecated( + value: object, + module_name: str, + message: str, + warning_class: type[Warning], + name: str | None = None, +) -> _DeprecatedValue: + module = sys.modules[module_name] + if not isinstance(module, _ModuleWithDeprecations): + sys.modules[module_name] = module = _ModuleWithDeprecations(module) + dv = _DeprecatedValue(value, message, warning_class) + # Maintain backwards compatibility with `name is None` for pyOpenSSL. + if name is not None: + setattr(module, name, dv) + return dv + + +def cached_property(func: Callable) -> property: + cached_name = f"_cached_{func}" + sentinel = object() + + def inner(instance: object): + cache = getattr(instance, cached_name, sentinel) + if cache is not sentinel: + return cache + result = func(instance) + setattr(instance, cached_name, result) + return result + + return property(inner) + + +# Python 3.10 changed representation of enums. We use well-defined object +# representation and string representation from Python 3.9. +class Enum(enum.Enum): + def __repr__(self) -> str: + return f"<{self.__class__.__name__}.{self._name_}: {self._value_!r}>" + + def __str__(self) -> str: + return f"{self.__class__.__name__}.{self._name_}" diff --git a/lib/cryptography/x509/__init__.py b/lib/cryptography/x509/__init__.py new file mode 100644 index 0000000..318eecc --- /dev/null +++ b/lib/cryptography/x509/__init__.py @@ -0,0 +1,270 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.x509 import certificate_transparency, verification +from cryptography.x509.base import ( + Attribute, + AttributeNotFound, + Attributes, + Certificate, + CertificateBuilder, + CertificateRevocationList, + CertificateRevocationListBuilder, + CertificateSigningRequest, + CertificateSigningRequestBuilder, + InvalidVersion, + RevokedCertificate, + RevokedCertificateBuilder, + Version, + load_der_x509_certificate, + load_der_x509_crl, + load_der_x509_csr, + load_pem_x509_certificate, + load_pem_x509_certificates, + load_pem_x509_crl, + load_pem_x509_csr, + random_serial_number, +) +from cryptography.x509.extensions import ( + AccessDescription, + Admission, + Admissions, + AuthorityInformationAccess, + AuthorityKeyIdentifier, + BasicConstraints, + CertificateIssuer, + CertificatePolicies, + CRLDistributionPoints, + CRLNumber, + CRLReason, + DeltaCRLIndicator, + DistributionPoint, + DuplicateExtension, + ExtendedKeyUsage, + Extension, + ExtensionNotFound, + Extensions, + ExtensionType, + FreshestCRL, + GeneralNames, + InhibitAnyPolicy, + InvalidityDate, + IssuerAlternativeName, + IssuingDistributionPoint, + KeyUsage, + MSCertificateTemplate, + NameConstraints, + NamingAuthority, + NoticeReference, + OCSPAcceptableResponses, + OCSPNoCheck, + OCSPNonce, + PolicyConstraints, + PolicyInformation, + PrecertificateSignedCertificateTimestamps, + PrecertPoison, + PrivateKeyUsagePeriod, + ProfessionInfo, + ReasonFlags, + SignedCertificateTimestamps, + SubjectAlternativeName, + SubjectInformationAccess, + SubjectKeyIdentifier, + TLSFeature, + TLSFeatureType, + UnrecognizedExtension, + UserNotice, +) +from cryptography.x509.general_name import ( + DirectoryName, + DNSName, + GeneralName, + IPAddress, + OtherName, + RegisteredID, + RFC822Name, + UniformResourceIdentifier, + UnsupportedGeneralNameType, +) +from cryptography.x509.name import ( + Name, + NameAttribute, + RelativeDistinguishedName, +) +from cryptography.x509.oid import ( + AuthorityInformationAccessOID, + CertificatePoliciesOID, + CRLEntryExtensionOID, + ExtendedKeyUsageOID, + ExtensionOID, + NameOID, + ObjectIdentifier, + PublicKeyAlgorithmOID, + SignatureAlgorithmOID, +) + +OID_AUTHORITY_INFORMATION_ACCESS = ExtensionOID.AUTHORITY_INFORMATION_ACCESS +OID_AUTHORITY_KEY_IDENTIFIER = ExtensionOID.AUTHORITY_KEY_IDENTIFIER +OID_BASIC_CONSTRAINTS = ExtensionOID.BASIC_CONSTRAINTS +OID_CERTIFICATE_POLICIES = ExtensionOID.CERTIFICATE_POLICIES +OID_CRL_DISTRIBUTION_POINTS = ExtensionOID.CRL_DISTRIBUTION_POINTS +OID_EXTENDED_KEY_USAGE = ExtensionOID.EXTENDED_KEY_USAGE +OID_FRESHEST_CRL = ExtensionOID.FRESHEST_CRL +OID_INHIBIT_ANY_POLICY = ExtensionOID.INHIBIT_ANY_POLICY +OID_ISSUER_ALTERNATIVE_NAME = ExtensionOID.ISSUER_ALTERNATIVE_NAME +OID_KEY_USAGE = ExtensionOID.KEY_USAGE +OID_PRIVATE_KEY_USAGE_PERIOD = ExtensionOID.PRIVATE_KEY_USAGE_PERIOD +OID_NAME_CONSTRAINTS = ExtensionOID.NAME_CONSTRAINTS +OID_OCSP_NO_CHECK = ExtensionOID.OCSP_NO_CHECK +OID_POLICY_CONSTRAINTS = ExtensionOID.POLICY_CONSTRAINTS +OID_POLICY_MAPPINGS = ExtensionOID.POLICY_MAPPINGS +OID_SUBJECT_ALTERNATIVE_NAME = ExtensionOID.SUBJECT_ALTERNATIVE_NAME +OID_SUBJECT_DIRECTORY_ATTRIBUTES = ExtensionOID.SUBJECT_DIRECTORY_ATTRIBUTES +OID_SUBJECT_INFORMATION_ACCESS = ExtensionOID.SUBJECT_INFORMATION_ACCESS +OID_SUBJECT_KEY_IDENTIFIER = ExtensionOID.SUBJECT_KEY_IDENTIFIER + +OID_DSA_WITH_SHA1 = SignatureAlgorithmOID.DSA_WITH_SHA1 +OID_DSA_WITH_SHA224 = SignatureAlgorithmOID.DSA_WITH_SHA224 +OID_DSA_WITH_SHA256 = SignatureAlgorithmOID.DSA_WITH_SHA256 +OID_ECDSA_WITH_SHA1 = SignatureAlgorithmOID.ECDSA_WITH_SHA1 +OID_ECDSA_WITH_SHA224 = SignatureAlgorithmOID.ECDSA_WITH_SHA224 +OID_ECDSA_WITH_SHA256 = SignatureAlgorithmOID.ECDSA_WITH_SHA256 +OID_ECDSA_WITH_SHA384 = SignatureAlgorithmOID.ECDSA_WITH_SHA384 +OID_ECDSA_WITH_SHA512 = SignatureAlgorithmOID.ECDSA_WITH_SHA512 +OID_RSA_WITH_MD5 = SignatureAlgorithmOID.RSA_WITH_MD5 +OID_RSA_WITH_SHA1 = SignatureAlgorithmOID.RSA_WITH_SHA1 +OID_RSA_WITH_SHA224 = SignatureAlgorithmOID.RSA_WITH_SHA224 +OID_RSA_WITH_SHA256 = SignatureAlgorithmOID.RSA_WITH_SHA256 +OID_RSA_WITH_SHA384 = SignatureAlgorithmOID.RSA_WITH_SHA384 +OID_RSA_WITH_SHA512 = SignatureAlgorithmOID.RSA_WITH_SHA512 +OID_RSASSA_PSS = SignatureAlgorithmOID.RSASSA_PSS + +OID_COMMON_NAME = NameOID.COMMON_NAME +OID_COUNTRY_NAME = NameOID.COUNTRY_NAME +OID_DOMAIN_COMPONENT = NameOID.DOMAIN_COMPONENT +OID_DN_QUALIFIER = NameOID.DN_QUALIFIER +OID_EMAIL_ADDRESS = NameOID.EMAIL_ADDRESS +OID_GENERATION_QUALIFIER = NameOID.GENERATION_QUALIFIER +OID_GIVEN_NAME = NameOID.GIVEN_NAME +OID_LOCALITY_NAME = NameOID.LOCALITY_NAME +OID_ORGANIZATIONAL_UNIT_NAME = NameOID.ORGANIZATIONAL_UNIT_NAME +OID_ORGANIZATION_NAME = NameOID.ORGANIZATION_NAME +OID_PSEUDONYM = NameOID.PSEUDONYM +OID_SERIAL_NUMBER = NameOID.SERIAL_NUMBER +OID_STATE_OR_PROVINCE_NAME = NameOID.STATE_OR_PROVINCE_NAME +OID_SURNAME = NameOID.SURNAME +OID_TITLE = NameOID.TITLE + +OID_CLIENT_AUTH = ExtendedKeyUsageOID.CLIENT_AUTH +OID_CODE_SIGNING = ExtendedKeyUsageOID.CODE_SIGNING +OID_EMAIL_PROTECTION = ExtendedKeyUsageOID.EMAIL_PROTECTION +OID_OCSP_SIGNING = ExtendedKeyUsageOID.OCSP_SIGNING +OID_SERVER_AUTH = ExtendedKeyUsageOID.SERVER_AUTH +OID_TIME_STAMPING = ExtendedKeyUsageOID.TIME_STAMPING + +OID_ANY_POLICY = CertificatePoliciesOID.ANY_POLICY +OID_CPS_QUALIFIER = CertificatePoliciesOID.CPS_QUALIFIER +OID_CPS_USER_NOTICE = CertificatePoliciesOID.CPS_USER_NOTICE + +OID_CERTIFICATE_ISSUER = CRLEntryExtensionOID.CERTIFICATE_ISSUER +OID_CRL_REASON = CRLEntryExtensionOID.CRL_REASON +OID_INVALIDITY_DATE = CRLEntryExtensionOID.INVALIDITY_DATE + +OID_CA_ISSUERS = AuthorityInformationAccessOID.CA_ISSUERS +OID_OCSP = AuthorityInformationAccessOID.OCSP + +__all__ = [ + "OID_CA_ISSUERS", + "OID_OCSP", + "AccessDescription", + "Admission", + "Admissions", + "Attribute", + "AttributeNotFound", + "Attributes", + "AuthorityInformationAccess", + "AuthorityKeyIdentifier", + "BasicConstraints", + "CRLDistributionPoints", + "CRLNumber", + "CRLReason", + "Certificate", + "CertificateBuilder", + "CertificateIssuer", + "CertificatePolicies", + "CertificateRevocationList", + "CertificateRevocationListBuilder", + "CertificateSigningRequest", + "CertificateSigningRequestBuilder", + "DNSName", + "DeltaCRLIndicator", + "DirectoryName", + "DistributionPoint", + "DuplicateExtension", + "ExtendedKeyUsage", + "Extension", + "ExtensionNotFound", + "ExtensionType", + "Extensions", + "FreshestCRL", + "GeneralName", + "GeneralNames", + "IPAddress", + "InhibitAnyPolicy", + "InvalidVersion", + "InvalidityDate", + "IssuerAlternativeName", + "IssuingDistributionPoint", + "KeyUsage", + "MSCertificateTemplate", + "Name", + "NameAttribute", + "NameConstraints", + "NameOID", + "NamingAuthority", + "NoticeReference", + "OCSPAcceptableResponses", + "OCSPNoCheck", + "OCSPNonce", + "ObjectIdentifier", + "OtherName", + "PolicyConstraints", + "PolicyInformation", + "PrecertPoison", + "PrecertificateSignedCertificateTimestamps", + "PrivateKeyUsagePeriod", + "ProfessionInfo", + "PublicKeyAlgorithmOID", + "RFC822Name", + "ReasonFlags", + "RegisteredID", + "RelativeDistinguishedName", + "RevokedCertificate", + "RevokedCertificateBuilder", + "SignatureAlgorithmOID", + "SignedCertificateTimestamps", + "SubjectAlternativeName", + "SubjectInformationAccess", + "SubjectKeyIdentifier", + "TLSFeature", + "TLSFeatureType", + "UniformResourceIdentifier", + "UnrecognizedExtension", + "UnsupportedGeneralNameType", + "UserNotice", + "Version", + "certificate_transparency", + "load_der_x509_certificate", + "load_der_x509_crl", + "load_der_x509_csr", + "load_pem_x509_certificate", + "load_pem_x509_certificates", + "load_pem_x509_crl", + "load_pem_x509_csr", + "random_serial_number", + "verification", + "verification", +] diff --git a/lib/cryptography/x509/__pycache__/__init__.cpython-314.pyc b/lib/cryptography/x509/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..373f69e Binary files /dev/null and b/lib/cryptography/x509/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/base.cpython-314.pyc b/lib/cryptography/x509/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..617cc1e Binary files /dev/null and b/lib/cryptography/x509/__pycache__/base.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/certificate_transparency.cpython-314.pyc b/lib/cryptography/x509/__pycache__/certificate_transparency.cpython-314.pyc new file mode 100644 index 0000000..63f148a Binary files /dev/null and b/lib/cryptography/x509/__pycache__/certificate_transparency.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/extensions.cpython-314.pyc b/lib/cryptography/x509/__pycache__/extensions.cpython-314.pyc new file mode 100644 index 0000000..2b741a8 Binary files /dev/null and b/lib/cryptography/x509/__pycache__/extensions.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/general_name.cpython-314.pyc b/lib/cryptography/x509/__pycache__/general_name.cpython-314.pyc new file mode 100644 index 0000000..40a0b45 Binary files /dev/null and b/lib/cryptography/x509/__pycache__/general_name.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/name.cpython-314.pyc b/lib/cryptography/x509/__pycache__/name.cpython-314.pyc new file mode 100644 index 0000000..8c44b14 Binary files /dev/null and b/lib/cryptography/x509/__pycache__/name.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/ocsp.cpython-314.pyc b/lib/cryptography/x509/__pycache__/ocsp.cpython-314.pyc new file mode 100644 index 0000000..fefc3e2 Binary files /dev/null and b/lib/cryptography/x509/__pycache__/ocsp.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/oid.cpython-314.pyc b/lib/cryptography/x509/__pycache__/oid.cpython-314.pyc new file mode 100644 index 0000000..266a45a Binary files /dev/null and b/lib/cryptography/x509/__pycache__/oid.cpython-314.pyc differ diff --git a/lib/cryptography/x509/__pycache__/verification.cpython-314.pyc b/lib/cryptography/x509/__pycache__/verification.cpython-314.pyc new file mode 100644 index 0000000..9d0e1e6 Binary files /dev/null and b/lib/cryptography/x509/__pycache__/verification.cpython-314.pyc differ diff --git a/lib/cryptography/x509/base.py b/lib/cryptography/x509/base.py new file mode 100644 index 0000000..1be612b --- /dev/null +++ b/lib/cryptography/x509/base.py @@ -0,0 +1,848 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import datetime +import os +import typing +import warnings +from collections.abc import Iterable + +from cryptography import utils +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed448, + ed25519, + padding, + rsa, + x448, + x25519, +) +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPrivateKeyTypes, + CertificatePublicKeyTypes, +) +from cryptography.x509.extensions import ( + Extension, + Extensions, + ExtensionType, + _make_sequence_methods, +) +from cryptography.x509.name import Name, _ASN1Type +from cryptography.x509.oid import ObjectIdentifier + +_EARLIEST_UTC_TIME = datetime.datetime(1950, 1, 1) + +# This must be kept in sync with sign.rs's list of allowable types in +# identify_hash_type +_AllowedHashTypes = typing.Union[ + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA3_224, + hashes.SHA3_256, + hashes.SHA3_384, + hashes.SHA3_512, +] + + +class AttributeNotFound(Exception): + def __init__(self, msg: str, oid: ObjectIdentifier) -> None: + super().__init__(msg) + self.oid = oid + + +def _reject_duplicate_extension( + extension: Extension[ExtensionType], + extensions: list[Extension[ExtensionType]], +) -> None: + # This is quadratic in the number of extensions + for e in extensions: + if e.oid == extension.oid: + raise ValueError("This extension has already been set.") + + +def _reject_duplicate_attribute( + oid: ObjectIdentifier, + attributes: list[tuple[ObjectIdentifier, bytes, int | None]], +) -> None: + # This is quadratic in the number of attributes + for attr_oid, _, _ in attributes: + if attr_oid == oid: + raise ValueError("This attribute has already been set.") + + +def _convert_to_naive_utc_time(time: datetime.datetime) -> datetime.datetime: + """Normalizes a datetime to a naive datetime in UTC. + + time -- datetime to normalize. Assumed to be in UTC if not timezone + aware. + """ + if time.tzinfo is not None: + offset = time.utcoffset() + offset = offset if offset else datetime.timedelta() + return time.replace(tzinfo=None) - offset + else: + return time + + +class Attribute: + def __init__( + self, + oid: ObjectIdentifier, + value: bytes, + _type: int = _ASN1Type.UTF8String.value, + ) -> None: + self._oid = oid + self._value = value + self._type = _type + + @property + def oid(self) -> ObjectIdentifier: + return self._oid + + @property + def value(self) -> bytes: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Attribute): + return NotImplemented + + return ( + self.oid == other.oid + and self.value == other.value + and self._type == other._type + ) + + def __hash__(self) -> int: + return hash((self.oid, self.value, self._type)) + + +class Attributes: + def __init__( + self, + attributes: Iterable[Attribute], + ) -> None: + self._attributes = list(attributes) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_attributes") + + def __repr__(self) -> str: + return f"" + + def get_attribute_for_oid(self, oid: ObjectIdentifier) -> Attribute: + for attr in self: + if attr.oid == oid: + return attr + + raise AttributeNotFound(f"No {oid} attribute was found", oid) + + +class Version(utils.Enum): + v1 = 0 + v3 = 2 + + +class InvalidVersion(Exception): + def __init__(self, msg: str, parsed_version: int) -> None: + super().__init__(msg) + self.parsed_version = parsed_version + + +Certificate = rust_x509.Certificate + + +class RevokedCertificate(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def serial_number(self) -> int: + """ + Returns the serial number of the revoked certificate. + """ + + @property + @abc.abstractmethod + def revocation_date(self) -> datetime.datetime: + """ + Returns the date of when this certificate was revoked. + """ + + @property + @abc.abstractmethod + def revocation_date_utc(self) -> datetime.datetime: + """ + Returns the date of when this certificate was revoked as a non-naive + UTC datetime. + """ + + @property + @abc.abstractmethod + def extensions(self) -> Extensions: + """ + Returns an Extensions object containing a list of Revoked extensions. + """ + + +# Runtime isinstance checks need this since the rust class is not a subclass. +RevokedCertificate.register(rust_x509.RevokedCertificate) + + +class _RawRevokedCertificate(RevokedCertificate): + def __init__( + self, + serial_number: int, + revocation_date: datetime.datetime, + extensions: Extensions, + ): + self._serial_number = serial_number + self._revocation_date = revocation_date + self._extensions = extensions + + @property + def serial_number(self) -> int: + return self._serial_number + + @property + def revocation_date(self) -> datetime.datetime: + warnings.warn( + "Properties that return a naïve datetime object have been " + "deprecated. Please switch to revocation_date_utc.", + utils.DeprecatedIn42, + stacklevel=2, + ) + return self._revocation_date + + @property + def revocation_date_utc(self) -> datetime.datetime: + return self._revocation_date.replace(tzinfo=datetime.timezone.utc) + + @property + def extensions(self) -> Extensions: + return self._extensions + + +CertificateRevocationList = rust_x509.CertificateRevocationList +CertificateSigningRequest = rust_x509.CertificateSigningRequest + + +load_pem_x509_certificate = rust_x509.load_pem_x509_certificate +load_der_x509_certificate = rust_x509.load_der_x509_certificate + +load_pem_x509_certificates = rust_x509.load_pem_x509_certificates + +load_pem_x509_csr = rust_x509.load_pem_x509_csr +load_der_x509_csr = rust_x509.load_der_x509_csr + +load_pem_x509_crl = rust_x509.load_pem_x509_crl +load_der_x509_crl = rust_x509.load_der_x509_crl + + +class CertificateSigningRequestBuilder: + def __init__( + self, + subject_name: Name | None = None, + extensions: list[Extension[ExtensionType]] = [], + attributes: list[tuple[ObjectIdentifier, bytes, int | None]] = [], + ): + """ + Creates an empty X.509 certificate request (v1). + """ + self._subject_name = subject_name + self._extensions = extensions + self._attributes = attributes + + def subject_name(self, name: Name) -> CertificateSigningRequestBuilder: + """ + Sets the certificate requestor's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._subject_name is not None: + raise ValueError("The subject name may only be set once.") + return CertificateSigningRequestBuilder( + name, self._extensions, self._attributes + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> CertificateSigningRequestBuilder: + """ + Adds an X.509 extension to the certificate request. + """ + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return CertificateSigningRequestBuilder( + self._subject_name, + [*self._extensions, extension], + self._attributes, + ) + + def add_attribute( + self, + oid: ObjectIdentifier, + value: bytes, + *, + _tag: _ASN1Type | None = None, + ) -> CertificateSigningRequestBuilder: + """ + Adds an X.509 attribute with an OID and associated value. + """ + if not isinstance(oid, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + + if not isinstance(value, bytes): + raise TypeError("value must be bytes") + + if _tag is not None and not isinstance(_tag, _ASN1Type): + raise TypeError("tag must be _ASN1Type") + + _reject_duplicate_attribute(oid, self._attributes) + + if _tag is not None: + tag = _tag.value + else: + tag = None + + return CertificateSigningRequestBuilder( + self._subject_name, + self._extensions, + [*self._attributes, (oid, value, tag)], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: _AllowedHashTypes | None, + backend: typing.Any = None, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ecdsa_deterministic: bool | None = None, + ) -> CertificateSigningRequest: + """ + Signs the request using the requestor's private key. + """ + if self._subject_name is None: + raise ValueError("A CertificateSigningRequest must have a subject") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + if ecdsa_deterministic is not None: + if not isinstance(private_key, ec.EllipticCurvePrivateKey): + raise TypeError( + "Deterministic ECDSA is only supported for EC keys" + ) + + return rust_x509.create_x509_csr( + self, + private_key, + algorithm, + rsa_padding, + ecdsa_deterministic, + ) + + +class CertificateBuilder: + _extensions: list[Extension[ExtensionType]] + + def __init__( + self, + issuer_name: Name | None = None, + subject_name: Name | None = None, + public_key: CertificatePublicKeyTypes | None = None, + serial_number: int | None = None, + not_valid_before: datetime.datetime | None = None, + not_valid_after: datetime.datetime | None = None, + extensions: list[Extension[ExtensionType]] = [], + ) -> None: + self._version = Version.v3 + self._issuer_name = issuer_name + self._subject_name = subject_name + self._public_key = public_key + self._serial_number = serial_number + self._not_valid_before = not_valid_before + self._not_valid_after = not_valid_after + self._extensions = extensions + + def issuer_name(self, name: Name) -> CertificateBuilder: + """ + Sets the CA's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._issuer_name is not None: + raise ValueError("The issuer name may only be set once.") + return CertificateBuilder( + name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def subject_name(self, name: Name) -> CertificateBuilder: + """ + Sets the requestor's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._subject_name is not None: + raise ValueError("The subject name may only be set once.") + return CertificateBuilder( + self._issuer_name, + name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def public_key( + self, + key: CertificatePublicKeyTypes, + ) -> CertificateBuilder: + """ + Sets the requestor's public key (as found in the signing request). + """ + if not isinstance( + key, + ( + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + x25519.X25519PublicKey, + x448.X448PublicKey, + ), + ): + raise TypeError( + "Expecting one of DSAPublicKey, RSAPublicKey," + " EllipticCurvePublicKey, Ed25519PublicKey," + " Ed448PublicKey, X25519PublicKey, or " + "X448PublicKey." + ) + if self._public_key is not None: + raise ValueError("The public key may only be set once.") + return CertificateBuilder( + self._issuer_name, + self._subject_name, + key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def serial_number(self, number: int) -> CertificateBuilder: + """ + Sets the certificate serial number. + """ + if not isinstance(number, int): + raise TypeError("Serial number must be of integral type.") + if self._serial_number is not None: + raise ValueError("The serial number may only be set once.") + if number <= 0: + raise ValueError("The serial number should be positive.") + + # ASN.1 integers are always signed, so most significant bit must be + # zero. + if number.bit_length() >= 160: # As defined in RFC 5280 + raise ValueError( + "The serial number should not be more than 159 bits." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def not_valid_before(self, time: datetime.datetime) -> CertificateBuilder: + """ + Sets the certificate activation time. + """ + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._not_valid_before is not None: + raise ValueError("The not valid before may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The not valid before date must be on or after" + " 1950 January 1)." + ) + if self._not_valid_after is not None and time > self._not_valid_after: + raise ValueError( + "The not valid before date must be before the not valid after " + "date." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + time, + self._not_valid_after, + self._extensions, + ) + + def not_valid_after(self, time: datetime.datetime) -> CertificateBuilder: + """ + Sets the certificate expiration time. + """ + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._not_valid_after is not None: + raise ValueError("The not valid after may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The not valid after date must be on or after 1950 January 1." + ) + if ( + self._not_valid_before is not None + and time < self._not_valid_before + ): + raise ValueError( + "The not valid after date must be after the not valid before " + "date." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + time, + self._extensions, + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> CertificateBuilder: + """ + Adds an X.509 extension to the certificate. + """ + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + [*self._extensions, extension], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: _AllowedHashTypes | None, + backend: typing.Any = None, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ecdsa_deterministic: bool | None = None, + ) -> Certificate: + """ + Signs the certificate using the CA's private key. + """ + if self._subject_name is None: + raise ValueError("A certificate must have a subject name") + + if self._issuer_name is None: + raise ValueError("A certificate must have an issuer name") + + if self._serial_number is None: + raise ValueError("A certificate must have a serial number") + + if self._not_valid_before is None: + raise ValueError("A certificate must have a not valid before time") + + if self._not_valid_after is None: + raise ValueError("A certificate must have a not valid after time") + + if self._public_key is None: + raise ValueError("A certificate must have a public key") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + if ecdsa_deterministic is not None: + if not isinstance(private_key, ec.EllipticCurvePrivateKey): + raise TypeError( + "Deterministic ECDSA is only supported for EC keys" + ) + + return rust_x509.create_x509_certificate( + self, + private_key, + algorithm, + rsa_padding, + ecdsa_deterministic, + ) + + +class CertificateRevocationListBuilder: + _extensions: list[Extension[ExtensionType]] + _revoked_certificates: list[RevokedCertificate] + + def __init__( + self, + issuer_name: Name | None = None, + last_update: datetime.datetime | None = None, + next_update: datetime.datetime | None = None, + extensions: list[Extension[ExtensionType]] = [], + revoked_certificates: list[RevokedCertificate] = [], + ): + self._issuer_name = issuer_name + self._last_update = last_update + self._next_update = next_update + self._extensions = extensions + self._revoked_certificates = revoked_certificates + + def issuer_name( + self, issuer_name: Name + ) -> CertificateRevocationListBuilder: + if not isinstance(issuer_name, Name): + raise TypeError("Expecting x509.Name object.") + if self._issuer_name is not None: + raise ValueError("The issuer name may only be set once.") + return CertificateRevocationListBuilder( + issuer_name, + self._last_update, + self._next_update, + self._extensions, + self._revoked_certificates, + ) + + def last_update( + self, last_update: datetime.datetime + ) -> CertificateRevocationListBuilder: + if not isinstance(last_update, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._last_update is not None: + raise ValueError("Last update may only be set once.") + last_update = _convert_to_naive_utc_time(last_update) + if last_update < _EARLIEST_UTC_TIME: + raise ValueError( + "The last update date must be on or after 1950 January 1." + ) + if self._next_update is not None and last_update > self._next_update: + raise ValueError( + "The last update date must be before the next update date." + ) + return CertificateRevocationListBuilder( + self._issuer_name, + last_update, + self._next_update, + self._extensions, + self._revoked_certificates, + ) + + def next_update( + self, next_update: datetime.datetime + ) -> CertificateRevocationListBuilder: + if not isinstance(next_update, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._next_update is not None: + raise ValueError("Last update may only be set once.") + next_update = _convert_to_naive_utc_time(next_update) + if next_update < _EARLIEST_UTC_TIME: + raise ValueError( + "The last update date must be on or after 1950 January 1." + ) + if self._last_update is not None and next_update < self._last_update: + raise ValueError( + "The next update date must be after the last update date." + ) + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + next_update, + self._extensions, + self._revoked_certificates, + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> CertificateRevocationListBuilder: + """ + Adds an X.509 extension to the certificate revocation list. + """ + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + self._next_update, + [*self._extensions, extension], + self._revoked_certificates, + ) + + def add_revoked_certificate( + self, revoked_certificate: RevokedCertificate + ) -> CertificateRevocationListBuilder: + """ + Adds a revoked certificate to the CRL. + """ + if not isinstance(revoked_certificate, RevokedCertificate): + raise TypeError("Must be an instance of RevokedCertificate") + + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + self._next_update, + self._extensions, + [*self._revoked_certificates, revoked_certificate], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: _AllowedHashTypes | None, + backend: typing.Any = None, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ecdsa_deterministic: bool | None = None, + ) -> CertificateRevocationList: + if self._issuer_name is None: + raise ValueError("A CRL must have an issuer name") + + if self._last_update is None: + raise ValueError("A CRL must have a last update time") + + if self._next_update is None: + raise ValueError("A CRL must have a next update time") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + if ecdsa_deterministic is not None: + if not isinstance(private_key, ec.EllipticCurvePrivateKey): + raise TypeError( + "Deterministic ECDSA is only supported for EC keys" + ) + + return rust_x509.create_x509_crl( + self, + private_key, + algorithm, + rsa_padding, + ecdsa_deterministic, + ) + + +class RevokedCertificateBuilder: + def __init__( + self, + serial_number: int | None = None, + revocation_date: datetime.datetime | None = None, + extensions: list[Extension[ExtensionType]] = [], + ): + self._serial_number = serial_number + self._revocation_date = revocation_date + self._extensions = extensions + + def serial_number(self, number: int) -> RevokedCertificateBuilder: + if not isinstance(number, int): + raise TypeError("Serial number must be of integral type.") + if self._serial_number is not None: + raise ValueError("The serial number may only be set once.") + if number <= 0: + raise ValueError("The serial number should be positive") + + # ASN.1 integers are always signed, so most significant bit must be + # zero. + if number.bit_length() >= 160: # As defined in RFC 5280 + raise ValueError( + "The serial number should not be more than 159 bits." + ) + return RevokedCertificateBuilder( + number, self._revocation_date, self._extensions + ) + + def revocation_date( + self, time: datetime.datetime + ) -> RevokedCertificateBuilder: + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._revocation_date is not None: + raise ValueError("The revocation date may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The revocation date must be on or after 1950 January 1." + ) + return RevokedCertificateBuilder( + self._serial_number, time, self._extensions + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> RevokedCertificateBuilder: + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + return RevokedCertificateBuilder( + self._serial_number, + self._revocation_date, + [*self._extensions, extension], + ) + + def build(self, backend: typing.Any = None) -> RevokedCertificate: + if self._serial_number is None: + raise ValueError("A revoked certificate must have a serial number") + if self._revocation_date is None: + raise ValueError( + "A revoked certificate must have a revocation date" + ) + return _RawRevokedCertificate( + self._serial_number, + self._revocation_date, + Extensions(self._extensions), + ) + + +def random_serial_number() -> int: + return int.from_bytes(os.urandom(20), "big") >> 1 diff --git a/lib/cryptography/x509/certificate_transparency.py b/lib/cryptography/x509/certificate_transparency.py new file mode 100644 index 0000000..fb66cc6 --- /dev/null +++ b/lib/cryptography/x509/certificate_transparency.py @@ -0,0 +1,35 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.bindings._rust import x509 as rust_x509 + + +class LogEntryType(utils.Enum): + X509_CERTIFICATE = 0 + PRE_CERTIFICATE = 1 + + +class Version(utils.Enum): + v1 = 0 + + +class SignatureAlgorithm(utils.Enum): + """ + Signature algorithms that are valid for SCTs. + + These are exactly the same as SignatureAlgorithm in RFC 5246 (TLS 1.2). + + See: + """ + + ANONYMOUS = 0 + RSA = 1 + DSA = 2 + ECDSA = 3 + + +SignedCertificateTimestamp = rust_x509.Sct diff --git a/lib/cryptography/x509/extensions.py b/lib/cryptography/x509/extensions.py new file mode 100644 index 0000000..dfa472d --- /dev/null +++ b/lib/cryptography/x509/extensions.py @@ -0,0 +1,2528 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import datetime +import hashlib +import ipaddress +import typing +from collections.abc import Iterable, Iterator + +from cryptography import utils +from cryptography.hazmat.bindings._rust import asn1 +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.hazmat.primitives import constant_time, serialization +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPublicKeyTypes, + CertificatePublicKeyTypes, +) +from cryptography.x509.certificate_transparency import ( + SignedCertificateTimestamp, +) +from cryptography.x509.general_name import ( + DirectoryName, + DNSName, + GeneralName, + IPAddress, + OtherName, + RegisteredID, + RFC822Name, + UniformResourceIdentifier, + _IPAddressTypes, +) +from cryptography.x509.name import Name, RelativeDistinguishedName +from cryptography.x509.oid import ( + CRLEntryExtensionOID, + ExtensionOID, + ObjectIdentifier, + OCSPExtensionOID, +) + +ExtensionTypeVar = typing.TypeVar( + "ExtensionTypeVar", bound="ExtensionType", covariant=True +) + + +def _key_identifier_from_public_key( + public_key: CertificatePublicKeyTypes, +) -> bytes: + if isinstance(public_key, RSAPublicKey): + data = public_key.public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.PKCS1, + ) + elif isinstance(public_key, EllipticCurvePublicKey): + data = public_key.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint, + ) + else: + # This is a very slow way to do this. + serialized = public_key.public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + data = asn1.parse_spki_for_data(serialized) + + return hashlib.sha1(data).digest() + + +def _make_sequence_methods(field_name: str): + def len_method(self) -> int: + return len(getattr(self, field_name)) + + def iter_method(self): + return iter(getattr(self, field_name)) + + def getitem_method(self, idx): + return getattr(self, field_name)[idx] + + return len_method, iter_method, getitem_method + + +class DuplicateExtension(Exception): + def __init__(self, msg: str, oid: ObjectIdentifier) -> None: + super().__init__(msg) + self.oid = oid + + +class ExtensionNotFound(Exception): + def __init__(self, msg: str, oid: ObjectIdentifier) -> None: + super().__init__(msg) + self.oid = oid + + +class ExtensionType(metaclass=abc.ABCMeta): + oid: typing.ClassVar[ObjectIdentifier] + + def public_bytes(self) -> bytes: + """ + Serializes the extension type to DER. + """ + raise NotImplementedError( + f"public_bytes is not implemented for extension type {self!r}" + ) + + +class Extensions: + def __init__(self, extensions: Iterable[Extension[ExtensionType]]) -> None: + self._extensions = list(extensions) + + def get_extension_for_oid( + self, oid: ObjectIdentifier + ) -> Extension[ExtensionType]: + for ext in self: + if ext.oid == oid: + return ext + + raise ExtensionNotFound(f"No {oid} extension was found", oid) + + def get_extension_for_class( + self, extclass: type[ExtensionTypeVar] + ) -> Extension[ExtensionTypeVar]: + if extclass is UnrecognizedExtension: + raise TypeError( + "UnrecognizedExtension can't be used with " + "get_extension_for_class because more than one instance of the" + " class may be present." + ) + + for ext in self: + if isinstance(ext.value, extclass): + return ext + + raise ExtensionNotFound( + f"No {extclass} extension was found", extclass.oid + ) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_extensions") + + def __repr__(self) -> str: + return f"" + + +class CRLNumber(ExtensionType): + oid = ExtensionOID.CRL_NUMBER + + def __init__(self, crl_number: int) -> None: + if not isinstance(crl_number, int): + raise TypeError("crl_number must be an integer") + + self._crl_number = crl_number + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CRLNumber): + return NotImplemented + + return self.crl_number == other.crl_number + + def __hash__(self) -> int: + return hash(self.crl_number) + + def __repr__(self) -> str: + return f"" + + @property + def crl_number(self) -> int: + return self._crl_number + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class AuthorityKeyIdentifier(ExtensionType): + oid = ExtensionOID.AUTHORITY_KEY_IDENTIFIER + + def __init__( + self, + key_identifier: bytes | None, + authority_cert_issuer: Iterable[GeneralName] | None, + authority_cert_serial_number: int | None, + ) -> None: + if (authority_cert_issuer is None) != ( + authority_cert_serial_number is None + ): + raise ValueError( + "authority_cert_issuer and authority_cert_serial_number " + "must both be present or both None" + ) + + if authority_cert_issuer is not None: + authority_cert_issuer = list(authority_cert_issuer) + if not all( + isinstance(x, GeneralName) for x in authority_cert_issuer + ): + raise TypeError( + "authority_cert_issuer must be a list of GeneralName " + "objects" + ) + + if authority_cert_serial_number is not None and not isinstance( + authority_cert_serial_number, int + ): + raise TypeError("authority_cert_serial_number must be an integer") + + self._key_identifier = key_identifier + self._authority_cert_issuer = authority_cert_issuer + self._authority_cert_serial_number = authority_cert_serial_number + + # This takes a subset of CertificatePublicKeyTypes because an issuer + # cannot have an X25519/X448 key. This introduces some unfortunate + # asymmetry that requires typing users to explicitly + # narrow their type, but we should make this accurate and not just + # convenient. + @classmethod + def from_issuer_public_key( + cls, public_key: CertificateIssuerPublicKeyTypes + ) -> AuthorityKeyIdentifier: + digest = _key_identifier_from_public_key(public_key) + return cls( + key_identifier=digest, + authority_cert_issuer=None, + authority_cert_serial_number=None, + ) + + @classmethod + def from_issuer_subject_key_identifier( + cls, ski: SubjectKeyIdentifier + ) -> AuthorityKeyIdentifier: + return cls( + key_identifier=ski.digest, + authority_cert_issuer=None, + authority_cert_serial_number=None, + ) + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AuthorityKeyIdentifier): + return NotImplemented + + return ( + self.key_identifier == other.key_identifier + and self.authority_cert_issuer == other.authority_cert_issuer + and self.authority_cert_serial_number + == other.authority_cert_serial_number + ) + + def __hash__(self) -> int: + if self.authority_cert_issuer is None: + aci = None + else: + aci = tuple(self.authority_cert_issuer) + return hash( + (self.key_identifier, aci, self.authority_cert_serial_number) + ) + + @property + def key_identifier(self) -> bytes | None: + return self._key_identifier + + @property + def authority_cert_issuer( + self, + ) -> list[GeneralName] | None: + return self._authority_cert_issuer + + @property + def authority_cert_serial_number(self) -> int | None: + return self._authority_cert_serial_number + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class SubjectKeyIdentifier(ExtensionType): + oid = ExtensionOID.SUBJECT_KEY_IDENTIFIER + + def __init__(self, digest: bytes) -> None: + self._digest = digest + + @classmethod + def from_public_key( + cls, public_key: CertificatePublicKeyTypes + ) -> SubjectKeyIdentifier: + return cls(_key_identifier_from_public_key(public_key)) + + @property + def digest(self) -> bytes: + return self._digest + + @property + def key_identifier(self) -> bytes: + return self._digest + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SubjectKeyIdentifier): + return NotImplemented + + return constant_time.bytes_eq(self.digest, other.digest) + + def __hash__(self) -> int: + return hash(self.digest) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class AuthorityInformationAccess(ExtensionType): + oid = ExtensionOID.AUTHORITY_INFORMATION_ACCESS + + def __init__(self, descriptions: Iterable[AccessDescription]) -> None: + descriptions = list(descriptions) + if not all(isinstance(x, AccessDescription) for x in descriptions): + raise TypeError( + "Every item in the descriptions list must be an " + "AccessDescription" + ) + + self._descriptions = descriptions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AuthorityInformationAccess): + return NotImplemented + + return self._descriptions == other._descriptions + + def __hash__(self) -> int: + return hash(tuple(self._descriptions)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class SubjectInformationAccess(ExtensionType): + oid = ExtensionOID.SUBJECT_INFORMATION_ACCESS + + def __init__(self, descriptions: Iterable[AccessDescription]) -> None: + descriptions = list(descriptions) + if not all(isinstance(x, AccessDescription) for x in descriptions): + raise TypeError( + "Every item in the descriptions list must be an " + "AccessDescription" + ) + + self._descriptions = descriptions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SubjectInformationAccess): + return NotImplemented + + return self._descriptions == other._descriptions + + def __hash__(self) -> int: + return hash(tuple(self._descriptions)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class AccessDescription: + def __init__( + self, access_method: ObjectIdentifier, access_location: GeneralName + ) -> None: + if not isinstance(access_method, ObjectIdentifier): + raise TypeError("access_method must be an ObjectIdentifier") + + if not isinstance(access_location, GeneralName): + raise TypeError("access_location must be a GeneralName") + + self._access_method = access_method + self._access_location = access_location + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AccessDescription): + return NotImplemented + + return ( + self.access_method == other.access_method + and self.access_location == other.access_location + ) + + def __hash__(self) -> int: + return hash((self.access_method, self.access_location)) + + @property + def access_method(self) -> ObjectIdentifier: + return self._access_method + + @property + def access_location(self) -> GeneralName: + return self._access_location + + +class BasicConstraints(ExtensionType): + oid = ExtensionOID.BASIC_CONSTRAINTS + + def __init__(self, ca: bool, path_length: int | None) -> None: + if not isinstance(ca, bool): + raise TypeError("ca must be a boolean value") + + if path_length is not None and not ca: + raise ValueError("path_length must be None when ca is False") + + if path_length is not None and ( + not isinstance(path_length, int) or path_length < 0 + ): + raise TypeError( + "path_length must be a non-negative integer or None" + ) + + self._ca = ca + self._path_length = path_length + + @property + def ca(self) -> bool: + return self._ca + + @property + def path_length(self) -> int | None: + return self._path_length + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, BasicConstraints): + return NotImplemented + + return self.ca == other.ca and self.path_length == other.path_length + + def __hash__(self) -> int: + return hash((self.ca, self.path_length)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class DeltaCRLIndicator(ExtensionType): + oid = ExtensionOID.DELTA_CRL_INDICATOR + + def __init__(self, crl_number: int) -> None: + if not isinstance(crl_number, int): + raise TypeError("crl_number must be an integer") + + self._crl_number = crl_number + + @property + def crl_number(self) -> int: + return self._crl_number + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DeltaCRLIndicator): + return NotImplemented + + return self.crl_number == other.crl_number + + def __hash__(self) -> int: + return hash(self.crl_number) + + def __repr__(self) -> str: + return f"" + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CRLDistributionPoints(ExtensionType): + oid = ExtensionOID.CRL_DISTRIBUTION_POINTS + + def __init__( + self, distribution_points: Iterable[DistributionPoint] + ) -> None: + distribution_points = list(distribution_points) + if not all( + isinstance(x, DistributionPoint) for x in distribution_points + ): + raise TypeError( + "distribution_points must be a list of DistributionPoint " + "objects" + ) + + self._distribution_points = distribution_points + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CRLDistributionPoints): + return NotImplemented + + return self._distribution_points == other._distribution_points + + def __hash__(self) -> int: + return hash(tuple(self._distribution_points)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class FreshestCRL(ExtensionType): + oid = ExtensionOID.FRESHEST_CRL + + def __init__( + self, distribution_points: Iterable[DistributionPoint] + ) -> None: + distribution_points = list(distribution_points) + if not all( + isinstance(x, DistributionPoint) for x in distribution_points + ): + raise TypeError( + "distribution_points must be a list of DistributionPoint " + "objects" + ) + + self._distribution_points = distribution_points + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, FreshestCRL): + return NotImplemented + + return self._distribution_points == other._distribution_points + + def __hash__(self) -> int: + return hash(tuple(self._distribution_points)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class DistributionPoint: + def __init__( + self, + full_name: Iterable[GeneralName] | None, + relative_name: RelativeDistinguishedName | None, + reasons: frozenset[ReasonFlags] | None, + crl_issuer: Iterable[GeneralName] | None, + ) -> None: + if full_name and relative_name: + raise ValueError( + "You cannot provide both full_name and relative_name, at " + "least one must be None." + ) + if not full_name and not relative_name and not crl_issuer: + raise ValueError( + "Either full_name, relative_name or crl_issuer must be " + "provided." + ) + + if full_name is not None: + full_name = list(full_name) + if not all(isinstance(x, GeneralName) for x in full_name): + raise TypeError( + "full_name must be a list of GeneralName objects" + ) + + if relative_name: + if not isinstance(relative_name, RelativeDistinguishedName): + raise TypeError( + "relative_name must be a RelativeDistinguishedName" + ) + + if crl_issuer is not None: + crl_issuer = list(crl_issuer) + if not all(isinstance(x, GeneralName) for x in crl_issuer): + raise TypeError( + "crl_issuer must be None or a list of general names" + ) + + if reasons and ( + not isinstance(reasons, frozenset) + or not all(isinstance(x, ReasonFlags) for x in reasons) + ): + raise TypeError("reasons must be None or frozenset of ReasonFlags") + + if reasons and ( + ReasonFlags.unspecified in reasons + or ReasonFlags.remove_from_crl in reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in a " + "DistributionPoint" + ) + + self._full_name = full_name + self._relative_name = relative_name + self._reasons = reasons + self._crl_issuer = crl_issuer + + def __repr__(self) -> str: + return ( + "".format(self) + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name + and self.relative_name == other.relative_name + and self.reasons == other.reasons + and self.crl_issuer == other.crl_issuer + ) + + def __hash__(self) -> int: + if self.full_name is not None: + fn: tuple[GeneralName, ...] | None = tuple(self.full_name) + else: + fn = None + + if self.crl_issuer is not None: + crl_issuer: tuple[GeneralName, ...] | None = tuple(self.crl_issuer) + else: + crl_issuer = None + + return hash((fn, self.relative_name, self.reasons, crl_issuer)) + + @property + def full_name(self) -> list[GeneralName] | None: + return self._full_name + + @property + def relative_name(self) -> RelativeDistinguishedName | None: + return self._relative_name + + @property + def reasons(self) -> frozenset[ReasonFlags] | None: + return self._reasons + + @property + def crl_issuer(self) -> list[GeneralName] | None: + return self._crl_issuer + + +class ReasonFlags(utils.Enum): + unspecified = "unspecified" + key_compromise = "keyCompromise" + ca_compromise = "cACompromise" + affiliation_changed = "affiliationChanged" + superseded = "superseded" + cessation_of_operation = "cessationOfOperation" + certificate_hold = "certificateHold" + privilege_withdrawn = "privilegeWithdrawn" + aa_compromise = "aACompromise" + remove_from_crl = "removeFromCRL" + + +# These are distribution point bit string mappings. Not to be confused with +# CRLReason reason flags bit string mappings. +# ReasonFlags ::= BIT STRING { +# unused (0), +# keyCompromise (1), +# cACompromise (2), +# affiliationChanged (3), +# superseded (4), +# cessationOfOperation (5), +# certificateHold (6), +# privilegeWithdrawn (7), +# aACompromise (8) } +_REASON_BIT_MAPPING = { + 1: ReasonFlags.key_compromise, + 2: ReasonFlags.ca_compromise, + 3: ReasonFlags.affiliation_changed, + 4: ReasonFlags.superseded, + 5: ReasonFlags.cessation_of_operation, + 6: ReasonFlags.certificate_hold, + 7: ReasonFlags.privilege_withdrawn, + 8: ReasonFlags.aa_compromise, +} + +_CRLREASONFLAGS = { + ReasonFlags.key_compromise: 1, + ReasonFlags.ca_compromise: 2, + ReasonFlags.affiliation_changed: 3, + ReasonFlags.superseded: 4, + ReasonFlags.cessation_of_operation: 5, + ReasonFlags.certificate_hold: 6, + ReasonFlags.privilege_withdrawn: 7, + ReasonFlags.aa_compromise: 8, +} + +# CRLReason ::= ENUMERATED { +# unspecified (0), +# keyCompromise (1), +# cACompromise (2), +# affiliationChanged (3), +# superseded (4), +# cessationOfOperation (5), +# certificateHold (6), +# -- value 7 is not used +# removeFromCRL (8), +# privilegeWithdrawn (9), +# aACompromise (10) } +_CRL_ENTRY_REASON_ENUM_TO_CODE = { + ReasonFlags.unspecified: 0, + ReasonFlags.key_compromise: 1, + ReasonFlags.ca_compromise: 2, + ReasonFlags.affiliation_changed: 3, + ReasonFlags.superseded: 4, + ReasonFlags.cessation_of_operation: 5, + ReasonFlags.certificate_hold: 6, + ReasonFlags.remove_from_crl: 8, + ReasonFlags.privilege_withdrawn: 9, + ReasonFlags.aa_compromise: 10, +} + + +class PolicyConstraints(ExtensionType): + oid = ExtensionOID.POLICY_CONSTRAINTS + + def __init__( + self, + require_explicit_policy: int | None, + inhibit_policy_mapping: int | None, + ) -> None: + if require_explicit_policy is not None and not isinstance( + require_explicit_policy, int + ): + raise TypeError( + "require_explicit_policy must be a non-negative integer or " + "None" + ) + + if inhibit_policy_mapping is not None and not isinstance( + inhibit_policy_mapping, int + ): + raise TypeError( + "inhibit_policy_mapping must be a non-negative integer or None" + ) + + if inhibit_policy_mapping is None and require_explicit_policy is None: + raise ValueError( + "At least one of require_explicit_policy and " + "inhibit_policy_mapping must not be None" + ) + + self._require_explicit_policy = require_explicit_policy + self._inhibit_policy_mapping = inhibit_policy_mapping + + def __repr__(self) -> str: + return ( + "".format(self) + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PolicyConstraints): + return NotImplemented + + return ( + self.require_explicit_policy == other.require_explicit_policy + and self.inhibit_policy_mapping == other.inhibit_policy_mapping + ) + + def __hash__(self) -> int: + return hash( + (self.require_explicit_policy, self.inhibit_policy_mapping) + ) + + @property + def require_explicit_policy(self) -> int | None: + return self._require_explicit_policy + + @property + def inhibit_policy_mapping(self) -> int | None: + return self._inhibit_policy_mapping + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CertificatePolicies(ExtensionType): + oid = ExtensionOID.CERTIFICATE_POLICIES + + def __init__(self, policies: Iterable[PolicyInformation]) -> None: + policies = list(policies) + if not all(isinstance(x, PolicyInformation) for x in policies): + raise TypeError( + "Every item in the policies list must be a PolicyInformation" + ) + + self._policies = policies + + __len__, __iter__, __getitem__ = _make_sequence_methods("_policies") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CertificatePolicies): + return NotImplemented + + return self._policies == other._policies + + def __hash__(self) -> int: + return hash(tuple(self._policies)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PolicyInformation: + def __init__( + self, + policy_identifier: ObjectIdentifier, + policy_qualifiers: Iterable[str | UserNotice] | None, + ) -> None: + if not isinstance(policy_identifier, ObjectIdentifier): + raise TypeError("policy_identifier must be an ObjectIdentifier") + + self._policy_identifier = policy_identifier + + if policy_qualifiers is not None: + policy_qualifiers = list(policy_qualifiers) + if not all( + isinstance(x, (str, UserNotice)) for x in policy_qualifiers + ): + raise TypeError( + "policy_qualifiers must be a list of strings and/or " + "UserNotice objects or None" + ) + + self._policy_qualifiers = policy_qualifiers + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PolicyInformation): + return NotImplemented + + return ( + self.policy_identifier == other.policy_identifier + and self.policy_qualifiers == other.policy_qualifiers + ) + + def __hash__(self) -> int: + if self.policy_qualifiers is not None: + pq = tuple(self.policy_qualifiers) + else: + pq = None + + return hash((self.policy_identifier, pq)) + + @property + def policy_identifier(self) -> ObjectIdentifier: + return self._policy_identifier + + @property + def policy_qualifiers( + self, + ) -> list[str | UserNotice] | None: + return self._policy_qualifiers + + +class UserNotice: + def __init__( + self, + notice_reference: NoticeReference | None, + explicit_text: str | None, + ) -> None: + if notice_reference and not isinstance( + notice_reference, NoticeReference + ): + raise TypeError( + "notice_reference must be None or a NoticeReference" + ) + + self._notice_reference = notice_reference + self._explicit_text = explicit_text + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UserNotice): + return NotImplemented + + return ( + self.notice_reference == other.notice_reference + and self.explicit_text == other.explicit_text + ) + + def __hash__(self) -> int: + return hash((self.notice_reference, self.explicit_text)) + + @property + def notice_reference(self) -> NoticeReference | None: + return self._notice_reference + + @property + def explicit_text(self) -> str | None: + return self._explicit_text + + +class NoticeReference: + def __init__( + self, + organization: str | None, + notice_numbers: Iterable[int], + ) -> None: + self._organization = organization + notice_numbers = list(notice_numbers) + if not all(isinstance(x, int) for x in notice_numbers): + raise TypeError("notice_numbers must be a list of integers") + + self._notice_numbers = notice_numbers + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NoticeReference): + return NotImplemented + + return ( + self.organization == other.organization + and self.notice_numbers == other.notice_numbers + ) + + def __hash__(self) -> int: + return hash((self.organization, tuple(self.notice_numbers))) + + @property + def organization(self) -> str | None: + return self._organization + + @property + def notice_numbers(self) -> list[int]: + return self._notice_numbers + + +class ExtendedKeyUsage(ExtensionType): + oid = ExtensionOID.EXTENDED_KEY_USAGE + + def __init__(self, usages: Iterable[ObjectIdentifier]) -> None: + usages = list(usages) + if not all(isinstance(x, ObjectIdentifier) for x in usages): + raise TypeError( + "Every item in the usages list must be an ObjectIdentifier" + ) + + self._usages = usages + + __len__, __iter__, __getitem__ = _make_sequence_methods("_usages") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ExtendedKeyUsage): + return NotImplemented + + return self._usages == other._usages + + def __hash__(self) -> int: + return hash(tuple(self._usages)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class OCSPNoCheck(ExtensionType): + oid = ExtensionOID.OCSP_NO_CHECK + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OCSPNoCheck): + return NotImplemented + + return True + + def __hash__(self) -> int: + return hash(OCSPNoCheck) + + def __repr__(self) -> str: + return "" + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PrecertPoison(ExtensionType): + oid = ExtensionOID.PRECERT_POISON + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PrecertPoison): + return NotImplemented + + return True + + def __hash__(self) -> int: + return hash(PrecertPoison) + + def __repr__(self) -> str: + return "" + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class TLSFeature(ExtensionType): + oid = ExtensionOID.TLS_FEATURE + + def __init__(self, features: Iterable[TLSFeatureType]) -> None: + features = list(features) + if ( + not all(isinstance(x, TLSFeatureType) for x in features) + or len(features) == 0 + ): + raise TypeError( + "features must be a list of elements from the TLSFeatureType " + "enum" + ) + + self._features = features + + __len__, __iter__, __getitem__ = _make_sequence_methods("_features") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TLSFeature): + return NotImplemented + + return self._features == other._features + + def __hash__(self) -> int: + return hash(tuple(self._features)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class TLSFeatureType(utils.Enum): + # status_request is defined in RFC 6066 and is used for what is commonly + # called OCSP Must-Staple when present in the TLS Feature extension in an + # X.509 certificate. + status_request = 5 + # status_request_v2 is defined in RFC 6961 and allows multiple OCSP + # responses to be provided. It is not currently in use by clients or + # servers. + status_request_v2 = 17 + + +_TLS_FEATURE_TYPE_TO_ENUM = {x.value: x for x in TLSFeatureType} + + +class InhibitAnyPolicy(ExtensionType): + oid = ExtensionOID.INHIBIT_ANY_POLICY + + def __init__(self, skip_certs: int) -> None: + if not isinstance(skip_certs, int): + raise TypeError("skip_certs must be an integer") + + if skip_certs < 0: + raise ValueError("skip_certs must be a non-negative integer") + + self._skip_certs = skip_certs + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, InhibitAnyPolicy): + return NotImplemented + + return self.skip_certs == other.skip_certs + + def __hash__(self) -> int: + return hash(self.skip_certs) + + @property + def skip_certs(self) -> int: + return self._skip_certs + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class KeyUsage(ExtensionType): + oid = ExtensionOID.KEY_USAGE + + def __init__( + self, + digital_signature: bool, + content_commitment: bool, + key_encipherment: bool, + data_encipherment: bool, + key_agreement: bool, + key_cert_sign: bool, + crl_sign: bool, + encipher_only: bool, + decipher_only: bool, + ) -> None: + if not key_agreement and (encipher_only or decipher_only): + raise ValueError( + "encipher_only and decipher_only can only be true when " + "key_agreement is true" + ) + + self._digital_signature = digital_signature + self._content_commitment = content_commitment + self._key_encipherment = key_encipherment + self._data_encipherment = data_encipherment + self._key_agreement = key_agreement + self._key_cert_sign = key_cert_sign + self._crl_sign = crl_sign + self._encipher_only = encipher_only + self._decipher_only = decipher_only + + @property + def digital_signature(self) -> bool: + return self._digital_signature + + @property + def content_commitment(self) -> bool: + return self._content_commitment + + @property + def key_encipherment(self) -> bool: + return self._key_encipherment + + @property + def data_encipherment(self) -> bool: + return self._data_encipherment + + @property + def key_agreement(self) -> bool: + return self._key_agreement + + @property + def key_cert_sign(self) -> bool: + return self._key_cert_sign + + @property + def crl_sign(self) -> bool: + return self._crl_sign + + @property + def encipher_only(self) -> bool: + if not self.key_agreement: + raise ValueError( + "encipher_only is undefined unless key_agreement is true" + ) + else: + return self._encipher_only + + @property + def decipher_only(self) -> bool: + if not self.key_agreement: + raise ValueError( + "decipher_only is undefined unless key_agreement is true" + ) + else: + return self._decipher_only + + def __repr__(self) -> str: + try: + encipher_only = self.encipher_only + decipher_only = self.decipher_only + except ValueError: + # Users found None confusing because even though encipher/decipher + # have no meaning unless key_agreement is true, to construct an + # instance of the class you still need to pass False. + encipher_only = False + decipher_only = False + + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, KeyUsage): + return NotImplemented + + return ( + self.digital_signature == other.digital_signature + and self.content_commitment == other.content_commitment + and self.key_encipherment == other.key_encipherment + and self.data_encipherment == other.data_encipherment + and self.key_agreement == other.key_agreement + and self.key_cert_sign == other.key_cert_sign + and self.crl_sign == other.crl_sign + and self._encipher_only == other._encipher_only + and self._decipher_only == other._decipher_only + ) + + def __hash__(self) -> int: + return hash( + ( + self.digital_signature, + self.content_commitment, + self.key_encipherment, + self.data_encipherment, + self.key_agreement, + self.key_cert_sign, + self.crl_sign, + self._encipher_only, + self._decipher_only, + ) + ) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PrivateKeyUsagePeriod(ExtensionType): + oid = ExtensionOID.PRIVATE_KEY_USAGE_PERIOD + + def __init__( + self, + not_before: datetime.datetime | None, + not_after: datetime.datetime | None, + ) -> None: + if ( + not isinstance(not_before, datetime.datetime) + and not_before is not None + ): + raise TypeError("not_before must be a datetime.datetime or None") + + if ( + not isinstance(not_after, datetime.datetime) + and not_after is not None + ): + raise TypeError("not_after must be a datetime.datetime or None") + + if not_before is None and not_after is None: + raise ValueError( + "At least one of not_before and not_after must not be None" + ) + + if ( + not_before is not None + and not_after is not None + and not_before > not_after + ): + raise ValueError("not_before must be before not_after") + + self._not_before = not_before + self._not_after = not_after + + @property + def not_before(self) -> datetime.datetime | None: + return self._not_before + + @property + def not_after(self) -> datetime.datetime | None: + return self._not_after + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PrivateKeyUsagePeriod): + return NotImplemented + + return ( + self.not_before == other.not_before + and self.not_after == other.not_after + ) + + def __hash__(self) -> int: + return hash((self.not_before, self.not_after)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class NameConstraints(ExtensionType): + oid = ExtensionOID.NAME_CONSTRAINTS + + def __init__( + self, + permitted_subtrees: Iterable[GeneralName] | None, + excluded_subtrees: Iterable[GeneralName] | None, + ) -> None: + if permitted_subtrees is not None: + permitted_subtrees = list(permitted_subtrees) + if not permitted_subtrees: + raise ValueError( + "permitted_subtrees must be a non-empty list or None" + ) + if not all(isinstance(x, GeneralName) for x in permitted_subtrees): + raise TypeError( + "permitted_subtrees must be a list of GeneralName objects " + "or None" + ) + + self._validate_tree(permitted_subtrees) + + if excluded_subtrees is not None: + excluded_subtrees = list(excluded_subtrees) + if not excluded_subtrees: + raise ValueError( + "excluded_subtrees must be a non-empty list or None" + ) + if not all(isinstance(x, GeneralName) for x in excluded_subtrees): + raise TypeError( + "excluded_subtrees must be a list of GeneralName objects " + "or None" + ) + + self._validate_tree(excluded_subtrees) + + if permitted_subtrees is None and excluded_subtrees is None: + raise ValueError( + "At least one of permitted_subtrees and excluded_subtrees " + "must not be None" + ) + + self._permitted_subtrees = permitted_subtrees + self._excluded_subtrees = excluded_subtrees + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NameConstraints): + return NotImplemented + + return ( + self.excluded_subtrees == other.excluded_subtrees + and self.permitted_subtrees == other.permitted_subtrees + ) + + def _validate_tree(self, tree: Iterable[GeneralName]) -> None: + self._validate_ip_name(tree) + self._validate_dns_name(tree) + + def _validate_ip_name(self, tree: Iterable[GeneralName]) -> None: + if any( + isinstance(name, IPAddress) + and not isinstance( + name.value, (ipaddress.IPv4Network, ipaddress.IPv6Network) + ) + for name in tree + ): + raise TypeError( + "IPAddress name constraints must be an IPv4Network or" + " IPv6Network object" + ) + + def _validate_dns_name(self, tree: Iterable[GeneralName]) -> None: + if any( + isinstance(name, DNSName) and "*" in name.value for name in tree + ): + raise ValueError( + "DNSName name constraints must not contain the '*' wildcard" + " character" + ) + + def __repr__(self) -> str: + return ( + f"" + ) + + def __hash__(self) -> int: + if self.permitted_subtrees is not None: + ps: tuple[GeneralName, ...] | None = tuple(self.permitted_subtrees) + else: + ps = None + + if self.excluded_subtrees is not None: + es: tuple[GeneralName, ...] | None = tuple(self.excluded_subtrees) + else: + es = None + + return hash((ps, es)) + + @property + def permitted_subtrees( + self, + ) -> list[GeneralName] | None: + return self._permitted_subtrees + + @property + def excluded_subtrees( + self, + ) -> list[GeneralName] | None: + return self._excluded_subtrees + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class Extension(typing.Generic[ExtensionTypeVar]): + def __init__( + self, oid: ObjectIdentifier, critical: bool, value: ExtensionTypeVar + ) -> None: + if not isinstance(oid, ObjectIdentifier): + raise TypeError( + "oid argument must be an ObjectIdentifier instance." + ) + + if not isinstance(critical, bool): + raise TypeError("critical must be a boolean value") + + self._oid = oid + self._critical = critical + self._value = value + + @property + def oid(self) -> ObjectIdentifier: + return self._oid + + @property + def critical(self) -> bool: + return self._critical + + @property + def value(self) -> ExtensionTypeVar: + return self._value + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Extension): + return NotImplemented + + return ( + self.oid == other.oid + and self.critical == other.critical + and self.value == other.value + ) + + def __hash__(self) -> int: + return hash((self.oid, self.critical, self.value)) + + +class GeneralNames: + def __init__(self, general_names: Iterable[GeneralName]) -> None: + general_names = list(general_names) + if not all(isinstance(x, GeneralName) for x in general_names): + raise TypeError( + "Every item in the general_names list must be an " + "object conforming to the GeneralName interface" + ) + + self._general_names = general_names + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + # Return the value of each GeneralName, except for OtherName instances + # which we return directly because it has two important properties not + # just one value. + objs = (i for i in self if isinstance(i, type)) + if type != OtherName: + return [i.value for i in objs] + return list(objs) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, GeneralNames): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(tuple(self._general_names)) + + +class SubjectAlternativeName(ExtensionType): + oid = ExtensionOID.SUBJECT_ALTERNATIVE_NAME + + def __init__(self, general_names: Iterable[GeneralName]) -> None: + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + return self._general_names.get_values_for_type(type) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SubjectAlternativeName): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(self._general_names) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class IssuerAlternativeName(ExtensionType): + oid = ExtensionOID.ISSUER_ALTERNATIVE_NAME + + def __init__(self, general_names: Iterable[GeneralName]) -> None: + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + return self._general_names.get_values_for_type(type) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, IssuerAlternativeName): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(self._general_names) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CertificateIssuer(ExtensionType): + oid = CRLEntryExtensionOID.CERTIFICATE_ISSUER + + def __init__(self, general_names: Iterable[GeneralName]) -> None: + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + return self._general_names.get_values_for_type(type) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CertificateIssuer): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(self._general_names) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CRLReason(ExtensionType): + oid = CRLEntryExtensionOID.CRL_REASON + + def __init__(self, reason: ReasonFlags) -> None: + if not isinstance(reason, ReasonFlags): + raise TypeError("reason must be an element from ReasonFlags") + + self._reason = reason + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CRLReason): + return NotImplemented + + return self.reason == other.reason + + def __hash__(self) -> int: + return hash(self.reason) + + @property + def reason(self) -> ReasonFlags: + return self._reason + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class InvalidityDate(ExtensionType): + oid = CRLEntryExtensionOID.INVALIDITY_DATE + + def __init__(self, invalidity_date: datetime.datetime) -> None: + if not isinstance(invalidity_date, datetime.datetime): + raise TypeError("invalidity_date must be a datetime.datetime") + + self._invalidity_date = invalidity_date + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, InvalidityDate): + return NotImplemented + + return self.invalidity_date == other.invalidity_date + + def __hash__(self) -> int: + return hash(self.invalidity_date) + + @property + def invalidity_date(self) -> datetime.datetime: + return self._invalidity_date + + @property + def invalidity_date_utc(self) -> datetime.datetime: + if self._invalidity_date.tzinfo is None: + return self._invalidity_date.replace(tzinfo=datetime.timezone.utc) + else: + return self._invalidity_date.astimezone(tz=datetime.timezone.utc) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PrecertificateSignedCertificateTimestamps(ExtensionType): + oid = ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS + + def __init__( + self, + signed_certificate_timestamps: Iterable[SignedCertificateTimestamp], + ) -> None: + signed_certificate_timestamps = list(signed_certificate_timestamps) + if not all( + isinstance(sct, SignedCertificateTimestamp) + for sct in signed_certificate_timestamps + ): + raise TypeError( + "Every item in the signed_certificate_timestamps list must be " + "a SignedCertificateTimestamp" + ) + self._signed_certificate_timestamps = signed_certificate_timestamps + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) + + def __repr__(self) -> str: + return f"" + + def __hash__(self) -> int: + return hash(tuple(self._signed_certificate_timestamps)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PrecertificateSignedCertificateTimestamps): + return NotImplemented + + return ( + self._signed_certificate_timestamps + == other._signed_certificate_timestamps + ) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class SignedCertificateTimestamps(ExtensionType): + oid = ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS + + def __init__( + self, + signed_certificate_timestamps: Iterable[SignedCertificateTimestamp], + ) -> None: + signed_certificate_timestamps = list(signed_certificate_timestamps) + if not all( + isinstance(sct, SignedCertificateTimestamp) + for sct in signed_certificate_timestamps + ): + raise TypeError( + "Every item in the signed_certificate_timestamps list must be " + "a SignedCertificateTimestamp" + ) + self._signed_certificate_timestamps = signed_certificate_timestamps + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) + + def __repr__(self) -> str: + return f"" + + def __hash__(self) -> int: + return hash(tuple(self._signed_certificate_timestamps)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SignedCertificateTimestamps): + return NotImplemented + + return ( + self._signed_certificate_timestamps + == other._signed_certificate_timestamps + ) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class OCSPNonce(ExtensionType): + oid = OCSPExtensionOID.NONCE + + def __init__(self, nonce: bytes) -> None: + if not isinstance(nonce, bytes): + raise TypeError("nonce must be bytes") + + self._nonce = nonce + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OCSPNonce): + return NotImplemented + + return self.nonce == other.nonce + + def __hash__(self) -> int: + return hash(self.nonce) + + def __repr__(self) -> str: + return f"" + + @property + def nonce(self) -> bytes: + return self._nonce + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class OCSPAcceptableResponses(ExtensionType): + oid = OCSPExtensionOID.ACCEPTABLE_RESPONSES + + def __init__(self, responses: Iterable[ObjectIdentifier]) -> None: + responses = list(responses) + if any(not isinstance(r, ObjectIdentifier) for r in responses): + raise TypeError("All responses must be ObjectIdentifiers") + + self._responses = responses + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OCSPAcceptableResponses): + return NotImplemented + + return self._responses == other._responses + + def __hash__(self) -> int: + return hash(tuple(self._responses)) + + def __repr__(self) -> str: + return f"" + + def __iter__(self) -> Iterator[ObjectIdentifier]: + return iter(self._responses) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class IssuingDistributionPoint(ExtensionType): + oid = ExtensionOID.ISSUING_DISTRIBUTION_POINT + + def __init__( + self, + full_name: Iterable[GeneralName] | None, + relative_name: RelativeDistinguishedName | None, + only_contains_user_certs: bool, + only_contains_ca_certs: bool, + only_some_reasons: frozenset[ReasonFlags] | None, + indirect_crl: bool, + only_contains_attribute_certs: bool, + ) -> None: + if full_name is not None: + full_name = list(full_name) + + if only_some_reasons and ( + not isinstance(only_some_reasons, frozenset) + or not all(isinstance(x, ReasonFlags) for x in only_some_reasons) + ): + raise TypeError( + "only_some_reasons must be None or frozenset of ReasonFlags" + ) + + if only_some_reasons and ( + ReasonFlags.unspecified in only_some_reasons + or ReasonFlags.remove_from_crl in only_some_reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in an " + "IssuingDistributionPoint" + ) + + if not ( + isinstance(only_contains_user_certs, bool) + and isinstance(only_contains_ca_certs, bool) + and isinstance(indirect_crl, bool) + and isinstance(only_contains_attribute_certs, bool) + ): + raise TypeError( + "only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl and only_contains_attribute_certs " + "must all be boolean." + ) + + # Per RFC5280 Section 5.2.5, the Issuing Distribution Point extension + # in a CRL can have only one of onlyContainsUserCerts, + # onlyContainsCACerts, onlyContainsAttributeCerts set to TRUE. + crl_constraints = [ + only_contains_user_certs, + only_contains_ca_certs, + only_contains_attribute_certs, + ] + + if len([x for x in crl_constraints if x]) > 1: + raise ValueError( + "Only one of the following can be set to True: " + "only_contains_user_certs, only_contains_ca_certs, " + "only_contains_attribute_certs" + ) + + if not any( + [ + only_contains_user_certs, + only_contains_ca_certs, + indirect_crl, + only_contains_attribute_certs, + full_name, + relative_name, + only_some_reasons, + ] + ): + raise ValueError( + "Cannot create empty extension: " + "if only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl, and only_contains_attribute_certs are all False" + ", then either full_name, relative_name, or only_some_reasons " + "must have a value." + ) + + self._only_contains_user_certs = only_contains_user_certs + self._only_contains_ca_certs = only_contains_ca_certs + self._indirect_crl = indirect_crl + self._only_contains_attribute_certs = only_contains_attribute_certs + self._only_some_reasons = only_some_reasons + self._full_name = full_name + self._relative_name = relative_name + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, IssuingDistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name + and self.relative_name == other.relative_name + and self.only_contains_user_certs == other.only_contains_user_certs + and self.only_contains_ca_certs == other.only_contains_ca_certs + and self.only_some_reasons == other.only_some_reasons + and self.indirect_crl == other.indirect_crl + and self.only_contains_attribute_certs + == other.only_contains_attribute_certs + ) + + def __hash__(self) -> int: + return hash( + ( + self.full_name, + self.relative_name, + self.only_contains_user_certs, + self.only_contains_ca_certs, + self.only_some_reasons, + self.indirect_crl, + self.only_contains_attribute_certs, + ) + ) + + @property + def full_name(self) -> list[GeneralName] | None: + return self._full_name + + @property + def relative_name(self) -> RelativeDistinguishedName | None: + return self._relative_name + + @property + def only_contains_user_certs(self) -> bool: + return self._only_contains_user_certs + + @property + def only_contains_ca_certs(self) -> bool: + return self._only_contains_ca_certs + + @property + def only_some_reasons( + self, + ) -> frozenset[ReasonFlags] | None: + return self._only_some_reasons + + @property + def indirect_crl(self) -> bool: + return self._indirect_crl + + @property + def only_contains_attribute_certs(self) -> bool: + return self._only_contains_attribute_certs + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class MSCertificateTemplate(ExtensionType): + oid = ExtensionOID.MS_CERTIFICATE_TEMPLATE + + def __init__( + self, + template_id: ObjectIdentifier, + major_version: int | None, + minor_version: int | None, + ) -> None: + if not isinstance(template_id, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + self._template_id = template_id + if ( + major_version is not None and not isinstance(major_version, int) + ) or ( + minor_version is not None and not isinstance(minor_version, int) + ): + raise TypeError( + "major_version and minor_version must be integers or None" + ) + self._major_version = major_version + self._minor_version = minor_version + + @property + def template_id(self) -> ObjectIdentifier: + return self._template_id + + @property + def major_version(self) -> int | None: + return self._major_version + + @property + def minor_version(self) -> int | None: + return self._minor_version + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MSCertificateTemplate): + return NotImplemented + + return ( + self.template_id == other.template_id + and self.major_version == other.major_version + and self.minor_version == other.minor_version + ) + + def __hash__(self) -> int: + return hash((self.template_id, self.major_version, self.minor_version)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class NamingAuthority: + def __init__( + self, + id: ObjectIdentifier | None, + url: str | None, + text: str | None, + ) -> None: + if id is not None and not isinstance(id, ObjectIdentifier): + raise TypeError("id must be an ObjectIdentifier") + + if url is not None and not isinstance(url, str): + raise TypeError("url must be a str") + + if text is not None and not isinstance(text, str): + raise TypeError("text must be a str") + + self._id = id + self._url = url + self._text = text + + @property + def id(self) -> ObjectIdentifier | None: + return self._id + + @property + def url(self) -> str | None: + return self._url + + @property + def text(self) -> str | None: + return self._text + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NamingAuthority): + return NotImplemented + + return ( + self.id == other.id + and self.url == other.url + and self.text == other.text + ) + + def __hash__(self) -> int: + return hash( + ( + self.id, + self.url, + self.text, + ) + ) + + +class ProfessionInfo: + def __init__( + self, + naming_authority: NamingAuthority | None, + profession_items: Iterable[str], + profession_oids: Iterable[ObjectIdentifier] | None, + registration_number: str | None, + add_profession_info: bytes | None, + ) -> None: + if naming_authority is not None and not isinstance( + naming_authority, NamingAuthority + ): + raise TypeError("naming_authority must be a NamingAuthority") + + profession_items = list(profession_items) + if not all(isinstance(item, str) for item in profession_items): + raise TypeError( + "Every item in the profession_items list must be a str" + ) + + if profession_oids is not None: + profession_oids = list(profession_oids) + if not all( + isinstance(oid, ObjectIdentifier) for oid in profession_oids + ): + raise TypeError( + "Every item in the profession_oids list must be an " + "ObjectIdentifier" + ) + + if registration_number is not None and not isinstance( + registration_number, str + ): + raise TypeError("registration_number must be a str") + + if add_profession_info is not None and not isinstance( + add_profession_info, bytes + ): + raise TypeError("add_profession_info must be bytes") + + self._naming_authority = naming_authority + self._profession_items = profession_items + self._profession_oids = profession_oids + self._registration_number = registration_number + self._add_profession_info = add_profession_info + + @property + def naming_authority(self) -> NamingAuthority | None: + return self._naming_authority + + @property + def profession_items(self) -> list[str]: + return self._profession_items + + @property + def profession_oids(self) -> list[ObjectIdentifier] | None: + return self._profession_oids + + @property + def registration_number(self) -> str | None: + return self._registration_number + + @property + def add_profession_info(self) -> bytes | None: + return self._add_profession_info + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ProfessionInfo): + return NotImplemented + + return ( + self.naming_authority == other.naming_authority + and self.profession_items == other.profession_items + and self.profession_oids == other.profession_oids + and self.registration_number == other.registration_number + and self.add_profession_info == other.add_profession_info + ) + + def __hash__(self) -> int: + if self.profession_oids is not None: + profession_oids = tuple(self.profession_oids) + else: + profession_oids = None + return hash( + ( + self.naming_authority, + tuple(self.profession_items), + profession_oids, + self.registration_number, + self.add_profession_info, + ) + ) + + +class Admission: + def __init__( + self, + admission_authority: GeneralName | None, + naming_authority: NamingAuthority | None, + profession_infos: Iterable[ProfessionInfo], + ) -> None: + if admission_authority is not None and not isinstance( + admission_authority, GeneralName + ): + raise TypeError("admission_authority must be a GeneralName") + + if naming_authority is not None and not isinstance( + naming_authority, NamingAuthority + ): + raise TypeError("naming_authority must be a NamingAuthority") + + profession_infos = list(profession_infos) + if not all( + isinstance(info, ProfessionInfo) for info in profession_infos + ): + raise TypeError( + "Every item in the profession_infos list must be a " + "ProfessionInfo" + ) + + self._admission_authority = admission_authority + self._naming_authority = naming_authority + self._profession_infos = profession_infos + + @property + def admission_authority(self) -> GeneralName | None: + return self._admission_authority + + @property + def naming_authority(self) -> NamingAuthority | None: + return self._naming_authority + + @property + def profession_infos(self) -> list[ProfessionInfo]: + return self._profession_infos + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Admission): + return NotImplemented + + return ( + self.admission_authority == other.admission_authority + and self.naming_authority == other.naming_authority + and self.profession_infos == other.profession_infos + ) + + def __hash__(self) -> int: + return hash( + ( + self.admission_authority, + self.naming_authority, + tuple(self.profession_infos), + ) + ) + + +class Admissions(ExtensionType): + oid = ExtensionOID.ADMISSIONS + + def __init__( + self, + authority: GeneralName | None, + admissions: Iterable[Admission], + ) -> None: + if authority is not None and not isinstance(authority, GeneralName): + raise TypeError("authority must be a GeneralName") + + admissions = list(admissions) + if not all( + isinstance(admission, Admission) for admission in admissions + ): + raise TypeError( + "Every item in the contents_of_admissions list must be an " + "Admission" + ) + + self._authority = authority + self._admissions = admissions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_admissions") + + @property + def authority(self) -> GeneralName | None: + return self._authority + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Admissions): + return NotImplemented + + return ( + self.authority == other.authority + and self._admissions == other._admissions + ) + + def __hash__(self) -> int: + return hash((self.authority, tuple(self._admissions))) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class UnrecognizedExtension(ExtensionType): + def __init__(self, oid: ObjectIdentifier, value: bytes) -> None: + if not isinstance(oid, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + self._oid = oid + self._value = value + + @property + def oid(self) -> ObjectIdentifier: # type: ignore[override] + return self._oid + + @property + def value(self) -> bytes: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnrecognizedExtension): + return NotImplemented + + return self.oid == other.oid and self.value == other.value + + def __hash__(self) -> int: + return hash((self.oid, self.value)) + + def public_bytes(self) -> bytes: + return self.value diff --git a/lib/cryptography/x509/general_name.py b/lib/cryptography/x509/general_name.py new file mode 100644 index 0000000..672f287 --- /dev/null +++ b/lib/cryptography/x509/general_name.py @@ -0,0 +1,281 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import ipaddress +import typing +from email.utils import parseaddr + +from cryptography.x509.name import Name +from cryptography.x509.oid import ObjectIdentifier + +_IPAddressTypes = typing.Union[ + ipaddress.IPv4Address, + ipaddress.IPv6Address, + ipaddress.IPv4Network, + ipaddress.IPv6Network, +] + + +class UnsupportedGeneralNameType(Exception): + pass + + +class GeneralName(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def value(self) -> typing.Any: + """ + Return the value of the object + """ + + +class RFC822Name(GeneralName): + def __init__(self, value: str) -> None: + if isinstance(value, str): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "RFC822Name values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + name, address = parseaddr(value) + if name or not address: + # parseaddr has found a name (e.g. Name ) or the entire + # value is an empty string. + raise ValueError("Invalid rfc822name value") + + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def _init_without_validation(cls, value: str) -> RFC822Name: + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RFC822Name): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class DNSName(GeneralName): + def __init__(self, value: str) -> None: + if isinstance(value, str): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "DNSName values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def _init_without_validation(cls, value: str) -> DNSName: + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DNSName): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class UniformResourceIdentifier(GeneralName): + def __init__(self, value: str) -> None: + if isinstance(value, str): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "URI values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def _init_without_validation(cls, value: str) -> UniformResourceIdentifier: + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UniformResourceIdentifier): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class DirectoryName(GeneralName): + def __init__(self, value: Name) -> None: + if not isinstance(value, Name): + raise TypeError("value must be a Name") + + self._value = value + + @property + def value(self) -> Name: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DirectoryName): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class RegisteredID(GeneralName): + def __init__(self, value: ObjectIdentifier) -> None: + if not isinstance(value, ObjectIdentifier): + raise TypeError("value must be an ObjectIdentifier") + + self._value = value + + @property + def value(self) -> ObjectIdentifier: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RegisteredID): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class IPAddress(GeneralName): + def __init__(self, value: _IPAddressTypes) -> None: + if not isinstance( + value, + ( + ipaddress.IPv4Address, + ipaddress.IPv6Address, + ipaddress.IPv4Network, + ipaddress.IPv6Network, + ), + ): + raise TypeError( + "value must be an instance of ipaddress.IPv4Address, " + "ipaddress.IPv6Address, ipaddress.IPv4Network, or " + "ipaddress.IPv6Network" + ) + + self._value = value + + @property + def value(self) -> _IPAddressTypes: + return self._value + + def _packed(self) -> bytes: + if isinstance( + self.value, (ipaddress.IPv4Address, ipaddress.IPv6Address) + ): + return self.value.packed + else: + return ( + self.value.network_address.packed + self.value.netmask.packed + ) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, IPAddress): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class OtherName(GeneralName): + def __init__(self, type_id: ObjectIdentifier, value: bytes) -> None: + if not isinstance(type_id, ObjectIdentifier): + raise TypeError("type_id must be an ObjectIdentifier") + if not isinstance(value, bytes): + raise TypeError("value must be a binary string") + + self._type_id = type_id + self._value = value + + @property + def type_id(self) -> ObjectIdentifier: + return self._type_id + + @property + def value(self) -> bytes: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OtherName): + return NotImplemented + + return self.type_id == other.type_id and self.value == other.value + + def __hash__(self) -> int: + return hash((self.type_id, self.value)) diff --git a/lib/cryptography/x509/name.py b/lib/cryptography/x509/name.py new file mode 100644 index 0000000..685f921 --- /dev/null +++ b/lib/cryptography/x509/name.py @@ -0,0 +1,476 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import binascii +import re +import sys +import typing +import warnings +from collections.abc import Iterable, Iterator + +from cryptography import utils +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.x509.oid import NameOID, ObjectIdentifier + + +class _ASN1Type(utils.Enum): + BitString = 3 + OctetString = 4 + UTF8String = 12 + NumericString = 18 + PrintableString = 19 + T61String = 20 + IA5String = 22 + UTCTime = 23 + GeneralizedTime = 24 + VisibleString = 26 + UniversalString = 28 + BMPString = 30 + + +_ASN1_TYPE_TO_ENUM = {i.value: i for i in _ASN1Type} +_NAMEOID_DEFAULT_TYPE: dict[ObjectIdentifier, _ASN1Type] = { + NameOID.COUNTRY_NAME: _ASN1Type.PrintableString, + NameOID.JURISDICTION_COUNTRY_NAME: _ASN1Type.PrintableString, + NameOID.SERIAL_NUMBER: _ASN1Type.PrintableString, + NameOID.DN_QUALIFIER: _ASN1Type.PrintableString, + NameOID.EMAIL_ADDRESS: _ASN1Type.IA5String, + NameOID.DOMAIN_COMPONENT: _ASN1Type.IA5String, +} + +# Type alias +_OidNameMap = typing.Mapping[ObjectIdentifier, str] +_NameOidMap = typing.Mapping[str, ObjectIdentifier] + +#: Short attribute names from RFC 4514: +#: https://tools.ietf.org/html/rfc4514#page-7 +_NAMEOID_TO_NAME: _OidNameMap = { + NameOID.COMMON_NAME: "CN", + NameOID.LOCALITY_NAME: "L", + NameOID.STATE_OR_PROVINCE_NAME: "ST", + NameOID.ORGANIZATION_NAME: "O", + NameOID.ORGANIZATIONAL_UNIT_NAME: "OU", + NameOID.COUNTRY_NAME: "C", + NameOID.STREET_ADDRESS: "STREET", + NameOID.DOMAIN_COMPONENT: "DC", + NameOID.USER_ID: "UID", +} +_NAME_TO_NAMEOID = {v: k for k, v in _NAMEOID_TO_NAME.items()} + +_NAMEOID_LENGTH_LIMIT = { + NameOID.COUNTRY_NAME: (2, 2), + NameOID.JURISDICTION_COUNTRY_NAME: (2, 2), + NameOID.COMMON_NAME: (1, 64), +} + + +def _escape_dn_value(val: str | bytes) -> str: + """Escape special characters in RFC4514 Distinguished Name value.""" + + if not val: + return "" + + # RFC 4514 Section 2.4 defines the value as being the # (U+0023) character + # followed by the hexadecimal encoding of the octets. + if isinstance(val, bytes): + return "#" + binascii.hexlify(val).decode("utf8") + + # See https://tools.ietf.org/html/rfc4514#section-2.4 + val = val.replace("\\", "\\\\") + val = val.replace('"', '\\"') + val = val.replace("+", "\\+") + val = val.replace(",", "\\,") + val = val.replace(";", "\\;") + val = val.replace("<", "\\<") + val = val.replace(">", "\\>") + val = val.replace("\0", "\\00") + + if val[0] in ("#", " "): + val = "\\" + val + if val[-1] == " ": + val = val[:-1] + "\\ " + + return val + + +def _unescape_dn_value(val: str) -> str: + if not val: + return "" + + # See https://tools.ietf.org/html/rfc4514#section-3 + + # special = escaped / SPACE / SHARP / EQUALS + # escaped = DQUOTE / PLUS / COMMA / SEMI / LANGLE / RANGLE + def sub(m): + val = m.group(1) + # Regular escape + if len(val) == 1: + return val + # Hex-value scape + return chr(int(val, 16)) + + return _RFC4514NameParser._PAIR_RE.sub(sub, val) + + +NameAttributeValueType = typing.TypeVar( + "NameAttributeValueType", + typing.Union[str, bytes], + str, + bytes, + covariant=True, +) + + +class NameAttribute(typing.Generic[NameAttributeValueType]): + def __init__( + self, + oid: ObjectIdentifier, + value: NameAttributeValueType, + _type: _ASN1Type | None = None, + *, + _validate: bool = True, + ) -> None: + if not isinstance(oid, ObjectIdentifier): + raise TypeError( + "oid argument must be an ObjectIdentifier instance." + ) + if _type == _ASN1Type.BitString: + if oid != NameOID.X500_UNIQUE_IDENTIFIER: + raise TypeError( + "oid must be X500_UNIQUE_IDENTIFIER for BitString type." + ) + if not isinstance(value, bytes): + raise TypeError("value must be bytes for BitString") + elif not isinstance(value, str): + raise TypeError("value argument must be a str") + + length_limits = _NAMEOID_LENGTH_LIMIT.get(oid) + if length_limits is not None: + min_length, max_length = length_limits + assert isinstance(value, str) + c_len = len(value.encode("utf8")) + if c_len < min_length or c_len > max_length: + msg = ( + f"Attribute's length must be >= {min_length} and " + f"<= {max_length}, but it was {c_len}" + ) + if _validate is True: + raise ValueError(msg) + else: + warnings.warn(msg, stacklevel=2) + + # The appropriate ASN1 string type varies by OID and is defined across + # multiple RFCs including 2459, 3280, and 5280. In general UTF8String + # is preferred (2459), but 3280 and 5280 specify several OIDs with + # alternate types. This means when we see the sentinel value we need + # to look up whether the OID has a non-UTF8 type. If it does, set it + # to that. Otherwise, UTF8! + if _type is None: + _type = _NAMEOID_DEFAULT_TYPE.get(oid, _ASN1Type.UTF8String) + + if not isinstance(_type, _ASN1Type): + raise TypeError("_type must be from the _ASN1Type enum") + + self._oid = oid + self._value: NameAttributeValueType = value + self._type: _ASN1Type = _type + + @property + def oid(self) -> ObjectIdentifier: + return self._oid + + @property + def value(self) -> NameAttributeValueType: + return self._value + + @property + def rfc4514_attribute_name(self) -> str: + """ + The short attribute name (for example "CN") if available, + otherwise the OID dotted string. + """ + return _NAMEOID_TO_NAME.get(self.oid, self.oid.dotted_string) + + def rfc4514_string( + self, attr_name_overrides: _OidNameMap | None = None + ) -> str: + """ + Format as RFC4514 Distinguished Name string. + + Use short attribute name if available, otherwise fall back to OID + dotted string. + """ + attr_name = ( + attr_name_overrides.get(self.oid) if attr_name_overrides else None + ) + if attr_name is None: + attr_name = self.rfc4514_attribute_name + + return f"{attr_name}={_escape_dn_value(self.value)}" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NameAttribute): + return NotImplemented + + return self.oid == other.oid and self.value == other.value + + def __hash__(self) -> int: + return hash((self.oid, self.value)) + + def __repr__(self) -> str: + return f"" + + +class RelativeDistinguishedName: + def __init__(self, attributes: Iterable[NameAttribute]): + attributes = list(attributes) + if not attributes: + raise ValueError("a relative distinguished name cannot be empty") + if not all(isinstance(x, NameAttribute) for x in attributes): + raise TypeError("attributes must be an iterable of NameAttribute") + + # Keep list and frozenset to preserve attribute order where it matters + self._attributes = attributes + self._attribute_set = frozenset(attributes) + + if len(self._attribute_set) != len(attributes): + raise ValueError("duplicate attributes are not allowed") + + def get_attributes_for_oid( + self, + oid: ObjectIdentifier, + ) -> list[NameAttribute[str | bytes]]: + return [i for i in self if i.oid == oid] + + def rfc4514_string( + self, attr_name_overrides: _OidNameMap | None = None + ) -> str: + """ + Format as RFC4514 Distinguished Name string. + + Within each RDN, attributes are joined by '+', although that is rarely + used in certificates. + """ + return "+".join( + attr.rfc4514_string(attr_name_overrides) + for attr in self._attributes + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RelativeDistinguishedName): + return NotImplemented + + return self._attribute_set == other._attribute_set + + def __hash__(self) -> int: + return hash(self._attribute_set) + + def __iter__(self) -> Iterator[NameAttribute]: + return iter(self._attributes) + + def __len__(self) -> int: + return len(self._attributes) + + def __repr__(self) -> str: + return f"" + + +class Name: + @typing.overload + def __init__(self, attributes: Iterable[NameAttribute]) -> None: ... + + @typing.overload + def __init__( + self, attributes: Iterable[RelativeDistinguishedName] + ) -> None: ... + + def __init__( + self, + attributes: Iterable[NameAttribute | RelativeDistinguishedName], + ) -> None: + attributes = list(attributes) + if all(isinstance(x, NameAttribute) for x in attributes): + self._attributes = [ + RelativeDistinguishedName([typing.cast(NameAttribute, x)]) + for x in attributes + ] + elif all(isinstance(x, RelativeDistinguishedName) for x in attributes): + self._attributes = typing.cast( + typing.List[RelativeDistinguishedName], attributes + ) + else: + raise TypeError( + "attributes must be a list of NameAttribute" + " or a list RelativeDistinguishedName" + ) + + @classmethod + def from_rfc4514_string( + cls, + data: str, + attr_name_overrides: _NameOidMap | None = None, + ) -> Name: + return _RFC4514NameParser(data, attr_name_overrides or {}).parse() + + def rfc4514_string( + self, attr_name_overrides: _OidNameMap | None = None + ) -> str: + """ + Format as RFC4514 Distinguished Name string. + For example 'CN=foobar.com,O=Foo Corp,C=US' + + An X.509 name is a two-level structure: a list of sets of attributes. + Each list element is separated by ',' and within each list element, set + elements are separated by '+'. The latter is almost never used in + real world certificates. According to RFC4514 section 2.1 the + RDNSequence must be reversed when converting to string representation. + """ + return ",".join( + attr.rfc4514_string(attr_name_overrides) + for attr in reversed(self._attributes) + ) + + def get_attributes_for_oid( + self, + oid: ObjectIdentifier, + ) -> list[NameAttribute[str | bytes]]: + return [i for i in self if i.oid == oid] + + @property + def rdns(self) -> list[RelativeDistinguishedName]: + return self._attributes + + def public_bytes(self, backend: typing.Any = None) -> bytes: + return rust_x509.encode_name_bytes(self) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Name): + return NotImplemented + + return self._attributes == other._attributes + + def __hash__(self) -> int: + # TODO: this is relatively expensive, if this looks like a bottleneck + # for you, consider optimizing! + return hash(tuple(self._attributes)) + + def __iter__(self) -> Iterator[NameAttribute]: + for rdn in self._attributes: + yield from rdn + + def __len__(self) -> int: + return sum(len(rdn) for rdn in self._attributes) + + def __repr__(self) -> str: + rdns = ",".join(attr.rfc4514_string() for attr in self._attributes) + return f"" + + +class _RFC4514NameParser: + _OID_RE = re.compile(r"(0|([1-9]\d*))(\.(0|([1-9]\d*)))+") + _DESCR_RE = re.compile(r"[a-zA-Z][a-zA-Z\d-]*") + + _PAIR = r"\\([\\ #=\"\+,;<>]|[\da-zA-Z]{2})" + _PAIR_RE = re.compile(_PAIR) + _LUTF1 = r"[\x01-\x1f\x21\x24-\x2A\x2D-\x3A\x3D\x3F-\x5B\x5D-\x7F]" + _SUTF1 = r"[\x01-\x21\x23-\x2A\x2D-\x3A\x3D\x3F-\x5B\x5D-\x7F]" + _TUTF1 = r"[\x01-\x1F\x21\x23-\x2A\x2D-\x3A\x3D\x3F-\x5B\x5D-\x7F]" + _UTFMB = rf"[\x80-{chr(sys.maxunicode)}]" + _LEADCHAR = rf"{_LUTF1}|{_UTFMB}" + _STRINGCHAR = rf"{_SUTF1}|{_UTFMB}" + _TRAILCHAR = rf"{_TUTF1}|{_UTFMB}" + _STRING_RE = re.compile( + rf""" + ( + ({_LEADCHAR}|{_PAIR}) + ( + ({_STRINGCHAR}|{_PAIR})* + ({_TRAILCHAR}|{_PAIR}) + )? + )? + """, + re.VERBOSE, + ) + _HEXSTRING_RE = re.compile(r"#([\da-zA-Z]{2})+") + + def __init__(self, data: str, attr_name_overrides: _NameOidMap) -> None: + self._data = data + self._idx = 0 + + self._attr_name_overrides = attr_name_overrides + + def _has_data(self) -> bool: + return self._idx < len(self._data) + + def _peek(self) -> str | None: + if self._has_data(): + return self._data[self._idx] + return None + + def _read_char(self, ch: str) -> None: + if self._peek() != ch: + raise ValueError + self._idx += 1 + + def _read_re(self, pat) -> str: + match = pat.match(self._data, pos=self._idx) + if match is None: + raise ValueError + val = match.group() + self._idx += len(val) + return val + + def parse(self) -> Name: + """ + Parses the `data` string and converts it to a Name. + + According to RFC4514 section 2.1 the RDNSequence must be + reversed when converting to string representation. So, when + we parse it, we need to reverse again to get the RDNs on the + correct order. + """ + + if not self._has_data(): + return Name([]) + + rdns = [self._parse_rdn()] + + while self._has_data(): + self._read_char(",") + rdns.append(self._parse_rdn()) + + return Name(reversed(rdns)) + + def _parse_rdn(self) -> RelativeDistinguishedName: + nas = [self._parse_na()] + while self._peek() == "+": + self._read_char("+") + nas.append(self._parse_na()) + + return RelativeDistinguishedName(nas) + + def _parse_na(self) -> NameAttribute: + try: + oid_value = self._read_re(self._OID_RE) + except ValueError: + name = self._read_re(self._DESCR_RE) + oid = self._attr_name_overrides.get( + name, _NAME_TO_NAMEOID.get(name) + ) + if oid is None: + raise ValueError + else: + oid = ObjectIdentifier(oid_value) + + self._read_char("=") + if self._peek() == "#": + value = self._read_re(self._HEXSTRING_RE) + value = binascii.unhexlify(value[1:]).decode() + else: + raw_value = self._read_re(self._STRING_RE) + value = _unescape_dn_value(raw_value) + + return NameAttribute(oid, value) diff --git a/lib/cryptography/x509/ocsp.py b/lib/cryptography/x509/ocsp.py new file mode 100644 index 0000000..f61ed80 --- /dev/null +++ b/lib/cryptography/x509/ocsp.py @@ -0,0 +1,379 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import datetime +from collections.abc import Iterable + +from cryptography import utils, x509 +from cryptography.hazmat.bindings._rust import ocsp +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPrivateKeyTypes, +) +from cryptography.x509.base import _reject_duplicate_extension + + +class OCSPResponderEncoding(utils.Enum): + HASH = "By Hash" + NAME = "By Name" + + +class OCSPResponseStatus(utils.Enum): + SUCCESSFUL = 0 + MALFORMED_REQUEST = 1 + INTERNAL_ERROR = 2 + TRY_LATER = 3 + SIG_REQUIRED = 5 + UNAUTHORIZED = 6 + + +_ALLOWED_HASHES = ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, +) + + +def _verify_algorithm(algorithm: hashes.HashAlgorithm) -> None: + if not isinstance(algorithm, _ALLOWED_HASHES): + raise ValueError( + "Algorithm must be SHA1, SHA224, SHA256, SHA384, or SHA512" + ) + + +class OCSPCertStatus(utils.Enum): + GOOD = 0 + REVOKED = 1 + UNKNOWN = 2 + + +class _SingleResponse: + def __init__( + self, + resp: tuple[x509.Certificate, x509.Certificate] | None, + resp_hash: tuple[bytes, bytes, int] | None, + algorithm: hashes.HashAlgorithm, + cert_status: OCSPCertStatus, + this_update: datetime.datetime, + next_update: datetime.datetime | None, + revocation_time: datetime.datetime | None, + revocation_reason: x509.ReasonFlags | None, + ): + _verify_algorithm(algorithm) + if not isinstance(this_update, datetime.datetime): + raise TypeError("this_update must be a datetime object") + if next_update is not None and not isinstance( + next_update, datetime.datetime + ): + raise TypeError("next_update must be a datetime object or None") + + self._resp = resp + self._resp_hash = resp_hash + self._algorithm = algorithm + self._this_update = this_update + self._next_update = next_update + + if not isinstance(cert_status, OCSPCertStatus): + raise TypeError( + "cert_status must be an item from the OCSPCertStatus enum" + ) + if cert_status is not OCSPCertStatus.REVOKED: + if revocation_time is not None: + raise ValueError( + "revocation_time can only be provided if the certificate " + "is revoked" + ) + if revocation_reason is not None: + raise ValueError( + "revocation_reason can only be provided if the certificate" + " is revoked" + ) + else: + if not isinstance(revocation_time, datetime.datetime): + raise TypeError("revocation_time must be a datetime object") + + if revocation_reason is not None and not isinstance( + revocation_reason, x509.ReasonFlags + ): + raise TypeError( + "revocation_reason must be an item from the ReasonFlags " + "enum or None" + ) + + self._cert_status = cert_status + self._revocation_time = revocation_time + self._revocation_reason = revocation_reason + + +OCSPRequest = ocsp.OCSPRequest +OCSPResponse = ocsp.OCSPResponse +OCSPSingleResponse = ocsp.OCSPSingleResponse + + +class OCSPRequestBuilder: + def __init__( + self, + request: tuple[ + x509.Certificate, x509.Certificate, hashes.HashAlgorithm + ] + | None = None, + request_hash: tuple[bytes, bytes, int, hashes.HashAlgorithm] + | None = None, + extensions: list[x509.Extension[x509.ExtensionType]] = [], + ) -> None: + self._request = request + self._request_hash = request_hash + self._extensions = extensions + + def add_certificate( + self, + cert: x509.Certificate, + issuer: x509.Certificate, + algorithm: hashes.HashAlgorithm, + ) -> OCSPRequestBuilder: + if self._request is not None or self._request_hash is not None: + raise ValueError("Only one certificate can be added to a request") + + _verify_algorithm(algorithm) + if not isinstance(cert, x509.Certificate) or not isinstance( + issuer, x509.Certificate + ): + raise TypeError("cert and issuer must be a Certificate") + + return OCSPRequestBuilder( + (cert, issuer, algorithm), self._request_hash, self._extensions + ) + + def add_certificate_by_hash( + self, + issuer_name_hash: bytes, + issuer_key_hash: bytes, + serial_number: int, + algorithm: hashes.HashAlgorithm, + ) -> OCSPRequestBuilder: + if self._request is not None or self._request_hash is not None: + raise ValueError("Only one certificate can be added to a request") + + if not isinstance(serial_number, int): + raise TypeError("serial_number must be an integer") + + _verify_algorithm(algorithm) + utils._check_bytes("issuer_name_hash", issuer_name_hash) + utils._check_bytes("issuer_key_hash", issuer_key_hash) + if algorithm.digest_size != len( + issuer_name_hash + ) or algorithm.digest_size != len(issuer_key_hash): + raise ValueError( + "issuer_name_hash and issuer_key_hash must be the same length " + "as the digest size of the algorithm" + ) + + return OCSPRequestBuilder( + self._request, + (issuer_name_hash, issuer_key_hash, serial_number, algorithm), + self._extensions, + ) + + def add_extension( + self, extval: x509.ExtensionType, critical: bool + ) -> OCSPRequestBuilder: + if not isinstance(extval, x509.ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = x509.Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return OCSPRequestBuilder( + self._request, self._request_hash, [*self._extensions, extension] + ) + + def build(self) -> OCSPRequest: + if self._request is None and self._request_hash is None: + raise ValueError("You must add a certificate before building") + + return ocsp.create_ocsp_request(self) + + +class OCSPResponseBuilder: + def __init__( + self, + response: _SingleResponse | None = None, + responder_id: tuple[x509.Certificate, OCSPResponderEncoding] + | None = None, + certs: list[x509.Certificate] | None = None, + extensions: list[x509.Extension[x509.ExtensionType]] = [], + ): + self._response = response + self._responder_id = responder_id + self._certs = certs + self._extensions = extensions + + def add_response( + self, + cert: x509.Certificate, + issuer: x509.Certificate, + algorithm: hashes.HashAlgorithm, + cert_status: OCSPCertStatus, + this_update: datetime.datetime, + next_update: datetime.datetime | None, + revocation_time: datetime.datetime | None, + revocation_reason: x509.ReasonFlags | None, + ) -> OCSPResponseBuilder: + if self._response is not None: + raise ValueError("Only one response per OCSPResponse.") + + if not isinstance(cert, x509.Certificate) or not isinstance( + issuer, x509.Certificate + ): + raise TypeError("cert and issuer must be a Certificate") + + singleresp = _SingleResponse( + (cert, issuer), + None, + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ) + return OCSPResponseBuilder( + singleresp, + self._responder_id, + self._certs, + self._extensions, + ) + + def add_response_by_hash( + self, + issuer_name_hash: bytes, + issuer_key_hash: bytes, + serial_number: int, + algorithm: hashes.HashAlgorithm, + cert_status: OCSPCertStatus, + this_update: datetime.datetime, + next_update: datetime.datetime | None, + revocation_time: datetime.datetime | None, + revocation_reason: x509.ReasonFlags | None, + ) -> OCSPResponseBuilder: + if self._response is not None: + raise ValueError("Only one response per OCSPResponse.") + + if not isinstance(serial_number, int): + raise TypeError("serial_number must be an integer") + + utils._check_bytes("issuer_name_hash", issuer_name_hash) + utils._check_bytes("issuer_key_hash", issuer_key_hash) + _verify_algorithm(algorithm) + if algorithm.digest_size != len( + issuer_name_hash + ) or algorithm.digest_size != len(issuer_key_hash): + raise ValueError( + "issuer_name_hash and issuer_key_hash must be the same length " + "as the digest size of the algorithm" + ) + + singleresp = _SingleResponse( + None, + (issuer_name_hash, issuer_key_hash, serial_number), + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ) + return OCSPResponseBuilder( + singleresp, + self._responder_id, + self._certs, + self._extensions, + ) + + def responder_id( + self, encoding: OCSPResponderEncoding, responder_cert: x509.Certificate + ) -> OCSPResponseBuilder: + if self._responder_id is not None: + raise ValueError("responder_id can only be set once") + if not isinstance(responder_cert, x509.Certificate): + raise TypeError("responder_cert must be a Certificate") + if not isinstance(encoding, OCSPResponderEncoding): + raise TypeError( + "encoding must be an element from OCSPResponderEncoding" + ) + + return OCSPResponseBuilder( + self._response, + (responder_cert, encoding), + self._certs, + self._extensions, + ) + + def certificates( + self, certs: Iterable[x509.Certificate] + ) -> OCSPResponseBuilder: + if self._certs is not None: + raise ValueError("certificates may only be set once") + certs = list(certs) + if len(certs) == 0: + raise ValueError("certs must not be an empty list") + if not all(isinstance(x, x509.Certificate) for x in certs): + raise TypeError("certs must be a list of Certificates") + return OCSPResponseBuilder( + self._response, + self._responder_id, + certs, + self._extensions, + ) + + def add_extension( + self, extval: x509.ExtensionType, critical: bool + ) -> OCSPResponseBuilder: + if not isinstance(extval, x509.ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = x509.Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return OCSPResponseBuilder( + self._response, + self._responder_id, + self._certs, + [*self._extensions, extension], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: hashes.HashAlgorithm | None, + ) -> OCSPResponse: + if self._response is None: + raise ValueError("You must add a response before signing") + if self._responder_id is None: + raise ValueError("You must add a responder_id before signing") + + return ocsp.create_ocsp_response( + OCSPResponseStatus.SUCCESSFUL, self, private_key, algorithm + ) + + @classmethod + def build_unsuccessful( + cls, response_status: OCSPResponseStatus + ) -> OCSPResponse: + if not isinstance(response_status, OCSPResponseStatus): + raise TypeError( + "response_status must be an item from OCSPResponseStatus" + ) + if response_status is OCSPResponseStatus.SUCCESSFUL: + raise ValueError("response_status cannot be SUCCESSFUL") + + return ocsp.create_ocsp_response(response_status, None, None, None) + + +load_der_ocsp_request = ocsp.load_der_ocsp_request +load_der_ocsp_response = ocsp.load_der_ocsp_response diff --git a/lib/cryptography/x509/oid.py b/lib/cryptography/x509/oid.py new file mode 100644 index 0000000..520fc7a --- /dev/null +++ b/lib/cryptography/x509/oid.py @@ -0,0 +1,37 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat._oid import ( + AttributeOID, + AuthorityInformationAccessOID, + CertificatePoliciesOID, + CRLEntryExtensionOID, + ExtendedKeyUsageOID, + ExtensionOID, + NameOID, + ObjectIdentifier, + OCSPExtensionOID, + OtherNameFormOID, + PublicKeyAlgorithmOID, + SignatureAlgorithmOID, + SubjectInformationAccessOID, +) + +__all__ = [ + "AttributeOID", + "AuthorityInformationAccessOID", + "CRLEntryExtensionOID", + "CertificatePoliciesOID", + "ExtendedKeyUsageOID", + "ExtensionOID", + "NameOID", + "OCSPExtensionOID", + "ObjectIdentifier", + "OtherNameFormOID", + "PublicKeyAlgorithmOID", + "SignatureAlgorithmOID", + "SubjectInformationAccessOID", +] diff --git a/lib/cryptography/x509/verification.py b/lib/cryptography/x509/verification.py new file mode 100644 index 0000000..2db4324 --- /dev/null +++ b/lib/cryptography/x509/verification.py @@ -0,0 +1,34 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.x509.general_name import DNSName, IPAddress + +__all__ = [ + "ClientVerifier", + "Criticality", + "ExtensionPolicy", + "Policy", + "PolicyBuilder", + "ServerVerifier", + "Store", + "Subject", + "VerificationError", + "VerifiedClient", +] + +Store = rust_x509.Store +Subject = typing.Union[DNSName, IPAddress] +VerifiedClient = rust_x509.VerifiedClient +ClientVerifier = rust_x509.ClientVerifier +ServerVerifier = rust_x509.ServerVerifier +PolicyBuilder = rust_x509.PolicyBuilder +Policy = rust_x509.Policy +ExtensionPolicy = rust_x509.ExtensionPolicy +Criticality = rust_x509.Criticality +VerificationError = rust_x509.VerificationError diff --git a/lib/ftp.py b/lib/ftp.py new file mode 100644 index 0000000..9c8f986 --- /dev/null +++ b/lib/ftp.py @@ -0,0 +1,190 @@ +import os +from ftplib import FTP +from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, DownloadColumn, TransferSpeedColumn +import lib.paramiko as paramiko +import logging +from pathlib import Path + + + +logging.basicConfig( + filename="sftp_debug.log", + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s" +) + +class SFTPSync: + def __init__(self, host, user, password, port=22): + self.host = host + self.user = user + self.password = password + self.port = int(port) + self.client = None + self.sftp = None + + def log(self, message): + """Helper to log to file and console if needed""" + logging.info(message) + + def connect(self): + try: + self.log(f"Initiating SSH connection to {self.host}:{self.port}") + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + self.log(f"Attempting login for user: {self.user}") + self.client.connect( + self.host, + port=self.port, + username=self.user, + password=self.password, + timeout=15, + allow_agent=False, # Prevents interference from local SSH agents + look_for_keys=False + ) + + self.log("SSH connection established. Opening SFTP session...") + self.sftp = self.client.open_sftp() + self.log("SFTP session opened successfully.") + return True + except paramiko.AuthenticationException: + self.log("Authentication failed: Check username or password.") + return "Auth failed: Invalid credentials." + except paramiko.SSHException as e: + self.log(f"SSH protocol error: {e}") + return f"SSH Error: {e}" + except Exception as e: + self.log(f"Unexpected connection error: {e}") + return f"Error: {e}" + + def upload_with_progress(self, local_path, remote_dir): + if not os.path.exists(local_path): + self.log(f"Local error: File {local_path} not found.") + return "Local file not found." + + file_size = os.path.getsize(local_path) + filename = os.path.basename(local_path) + # Ensure remote path uses forward slashes for Linux servers + remote_path = (Path(remote_dir) / filename).as_posix() + + self.log(f"Starting upload: {local_path} -> {remote_path} ({file_size} bytes)") + + try: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + ) as progress: + + task = progress.add_task(f"Uploading...", total=file_size) + + def callback(transferred, total): + progress.update(task, completed=transferred) + # Log every 25% to avoid bloating the log file + if transferred % (total // 4 + 1) < 8192: + logging.debug(f"Progress: {transferred}/{total} bytes") + + self.sftp.put(local_path, remote_path, callback=callback) + + self.log(f"Upload completed successfully: {filename}") + return True + except PermissionError: + self.log(f"Permission denied on server: Cannot write to {remote_dir}") + return "Server Error: Permission denied." + except Exception as e: + self.log(f"Upload failed mid-transfer: {e}") + return f"Upload error: {e}" + def upload_directory(self, local_dir, remote_root): + """Recursively uploads a directory, ensuring empty folders are created.""" + self.log(f"Scanning directory: {local_dir}") + + for root, dirs, files in os.walk(local_dir): + rel_path = os.path.relpath(root, local_dir) + + # Build the remote path + if rel_path == ".": + # This is the root folder itself (e.g., 'core') + remote_dir = (Path(remote_root) / Path(local_dir).name).as_posix() + else: + # These are subfolders (e.g., 'core/utils') + remote_dir = (Path(remote_root) / Path(local_dir).name / rel_path).as_posix() + + # --- THE FIX: Create folder even if 'files' is empty --- + try: + self.sftp.mkdir(remote_dir) + self.log(f"Created remote folder: {remote_dir}") + print(f"Created: {remote_dir}") # Feedback for the user + except IOError: + # Folder likely exists + self.log(f"Folder already exists: {remote_dir}") + + # Now upload files if there are any + for filename in files: + local_file = os.path.join(root, filename) + self.upload_with_progress(local_file, remote_dir) + + + def disconnect(self): + self.log("Closing SFTP and SSH connections.") + if self.sftp: self.sftp.close() + if self.client: self.client.close() + + + +class FTPSync: + def __init__(self, host, user, password, port=21): + self.host = host + self.user = user + self.password = password + self.port = int(port) + self.ftp = FTP() + + def connect(self): + try: + self.ftp.connect(self.host, self.port, timeout=10) + self.ftp.login(self.user, self.password) + self.ftp.set_pasv(True) # Passive mode is safer for most firewalls + return True + except Exception as e: + return f"Connection error: {e}" + + def upload_with_progress(self, local_path, remote_dir): + if not os.path.exists(local_path): + return "Local file not found." + + file_size = os.path.getsize(local_path) + filename = os.path.basename(local_path) + + try: + self.ftp.cwd(remote_dir) + + # Define the Rich Progress Bar layout + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + ) as progress: + + task = progress.add_task(f"Uploading {filename}...", total=file_size) + + with open(local_path, "rb") as f: + # Callback function called every time a chunk is sent + def callback(chunk): + progress.update(task, advance=len(chunk)) + + # 8KB is a standard buffer size for FTP + self.ftp.storbinary(f"STOR {filename}", f, blocksize=8192, callback=callback) + + return True + except Exception as e: + return f"Upload failed: {e}" + + def disconnect(self): + try: + self.ftp.quit() + except: + self.ftp.close() diff --git a/lib/invoke-2.2.1.dist-info/INSTALLER b/lib/invoke-2.2.1.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/invoke-2.2.1.dist-info/LICENSE b/lib/invoke-2.2.1.dist-info/LICENSE new file mode 100644 index 0000000..10e0dce --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2020 Jeff Forcier. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/invoke-2.2.1.dist-info/METADATA b/lib/invoke-2.2.1.dist-info/METADATA new file mode 100644 index 0000000..96599c1 --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/METADATA @@ -0,0 +1,77 @@ +Metadata-Version: 2.1 +Name: invoke +Version: 2.2.1 +Summary: Pythonic task execution +Home-page: https://pyinvoke.org +Author: Jeff Forcier +Author-email: jeff@bitprophet.org +License: BSD +Project-URL: Docs, https://docs.pyinvoke.org +Project-URL: Source, https://github.com/pyinvoke/invoke +Project-URL: Issues, https://github.com/pyinvoke/invoke/issues +Project-URL: Changelog, https://www.pyinvoke.org/changelog.html +Project-URL: CI, https://app.circleci.com/pipelines/github/pyinvoke/invoke +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: POSIX +Classifier: Operating System :: Unix +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Topic :: Software Development +Classifier: Topic :: Software Development :: Build Tools +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System :: Software Distribution +Classifier: Topic :: System :: Systems Administration +Requires-Python: >=3.6 +License-File: LICENSE + + +|version| |python| |license| |ci| |coverage| + +.. |version| image:: https://img.shields.io/pypi/v/invoke + :target: https://pypi.org/project/invoke/ + :alt: PyPI - Package Version +.. |python| image:: https://img.shields.io/pypi/pyversions/invoke + :target: https://pypi.org/project/invoke/ + :alt: PyPI - Python Version +.. |license| image:: https://img.shields.io/pypi/l/invoke + :target: https://github.com/pyinvoke/invoke/blob/main/LICENSE + :alt: PyPI - License +.. |ci| image:: https://img.shields.io/circleci/build/github/pyinvoke/invoke/main + :target: https://app.circleci.com/pipelines/github/pyinvoke/invoke + :alt: CircleCI +.. |coverage| image:: https://img.shields.io/codecov/c/gh/pyinvoke/invoke + :target: https://app.codecov.io/gh/pyinvoke/invoke + :alt: Codecov + +Welcome to Invoke! +================== + +Invoke is a Python (2.7 and 3.4+) library for managing shell-oriented +subprocesses and organizing executable Python code into CLI-invokable tasks. It +draws inspiration from various sources (``make``/``rake``, Fabric 1.x, etc) to +arrive at a powerful & clean feature set. + +To find out what's new in this version of Invoke, please see `the changelog +`_. + +The project maintainer keeps a `roadmap +`_ on his website. + + +For a high level introduction, including example code, please see `our main +project website `_; or for detailed API docs, see `the +versioned API website `_. diff --git a/lib/invoke-2.2.1.dist-info/RECORD b/lib/invoke-2.2.1.dist-info/RECORD new file mode 100644 index 0000000..bea5966 --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/RECORD @@ -0,0 +1,109 @@ +../../bin/inv,sha256=funfUXpVM3BJWCQYR4gEDKGA5G-HwnG-f66iGHAG7pU,192 +../../bin/invoke,sha256=funfUXpVM3BJWCQYR4gEDKGA5G-HwnG-f66iGHAG7pU,192 +invoke-2.2.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +invoke-2.2.1.dist-info/LICENSE,sha256=eSL5f4lvHRYeHr9HCD4wSiNjg2GyVcaQK15PNO7aDa0,1314 +invoke-2.2.1.dist-info/METADATA,sha256=_T4J728FxhvzvMX4Aw6nXnbMFZHU7VoPnoTA45HDVfw,3270 +invoke-2.2.1.dist-info/RECORD,, +invoke-2.2.1.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91 +invoke-2.2.1.dist-info/entry_points.txt,sha256=fz7lDPipw_V1nnuk41CNxa739dBpJ8TO9cpKuUXVDPs,81 +invoke-2.2.1.dist-info/top_level.txt,sha256=ZlTlAVMd8lzn3sXyAhCRi0LNslCasA7rlucRHq9w79w,7 +invoke/__init__.py,sha256=XBXrLV9I81Nq6ELEoN_XAE4vTNmM6Big-1eWt1dQH3k,2229 +invoke/__main__.py,sha256=nwyePAl8dedcetg1CiEQNC0-CHESN0DJy5tK2YEqGeo,47 +invoke/__pycache__/__init__.cpython-314.pyc,, +invoke/__pycache__/__main__.cpython-314.pyc,, +invoke/__pycache__/_version.cpython-314.pyc,, +invoke/__pycache__/collection.cpython-314.pyc,, +invoke/__pycache__/config.cpython-314.pyc,, +invoke/__pycache__/context.cpython-314.pyc,, +invoke/__pycache__/env.cpython-314.pyc,, +invoke/__pycache__/exceptions.cpython-314.pyc,, +invoke/__pycache__/executor.cpython-314.pyc,, +invoke/__pycache__/loader.cpython-314.pyc,, +invoke/__pycache__/main.cpython-314.pyc,, +invoke/__pycache__/program.cpython-314.pyc,, +invoke/__pycache__/runners.cpython-314.pyc,, +invoke/__pycache__/tasks.cpython-314.pyc,, +invoke/__pycache__/terminals.cpython-314.pyc,, +invoke/__pycache__/util.cpython-314.pyc,, +invoke/__pycache__/watchers.cpython-314.pyc,, +invoke/_version.py,sha256=woNEa1C-mSdvI4h8gKEotsc-bB7rAF7WZDpFO8oV7C0,80 +invoke/collection.py,sha256=0Qv9bnfKeUbkAaTLINxrzG9MBrhjwrhd7Rv6v6WgdKI,23060 +invoke/completion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +invoke/completion/__pycache__/__init__.cpython-314.pyc,, +invoke/completion/__pycache__/complete.cpython-314.pyc,, +invoke/completion/bash.completion,sha256=OvJRVRvoW7aMDf5qMKidi8x-ND-KkIM5JS5-6iJ3eHE,1356 +invoke/completion/complete.py,sha256=j3Tv2oNrKyuQHPi5tavAUYN5vo8v4vABNl_T9t_0YOY,5222 +invoke/completion/fish.completion,sha256=G28g1dpA-Q9-Q6KmlcGd_6XMw8h7ExHjAJ6ny04ufr8,382 +invoke/completion/zsh.completion,sha256=YiekKS7ZDCIqLmlC31Ht9H95S2n8aNsfnn4bjBcFRLQ,1429 +invoke/config.py,sha256=NGBAmHWoUI_PHBxTAZNtRt2X4P09ouWRRzBdVj3MTac,49653 +invoke/context.py,sha256=k1KbmT2hDk846Su4wUuElZ4nXIYXYjqeIxuS0g_0_lc,25486 +invoke/env.py,sha256=T_ejssb-lmdZwPHFjJGH62KnhQKim-ziiWBxM-8ejW8,4394 +invoke/exceptions.py,sha256=e5vwp9cJS8teQK30WLY4ICKU8ateSUexU6BuFnEllDA,12227 +invoke/executor.py,sha256=T_iQ6uNTC2Z_LB7WWVxV4ab0jvYVKoKO_3hCFQ3FBlA,8855 +invoke/loader.py,sha256=C5dtubZRfjEdaF8WUi3blZjQxv8yGCtvipXrMMMSG4Y,6005 +invoke/main.py,sha256=njYYo2anK4krAE4mCV9Z9I1bRJFaOCp-UBL1vRE-Wq4,235 +invoke/parser/__init__.py,sha256=HpSB_sx2aZrCOUy5Cl_LMyfCBuGwrB9-d1OS4821Dzs,181 +invoke/parser/__pycache__/__init__.cpython-314.pyc,, +invoke/parser/__pycache__/argument.cpython-314.pyc,, +invoke/parser/__pycache__/context.cpython-314.pyc,, +invoke/parser/__pycache__/parser.cpython-314.pyc,, +invoke/parser/argument.py,sha256=eyIGaOtjyEz2M2BMiIz0cvn8J6zinpUhgxoXexJzHpk,6045 +invoke/parser/context.py,sha256=NaqvcN4E9W-7Ems48tJzIedBqcrZnbR41TA_rSE5JGo,9815 +invoke/parser/parser.py,sha256=g99NcgzHGfEjsNzwyFmAo_MRhqlGf82SXXBzOZXr6Rw,19809 +invoke/program.py,sha256=oBrtCjtAdycz3UDCJM5vTrdHE37SL7UPw-AzzcwWsRU,38177 +invoke/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +invoke/runners.py,sha256=4WJHoj8UFggD6e-F12hFsTiHTHBO5fHEZZAIduFNWNo,65509 +invoke/tasks.py,sha256=FBj_EStMXfQ0Y9Nw8gPctRxwKp4Lya8QdnrDklVVJVk,19946 +invoke/terminals.py,sha256=COszJimzGyA7FcPo8kgOmVDl4v5kciU2w3-Ts3LOo4g,8148 +invoke/util.py,sha256=NKjJoZA4Tb8VzAvr58gIAWvQBaTFDKRg3bjRSxdeMcs,10018 +invoke/vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +invoke/vendor/__pycache__/__init__.cpython-314.pyc,, +invoke/vendor/fluidity/__init__.py,sha256=f5SF3sMYo71808nX-9bJt0pb5-BCZ2srikX1aCqczpw,196 +invoke/vendor/fluidity/__pycache__/__init__.cpython-314.pyc,, +invoke/vendor/fluidity/__pycache__/backwardscompat.cpython-314.pyc,, +invoke/vendor/fluidity/__pycache__/machine.cpython-314.pyc,, +invoke/vendor/fluidity/backwardscompat.py,sha256=P_qC1dIhIHq6LQYDGxWed-V8o3viMcPOYv4mx0NY52U,135 +invoke/vendor/fluidity/machine.py,sha256=9ZhzmAg-3Q_5jCWHOih1WtoTMAKhkgT2EePkXW-258M,8686 +invoke/vendor/lexicon/__init__.py,sha256=iFssP2WfLyW5pmp4EDFvd_63zvAyZ_7YKh2nrfbvlHY,1133 +invoke/vendor/lexicon/__pycache__/__init__.cpython-314.pyc,, +invoke/vendor/lexicon/__pycache__/_version.cpython-314.pyc,, +invoke/vendor/lexicon/__pycache__/alias_dict.cpython-314.pyc,, +invoke/vendor/lexicon/__pycache__/attribute_dict.cpython-314.pyc,, +invoke/vendor/lexicon/_version.py,sha256=GFgEreRHgT-8UlwM974VYEId1Wj19fqMAk-hpObAjeI,80 +invoke/vendor/lexicon/alias_dict.py,sha256=gsSlVPy3wt5HGJSzjhdBiMVcPnIuVLU3Sl1VbybVs5w,3223 +invoke/vendor/lexicon/attribute_dict.py,sha256=j2myombp3ZH3fTOy4RhVAyZXrP_-1iwVqdenEvY2u-Q,407 +invoke/vendor/yaml/__init__.py,sha256=gfp2CbRVhzknghkiiJD2l6Z0pI-mv_iZHPSJ4aj0-nY,13170 +invoke/vendor/yaml/__pycache__/__init__.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/composer.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/constructor.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/cyaml.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/dumper.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/emitter.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/error.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/events.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/loader.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/nodes.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/parser.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/reader.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/representer.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/resolver.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/scanner.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/serializer.cpython-314.pyc,, +invoke/vendor/yaml/__pycache__/tokens.cpython-314.pyc,, +invoke/vendor/yaml/composer.py,sha256=_Ko30Wr6eDWUeUpauUGT3Lcg9QPBnOPVlTnIMRGJ9FM,4883 +invoke/vendor/yaml/constructor.py,sha256=kNgkfaeLUkwQYY_Q6Ff1Tz2XVw_pG1xVE9Ak7z-viLA,28639 +invoke/vendor/yaml/cyaml.py,sha256=6ZrAG9fAYvdVe2FK_w0hmXoG7ZYsoYUwapG8CiC72H0,3851 +invoke/vendor/yaml/dumper.py,sha256=PLctZlYwZLp7XmeUdwRuv4nYOZ2UBnDIUy8-lKfLF-o,2837 +invoke/vendor/yaml/emitter.py,sha256=jghtaU7eFwg31bG0B7RZea_29Adi9CKmXq_QjgQpCkQ,43006 +invoke/vendor/yaml/error.py,sha256=Ah9z-toHJUbE9j-M8YpxgSRM5CgLCcwVzJgLLRF2Fxo,2533 +invoke/vendor/yaml/events.py,sha256=50_TksgQiE4up-lKo_V-nBy-tAIxkIPQxY5qDhKCeHw,2445 +invoke/vendor/yaml/loader.py,sha256=UVa-zIqmkFSCIYq_PgSGm4NSJttHY2Rf_zQ4_b1fHN0,2061 +invoke/vendor/yaml/nodes.py,sha256=gPKNj8pKCdh2d4gr3gIYINnPOaOxGhJAUiYhGRnPE84,1440 +invoke/vendor/yaml/parser.py,sha256=ilWp5vvgoHFGzvOZDItFoGjD6D42nhlZrZyjAwa0oJo,25495 +invoke/vendor/yaml/reader.py,sha256=0dmzirOiDG4Xo41RnuQS7K9rkY3xjHiVasfDMNTqCNw,6794 +invoke/vendor/yaml/representer.py,sha256=82UM3ZxUQKqsKAF4ltWOxCS6jGPIFtXpGs7mvqyv4Xs,14184 +invoke/vendor/yaml/resolver.py,sha256=Z1W8AOMA6Proy4gIO2OhUO4IPS_bFNAl0Ca3rwChpPg,8999 +invoke/vendor/yaml/scanner.py,sha256=KeQIKGNlSyPE8QDwionHxy9CgbqE5teJEz05FR9-nAg,51277 +invoke/vendor/yaml/serializer.py,sha256=ChuFgmhU01hj4xgI8GaKv6vfM2Bujwa9i7d2FAHj7cA,4165 +invoke/vendor/yaml/tokens.py,sha256=lTQIzSVw8Mg9wv459-TjiOQe6wVziqaRlqX2_89rp54,2573 +invoke/watchers.py,sha256=E8CB8ikiXFw-15snsFnbZjwq-exFMqNqFOqkGgcsqLs,5097 diff --git a/lib/invoke-2.2.1.dist-info/WHEEL b/lib/invoke-2.2.1.dist-info/WHEEL new file mode 100644 index 0000000..1f64615 --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (75.3.2) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/invoke-2.2.1.dist-info/entry_points.txt b/lib/invoke-2.2.1.dist-info/entry_points.txt new file mode 100644 index 0000000..56faa37 --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +inv = invoke.main:program.run +invoke = invoke.main:program.run diff --git a/lib/invoke-2.2.1.dist-info/top_level.txt b/lib/invoke-2.2.1.dist-info/top_level.txt new file mode 100644 index 0000000..460820d --- /dev/null +++ b/lib/invoke-2.2.1.dist-info/top_level.txt @@ -0,0 +1 @@ +invoke diff --git a/lib/invoke/__init__.py b/lib/invoke/__init__.py new file mode 100644 index 0000000..b707267 --- /dev/null +++ b/lib/invoke/__init__.py @@ -0,0 +1,70 @@ +from typing import Any, Optional + +from ._version import __version_info__, __version__ # noqa +from .collection import Collection # noqa +from .config import Config # noqa +from .context import Context, MockContext # noqa +from .exceptions import ( # noqa + AmbiguousEnvVar, + AuthFailure, + CollectionNotFound, + Exit, + ParseError, + PlatformError, + ResponseNotAccepted, + SubprocessPipeError, + ThreadException, + UncastableEnvVar, + UnexpectedExit, + UnknownFileType, + UnpicklableConfigMember, + WatcherError, + CommandTimedOut, +) +from .executor import Executor # noqa +from .loader import FilesystemLoader # noqa +from .parser import Argument, Parser, ParserContext, ParseResult # noqa +from .program import Program # noqa +from .runners import Runner, Local, Failure, Result, Promise # noqa +from .tasks import task, call, Call, Task # noqa +from .terminals import pty_size # noqa +from .watchers import FailingResponder, Responder, StreamWatcher # noqa + + +def run(command: str, **kwargs: Any) -> Optional[Result]: + """ + Run ``command`` in a subprocess and return a `.Result` object. + + See `.Runner.run` for API details. + + .. note:: + This function is a convenience wrapper around Invoke's `.Context` and + `.Runner` APIs. + + Specifically, it creates an anonymous `.Context` instance and calls its + `~.Context.run` method, which in turn defaults to using a `.Local` + runner subclass for command execution. + + .. versionadded:: 1.0 + """ + return Context().run(command, **kwargs) + + +def sudo(command: str, **kwargs: Any) -> Optional[Result]: + """ + Run ``command`` in a ``sudo`` subprocess and return a `.Result` object. + + See `.Context.sudo` for API details, such as the ``password`` kwarg. + + .. note:: + This function is a convenience wrapper around Invoke's `.Context` and + `.Runner` APIs. + + Specifically, it creates an anonymous `.Context` instance and calls its + `~.Context.sudo` method, which in turn defaults to using a `.Local` + runner subclass for command execution (plus sudo-related bits & + pieces). + + .. versionadded:: 1.4 + """ + return Context().sudo(command, **kwargs) diff --git a/lib/invoke/__main__.py b/lib/invoke/__main__.py new file mode 100644 index 0000000..2c8118c --- /dev/null +++ b/lib/invoke/__main__.py @@ -0,0 +1,3 @@ +from invoke.main import program + +program.run() diff --git a/lib/invoke/__pycache__/__init__.cpython-314.pyc b/lib/invoke/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..c679376 Binary files /dev/null and b/lib/invoke/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/__main__.cpython-314.pyc b/lib/invoke/__pycache__/__main__.cpython-314.pyc new file mode 100644 index 0000000..2cc5008 Binary files /dev/null and b/lib/invoke/__pycache__/__main__.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/_version.cpython-314.pyc b/lib/invoke/__pycache__/_version.cpython-314.pyc new file mode 100644 index 0000000..0cdeefb Binary files /dev/null and b/lib/invoke/__pycache__/_version.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/collection.cpython-314.pyc b/lib/invoke/__pycache__/collection.cpython-314.pyc new file mode 100644 index 0000000..5c7f1d1 Binary files /dev/null and b/lib/invoke/__pycache__/collection.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/config.cpython-314.pyc b/lib/invoke/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000..529db73 Binary files /dev/null and b/lib/invoke/__pycache__/config.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/context.cpython-314.pyc b/lib/invoke/__pycache__/context.cpython-314.pyc new file mode 100644 index 0000000..7f74be6 Binary files /dev/null and b/lib/invoke/__pycache__/context.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/env.cpython-314.pyc b/lib/invoke/__pycache__/env.cpython-314.pyc new file mode 100644 index 0000000..c25e722 Binary files /dev/null and b/lib/invoke/__pycache__/env.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/exceptions.cpython-314.pyc b/lib/invoke/__pycache__/exceptions.cpython-314.pyc new file mode 100644 index 0000000..798e082 Binary files /dev/null and b/lib/invoke/__pycache__/exceptions.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/executor.cpython-314.pyc b/lib/invoke/__pycache__/executor.cpython-314.pyc new file mode 100644 index 0000000..3a68e6e Binary files /dev/null and b/lib/invoke/__pycache__/executor.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/loader.cpython-314.pyc b/lib/invoke/__pycache__/loader.cpython-314.pyc new file mode 100644 index 0000000..32f96c8 Binary files /dev/null and b/lib/invoke/__pycache__/loader.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/main.cpython-314.pyc b/lib/invoke/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000..15ba6de Binary files /dev/null and b/lib/invoke/__pycache__/main.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/program.cpython-314.pyc b/lib/invoke/__pycache__/program.cpython-314.pyc new file mode 100644 index 0000000..d1ae76f Binary files /dev/null and b/lib/invoke/__pycache__/program.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/runners.cpython-314.pyc b/lib/invoke/__pycache__/runners.cpython-314.pyc new file mode 100644 index 0000000..67bc34a Binary files /dev/null and b/lib/invoke/__pycache__/runners.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/tasks.cpython-314.pyc b/lib/invoke/__pycache__/tasks.cpython-314.pyc new file mode 100644 index 0000000..14866d2 Binary files /dev/null and b/lib/invoke/__pycache__/tasks.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/terminals.cpython-314.pyc b/lib/invoke/__pycache__/terminals.cpython-314.pyc new file mode 100644 index 0000000..5feceaf Binary files /dev/null and b/lib/invoke/__pycache__/terminals.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/util.cpython-314.pyc b/lib/invoke/__pycache__/util.cpython-314.pyc new file mode 100644 index 0000000..001f81d Binary files /dev/null and b/lib/invoke/__pycache__/util.cpython-314.pyc differ diff --git a/lib/invoke/__pycache__/watchers.cpython-314.pyc b/lib/invoke/__pycache__/watchers.cpython-314.pyc new file mode 100644 index 0000000..9f7fa58 Binary files /dev/null and b/lib/invoke/__pycache__/watchers.cpython-314.pyc differ diff --git a/lib/invoke/_version.py b/lib/invoke/_version.py new file mode 100644 index 0000000..14efac7 --- /dev/null +++ b/lib/invoke/_version.py @@ -0,0 +1,2 @@ +__version_info__ = (2, 2, 1) +__version__ = ".".join(map(str, __version_info__)) diff --git a/lib/invoke/collection.py b/lib/invoke/collection.py new file mode 100644 index 0000000..23dcff9 --- /dev/null +++ b/lib/invoke/collection.py @@ -0,0 +1,608 @@ +import copy +from types import ModuleType +from typing import Any, Callable, Dict, List, Optional, Tuple + +from .util import Lexicon, helpline + +from .config import merge_dicts, copy_dict +from .parser import Context as ParserContext +from .tasks import Task + + +class Collection: + """ + A collection of executable tasks. See :doc:`/concepts/namespaces`. + + .. versionadded:: 1.0 + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """ + Create a new task collection/namespace. + + `.Collection` offers a set of methods for building a collection of + tasks from scratch, plus a convenient constructor wrapping said API. + + In either case: + + * The first positional argument may be a string, which (if given) is + used as the collection's default name when performing namespace + lookups; + * A ``loaded_from`` keyword argument may be given, which sets metadata + indicating the filesystem path the collection was loaded from. This + is used as a guide when loading per-project :ref:`configuration files + `. + * An ``auto_dash_names`` kwarg may be given, controlling whether task + and collection names have underscores turned to dashes in most cases; + it defaults to ``True`` but may be set to ``False`` to disable. + + The CLI machinery will pass in the value of the + ``tasks.auto_dash_names`` config value to this kwarg. + + **The method approach** + + May initialize with no arguments and use methods (e.g. + `.add_task`/`.add_collection`) to insert objects:: + + c = Collection() + c.add_task(some_task) + + If an initial string argument is given, it is used as the default name + for this collection, should it be inserted into another collection as a + sub-namespace:: + + docs = Collection('docs') + docs.add_task(doc_task) + ns = Collection() + ns.add_task(top_level_task) + ns.add_collection(docs) + # Valid identifiers are now 'top_level_task' and 'docs.doc_task' + # (assuming the task objects were actually named the same as the + # variables we're using :)) + + For details, see the API docs for the rest of the class. + + **The constructor approach** + + All ``*args`` given to `.Collection` (besides the abovementioned + optional positional 'name' argument and ``loaded_from`` kwarg) are + expected to be `.Task` or `.Collection` instances which will be passed + to `.add_task`/`.add_collection` as appropriate. Module objects are + also valid (as they are for `.add_collection`). For example, the below + snippet results in the same two task identifiers as the one above:: + + ns = Collection(top_level_task, Collection('docs', doc_task)) + + If any ``**kwargs`` are given, the keywords are used as the initial + name arguments for the respective values:: + + ns = Collection( + top_level_task=some_other_task, + docs=Collection(doc_task) + ) + + That's exactly equivalent to:: + + docs = Collection(doc_task) + ns = Collection() + ns.add_task(some_other_task, 'top_level_task') + ns.add_collection(docs, 'docs') + + See individual methods' API docs for details. + """ + # Initialize + self.tasks = Lexicon() + self.collections = Lexicon() + self.default: Optional[str] = None + self.name = None + self._configuration: Dict[str, Any] = {} + # Specific kwargs if applicable + self.loaded_from = kwargs.pop("loaded_from", None) + self.auto_dash_names = kwargs.pop("auto_dash_names", None) + # splat-kwargs version of default value (auto_dash_names=True) + if self.auto_dash_names is None: + self.auto_dash_names = True + # Name if applicable + _args = list(args) + if _args and isinstance(args[0], str): + self.name = self.transform(_args.pop(0)) + # Dispatch args/kwargs + for arg in _args: + self._add_object(arg) + # Dispatch kwargs + for name, obj in kwargs.items(): + self._add_object(obj, name) + + def _add_object(self, obj: Any, name: Optional[str] = None) -> None: + method: Callable + if isinstance(obj, Task): + method = self.add_task + elif isinstance(obj, (Collection, ModuleType)): + method = self.add_collection + else: + raise TypeError("No idea how to insert {!r}!".format(type(obj))) + method(obj, name=name) + + def __repr__(self) -> str: + task_names = list(self.tasks.keys()) + collections = ["{}...".format(x) for x in self.collections.keys()] + return "".format( + self.name, ", ".join(sorted(task_names) + sorted(collections)) + ) + + def __eq__(self, other: object) -> bool: + if isinstance(other, Collection): + return ( + self.name == other.name + and self.tasks == other.tasks + and self.collections == other.collections + ) + return False + + def __bool__(self) -> bool: + return bool(self.task_names) + + @classmethod + def from_module( + cls, + module: ModuleType, + name: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + loaded_from: Optional[str] = None, + auto_dash_names: Optional[bool] = None, + ) -> "Collection": + """ + Return a new `.Collection` created from ``module``. + + Inspects ``module`` for any `.Task` instances and adds them to a new + `.Collection`, returning it. If any explicit namespace collections + exist (named ``ns`` or ``namespace``) a copy of that collection object + is preferentially loaded instead. + + When the implicit/default collection is generated, it will be named + after the module's ``__name__`` attribute, or its last dotted section + if it's a submodule. (I.e. it should usually map to the actual ``.py`` + filename.) + + Explicitly given collections will only be given that module-derived + name if they don't already have a valid ``.name`` attribute. + + If the module has a docstring (``__doc__``) it is copied onto the + resulting `.Collection` (and used for display in help, list etc + output.) + + :param str name: + A string, which if given will override any automatically derived + collection name (or name set on the module's root namespace, if it + has one.) + + :param dict config: + Used to set config options on the newly created `.Collection` + before returning it (saving you a call to `.configure`.) + + If the imported module had a root namespace object, ``config`` is + merged on top of it (i.e. overriding any conflicts.) + + :param str loaded_from: + Identical to the same-named kwarg from the regular class + constructor - should be the path where the module was + found. + + :param bool auto_dash_names: + Identical to the same-named kwarg from the regular class + constructor - determines whether emitted names are auto-dashed. + + .. versionadded:: 1.0 + """ + module_name = module.__name__.split(".")[-1] + + def instantiate(obj_name: Optional[str] = None) -> "Collection": + # Explicitly given name wins over root ns name (if applicable), + # which wins over actual module name. + args = [name or obj_name or module_name] + kwargs = dict( + loaded_from=loaded_from, auto_dash_names=auto_dash_names + ) + instance = cls(*args, **kwargs) + instance.__doc__ = module.__doc__ + return instance + + # See if the module provides a default NS to use in lieu of creating + # our own collection. + for candidate in ("ns", "namespace"): + obj = getattr(module, candidate, None) + if obj and isinstance(obj, Collection): + # TODO: make this into Collection.clone() or similar? + ret = instantiate(obj_name=obj.name) + ret.tasks = ret._transform_lexicon(obj.tasks) + ret.collections = ret._transform_lexicon(obj.collections) + ret.default = ( + ret.transform(obj.default) if obj.default else None + ) + # Explicitly given config wins over root ns config + obj_config = copy_dict(obj._configuration) + if config: + merge_dicts(obj_config, config) + ret._configuration = obj_config + return ret + # Failing that, make our own collection from the module's tasks. + tasks = filter(lambda x: isinstance(x, Task), vars(module).values()) + # Again, explicit name wins over implicit one from module path + collection = instantiate() + for task in tasks: + collection.add_task(task) + if config: + collection.configure(config) + return collection + + def add_task( + self, + task: "Task", + name: Optional[str] = None, + aliases: Optional[Tuple[str, ...]] = None, + default: Optional[bool] = None, + ) -> None: + """ + Add `.Task` ``task`` to this collection. + + :param task: The `.Task` object to add to this collection. + + :param name: + Optional string name to bind to (overrides the task's own + self-defined ``name`` attribute and/or any Python identifier (i.e. + ``.func_name``.) + + :param aliases: + Optional iterable of additional names to bind the task as, on top + of the primary name. These will be used in addition to any aliases + the task itself declares internally. + + :param default: Whether this task should be the collection default. + + .. versionadded:: 1.0 + """ + if name is None: + if task.name: + name = task.name + # XXX https://github.com/python/mypy/issues/1424 + elif hasattr(task.body, "func_name"): + name = task.body.func_name # type: ignore + elif hasattr(task.body, "__name__"): + name = task.__name__ + else: + raise ValueError("Could not obtain a name for this task!") + name = self.transform(name) + if name in self.collections: + err = "Name conflict: this collection has a sub-collection named {!r} already" # noqa + raise ValueError(err.format(name)) + self.tasks[name] = task + for alias in list(task.aliases) + list(aliases or []): + self.tasks.alias(self.transform(alias), to=name) + if default is True or (default is None and task.is_default): + self._check_default_collision(name) + self.default = name + + def add_collection( + self, + coll: "Collection", + name: Optional[str] = None, + default: Optional[bool] = None, + ) -> None: + """ + Add `.Collection` ``coll`` as a sub-collection of this one. + + :param coll: The `.Collection` to add. + + :param str name: + The name to attach the collection as. Defaults to the collection's + own internal name. + + :param default: + Whether this sub-collection('s default task-or-collection) should + be the default invocation of the parent collection. + + .. versionadded:: 1.0 + .. versionchanged:: 1.5 + Added the ``default`` parameter. + """ + # Handle module-as-collection + if isinstance(coll, ModuleType): + coll = Collection.from_module(coll) + # Ensure we have a name, or die trying + name = name or coll.name + if not name: + raise ValueError("Non-root collections must have a name!") + name = self.transform(name) + # Test for conflict + if name in self.tasks: + err = "Name conflict: this collection has a task named {!r} already" # noqa + raise ValueError(err.format(name)) + # Insert + self.collections[name] = coll + if default: + self._check_default_collision(name) + self.default = name + + def _check_default_collision(self, name: str) -> None: + if self.default: + msg = "'{}' cannot be the default because '{}' already is!" + raise ValueError(msg.format(name, self.default)) + + def _split_path(self, path: str) -> Tuple[str, str]: + """ + Obtain first collection + remainder, of a task path. + + E.g. for ``"subcollection.taskname"``, return ``("subcollection", + "taskname")``; for ``"subcollection.nested.taskname"`` return + ``("subcollection", "nested.taskname")``, etc. + + An empty path becomes simply ``('', '')``. + """ + parts = path.split(".") + coll = parts.pop(0) + rest = ".".join(parts) + return coll, rest + + def subcollection_from_path(self, path: str) -> "Collection": + """ + Given a ``path`` to a subcollection, return that subcollection. + + .. versionadded:: 1.0 + """ + parts = path.split(".") + collection = self + while parts: + collection = collection.collections[parts.pop(0)] + return collection + + def __getitem__(self, name: Optional[str] = None) -> Any: + """ + Returns task named ``name``. Honors aliases and subcollections. + + If this collection has a default task, it is returned when ``name`` is + empty or ``None``. If empty input is given and no task has been + selected as the default, ValueError will be raised. + + Tasks within subcollections should be given in dotted form, e.g. + 'foo.bar'. Subcollection default tasks will be returned on the + subcollection's name. + + .. versionadded:: 1.0 + """ + return self.task_with_config(name)[0] + + def _task_with_merged_config( + self, coll: str, rest: str, ours: Dict[str, Any] + ) -> Tuple[str, Dict[str, Any]]: + task, config = self.collections[coll].task_with_config(rest) + return task, dict(config, **ours) + + def task_with_config( + self, name: Optional[str] + ) -> Tuple[str, Dict[str, Any]]: + """ + Return task named ``name`` plus its configuration dict. + + E.g. in a deeply nested tree, this method returns the `.Task`, and a + configuration dict created by merging that of this `.Collection` and + any nested `Collections <.Collection>`, up through the one actually + holding the `.Task`. + + See `~.Collection.__getitem__` for semantics of the ``name`` argument. + + :returns: Two-tuple of (`.Task`, `dict`). + + .. versionadded:: 1.0 + """ + # Our top level configuration + ours = self.configuration() + # Default task for this collection itself + if not name: + if not self.default: + raise ValueError("This collection has no default task.") + return self[self.default], ours + # Normalize name to the format we're expecting + name = self.transform(name) + # Non-default tasks within subcollections -> recurse (sorta) + if "." in name: + coll, rest = self._split_path(name) + return self._task_with_merged_config(coll, rest, ours) + # Default task for subcollections (via empty-name lookup) + if name in self.collections: + return self._task_with_merged_config(name, "", ours) + # Regular task lookup + return self.tasks[name], ours + + def __contains__(self, name: str) -> bool: + try: + self[name] + return True + except KeyError: + return False + + def to_contexts( + self, ignore_unknown_help: Optional[bool] = None + ) -> List[ParserContext]: + """ + Returns all contained tasks and subtasks as a list of parser contexts. + + :param bool ignore_unknown_help: + Passed on to each task's ``get_arguments()`` method. See the config + option by the same name for details. + + .. versionadded:: 1.0 + .. versionchanged:: 1.7 + Added the ``ignore_unknown_help`` kwarg. + """ + result = [] + for primary, aliases in self.task_names.items(): + task = self[primary] + result.append( + ParserContext( + name=primary, + aliases=aliases, + args=task.get_arguments( + ignore_unknown_help=ignore_unknown_help + ), + ) + ) + return result + + def subtask_name(self, collection_name: str, task_name: str) -> str: + return ".".join( + [self.transform(collection_name), self.transform(task_name)] + ) + + def transform(self, name: str) -> str: + """ + Transform ``name`` with the configured auto-dashes behavior. + + If the collection's ``auto_dash_names`` attribute is ``True`` + (default), all non leading/trailing underscores are turned into dashes. + (Leading/trailing underscores tend to get stripped elsewhere in the + stack.) + + If it is ``False``, the inverse is applied - all dashes are turned into + underscores. + + .. versionadded:: 1.0 + """ + # Short-circuit on anything non-applicable, e.g. empty strings, bools, + # None, etc. + if not name: + return name + from_, to = "_", "-" + if not self.auto_dash_names: + from_, to = "-", "_" + replaced = [] + end = len(name) - 1 + for i, char in enumerate(name): + # Don't replace leading or trailing underscores (+ taking dotted + # names into account) + # TODO: not 100% convinced of this / it may be exposing a + # discrepancy between this level & higher levels which tend to + # strip out leading/trailing underscores entirely. + if ( + i not in (0, end) + and char == from_ + and name[i - 1] != "." + and name[i + 1] != "." + ): + char = to + replaced.append(char) + return "".join(replaced) + + def _transform_lexicon(self, old: Lexicon) -> Lexicon: + """ + Take a Lexicon and apply `.transform` to its keys and aliases. + + :returns: A new Lexicon. + """ + new = Lexicon() + # Lexicons exhibit only their real keys in most places, so this will + # only grab those, not aliases. + for key, value in old.items(): + # Deepcopy the value so we're not just copying a reference + new[self.transform(key)] = copy.deepcopy(value) + # Also copy all aliases, which are string-to-string key mappings + for key, value in old.aliases.items(): + new.alias(from_=self.transform(key), to=self.transform(value)) + return new + + @property + def task_names(self) -> Dict[str, List[str]]: + """ + Return all task identifiers for this collection as a one-level dict. + + Specifically, a dict with the primary/"real" task names as the key, and + any aliases as a list value. + + It basically collapses the namespace tree into a single + easily-scannable collection of invocation strings, and is thus suitable + for things like flat-style task listings or transformation into parser + contexts. + + .. versionadded:: 1.0 + """ + ret = {} + # Our own tasks get no prefix, just go in as-is: {name: [aliases]} + for name, task in self.tasks.items(): + ret[name] = list(map(self.transform, task.aliases)) + # Subcollection tasks get both name + aliases prefixed + for coll_name, coll in self.collections.items(): + for task_name, aliases in coll.task_names.items(): + aliases = list( + map(lambda x: self.subtask_name(coll_name, x), aliases) + ) + # Tack on collection name to alias list if this task is the + # collection's default. + if coll.default == task_name: + aliases += (coll_name,) + ret[self.subtask_name(coll_name, task_name)] = aliases + return ret + + def configuration(self, taskpath: Optional[str] = None) -> Dict[str, Any]: + """ + Obtain merged configuration values from collection & children. + + :param taskpath: + (Optional) Task name/path, identical to that used for + `~.Collection.__getitem__` (e.g. may be dotted for nested tasks, + etc.) Used to decide which path to follow in the collection tree + when merging config values. + + :returns: A `dict` containing configuration values. + + .. versionadded:: 1.0 + """ + if taskpath is None: + return copy_dict(self._configuration) + return self.task_with_config(taskpath)[1] + + def configure(self, options: Dict[str, Any]) -> None: + """ + (Recursively) merge ``options`` into the current `.configuration`. + + Options configured this way will be available to all tasks. It is + recommended to use unique keys to avoid potential clashes with other + config options + + For example, if you were configuring a Sphinx docs build target + directory, it's better to use a key like ``'sphinx.target'`` than + simply ``'target'``. + + :param options: An object implementing the dictionary protocol. + :returns: ``None``. + + .. versionadded:: 1.0 + """ + merge_dicts(self._configuration, options) + + def serialized(self) -> Dict[str, Any]: + """ + Return an appropriate-for-serialization version of this object. + + See the documentation for `.Program` and its ``json`` task listing + format; this method is the driver for that functionality. + + .. versionadded:: 1.0 + """ + return { + "name": self.name, + "help": helpline(self), + "default": self.default, + "tasks": [ + { + "name": self.transform(x.name), + "help": helpline(x), + "aliases": [self.transform(y) for y in x.aliases], + } + for x in sorted(self.tasks.values(), key=lambda x: x.name) + ], + "collections": [ + x.serialized() + for x in sorted( + self.collections.values(), key=lambda x: x.name or "" + ) + ], + } diff --git a/lib/invoke/completion/__init__.py b/lib/invoke/completion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/invoke/completion/__pycache__/__init__.cpython-314.pyc b/lib/invoke/completion/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..bb30570 Binary files /dev/null and b/lib/invoke/completion/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/completion/__pycache__/complete.cpython-314.pyc b/lib/invoke/completion/__pycache__/complete.cpython-314.pyc new file mode 100644 index 0000000..ae4721d Binary files /dev/null and b/lib/invoke/completion/__pycache__/complete.cpython-314.pyc differ diff --git a/lib/invoke/completion/bash.completion b/lib/invoke/completion/bash.completion new file mode 100644 index 0000000..55f7c39 --- /dev/null +++ b/lib/invoke/completion/bash.completion @@ -0,0 +1,32 @@ +# Invoke tab-completion script to be sourced with Bash shell. +# Known to work on Bash 3.x, untested on 4.x. + +_complete_{binary}() {{ + local candidates + + # COMP_WORDS contains the entire command string up til now (including + # program name). + # We hand it to Invoke so it can figure out the current context: spit back + # core options, task names, the current task's options, or some combo. + candidates=`{binary} --complete -- ${{COMP_WORDS[*]}}` + + # `compgen -W` takes list of valid options & a partial word & spits back + # possible matches. Necessary for any partial word completions (vs + # completions performed when no partial words are present). + # + # $2 is the current word or token being tabbed on, either empty string or a + # partial word, and thus wants to be compgen'd to arrive at some subset of + # our candidate list which actually matches. + # + # COMPREPLY is the list of valid completions handed back to `complete`. + COMPREPLY=( $(compgen -W "${{candidates}}" -- $2) ) +}} + + +# Tell shell builtin to use the above for completing our invocations. +# * -F: use given function name to generate completions. +# * -o default: when function generates no results, use filenames. +# * positional args: program names to complete for. +complete -F _complete_{binary} -o default {spaced_names} + +# vim: set ft=sh : diff --git a/lib/invoke/completion/complete.py b/lib/invoke/completion/complete.py new file mode 100644 index 0000000..97e9a95 --- /dev/null +++ b/lib/invoke/completion/complete.py @@ -0,0 +1,129 @@ +""" +Command-line completion mechanisms, executed by the core ``--complete`` flag. +""" + +from typing import List +import glob +import os +import re +import shlex +from typing import TYPE_CHECKING + +from ..exceptions import Exit, ParseError +from ..util import debug, task_name_sort_key + +if TYPE_CHECKING: + from ..collection import Collection + from ..parser import Parser, ParseResult, ParserContext + + +def complete( + names: List[str], + core: "ParseResult", + initial_context: "ParserContext", + collection: "Collection", + parser: "Parser", +) -> Exit: + # Strip out program name (scripts give us full command line) + # TODO: this may not handle path/to/script though? + invocation = re.sub(r"^({}) ".format("|".join(names)), "", core.remainder) + debug("Completing for invocation: {!r}".format(invocation)) + # Tokenize (shlex will have to do) + tokens = shlex.split(invocation) + # Handle flags (partial or otherwise) + if tokens and tokens[-1].startswith("-"): + tail = tokens[-1] + debug("Invocation's tail {!r} is flag-like".format(tail)) + # Gently parse invocation to obtain 'current' context. + # Use last seen context in case of failure (required for + # otherwise-invalid partial invocations being completed). + + contexts: List[ParserContext] + try: + debug("Seeking context name in tokens: {!r}".format(tokens)) + contexts = parser.parse_argv(tokens) + except ParseError as e: + msg = "Got parser error ({!r}), grabbing its last-seen context {!r}" # noqa + debug(msg.format(e, e.context)) + contexts = [e.context] if e.context is not None else [] + # Fall back to core context if no context seen. + debug("Parsed invocation, contexts: {!r}".format(contexts)) + if not contexts or not contexts[-1]: + context = initial_context + else: + context = contexts[-1] + debug("Selected context: {!r}".format(context)) + # Unknown flags (could be e.g. only partially typed out; could be + # wholly invalid; doesn't matter) complete with flags. + debug("Looking for {!r} in {!r}".format(tail, context.flags)) + if tail not in context.flags: + debug("Not found, completing with flag names") + # Long flags - partial or just the dashes - complete w/ long flags + if tail.startswith("--"): + for name in filter( + lambda x: x.startswith("--"), context.flag_names() + ): + print(name) + # Just a dash, completes with all flags + elif tail == "-": + for name in context.flag_names(): + print(name) + # Otherwise, it's something entirely invalid (a shortflag not + # recognized, or a java style flag like -foo) so return nothing + # (the shell will still try completing with files, but that doesn't + # hurt really.) + else: + pass + # Known flags complete w/ nothing or tasks, depending + else: + # Flags expecting values: do nothing, to let default (usually + # file) shell completion occur (which we actively want in this + # case.) + if context.flags[tail].takes_value: + debug("Found, and it takes a value, so no completion") + pass + # Not taking values (eg bools): print task names + else: + debug("Found, takes no value, printing task names") + print_task_names(collection) + # If not a flag, is either task name or a flag value, so just complete + # task names. + else: + debug("Last token isn't flag-like, just printing task names") + print_task_names(collection) + raise Exit + + +def print_task_names(collection: "Collection") -> None: + for name in sorted(collection.task_names, key=task_name_sort_key): + print(name) + # Just stick aliases after the thing they're aliased to. Sorting isn't + # so important that it's worth bending over backwards here. + for alias in collection.task_names[name]: + print(alias) + + +def print_completion_script(shell: str, names: List[str]) -> None: + # Grab all .completion files in invoke/completion/. (These used to have no + # suffix, but surprise, that's super fragile. + completions = { + os.path.splitext(os.path.basename(x))[0]: x + for x in glob.glob( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), "*.completion" + ) + ) + } + try: + path = completions[shell] + except KeyError: + err = 'Completion for shell "{}" not supported (options are: {}).' + raise ParseError(err.format(shell, ", ".join(sorted(completions)))) + debug("Printing completion script from {}".format(path)) + # Choose one arbitrary program name for script's own internal invocation + # (also used to construct completion function names when necessary) + binary = names[0] + with open(path, "r") as script: + print( + script.read().format(binary=binary, spaced_names=" ".join(names)) + ) diff --git a/lib/invoke/completion/fish.completion b/lib/invoke/completion/fish.completion new file mode 100644 index 0000000..5f479a1 --- /dev/null +++ b/lib/invoke/completion/fish.completion @@ -0,0 +1,10 @@ +# Invoke tab-completion script for the fish shell +# Copy it to the ~/.config/fish/completions directory + +function __complete_{binary} + {binary} --complete -- (commandline --tokenize) +end + +# --no-files: Don't complete files unless invoke gives an empty result +# TODO: find a way to honor all binary_names +complete --command {binary} --no-files --arguments '(__complete_{binary})' diff --git a/lib/invoke/completion/zsh.completion b/lib/invoke/completion/zsh.completion new file mode 100644 index 0000000..2fb7d12 --- /dev/null +++ b/lib/invoke/completion/zsh.completion @@ -0,0 +1,33 @@ +# Invoke tab-completion script to be sourced with the Z shell. +# Known to work on zsh 5.0.x, probably works on later 4.x releases as well (as +# it uses the older compctl completion system). + +_complete_{binary}() {{ + # `words` contains the entire command string up til now (including + # program name). + # + # We hand it to Invoke so it can figure out the current context: spit back + # core options, task names, the current task's options, or some combo. + # + # Before doing so, we attempt to tease out any collection flag+arg so we + # can ensure it is applied correctly. + collection_arg='' + if [[ "${{words}}" =~ "(-c|--collection) [^ ]+" ]]; then + collection_arg=$MATCH + fi + # `reply` is the array of valid completions handed back to `compctl`. + # Use ${{=...}} to force whitespace splitting in expansion of + # $collection_arg + reply=( $({binary} ${{=collection_arg}} --complete -- ${{words}}) ) +}} + + +# Tell shell builtin to use the above for completing our given binary name(s). +# * -K: use given function name to generate completions. +# * +: specifies 'alternative' completion, where options after the '+' are only +# used if the completion from the options before the '+' result in no matches. +# * -f: when function generates no results, use filenames. +# * positional args: program names to complete for. +compctl -K _complete_{binary} + -f {spaced_names} + +# vim: set ft=sh : diff --git a/lib/invoke/config.py b/lib/invoke/config.py new file mode 100644 index 0000000..64e3846 --- /dev/null +++ b/lib/invoke/config.py @@ -0,0 +1,1283 @@ +import copy +import json +import os +import types +from importlib.util import spec_from_loader +from os import PathLike +from os.path import join, splitext, expanduser +from types import ModuleType +from typing import Any, Dict, Iterator, Optional, Tuple, Type, Union + +from .env import Environment +from .exceptions import UnknownFileType, UnpicklableConfigMember +from .runners import Local +from .terminals import WINDOWS +from .util import debug, yaml + + +try: + from importlib.machinery import SourceFileLoader +except ImportError: # PyPy3 + from importlib._bootstrap import ( # type: ignore[no-redef] + _SourceFileLoader as SourceFileLoader, + ) + + +def load_source(name: str, path: str) -> Dict[str, Any]: + if not os.path.exists(path): + return {} + loader = SourceFileLoader("mod", path) + mod = ModuleType("mod") + mod.__spec__ = spec_from_loader("mod", loader) + loader.exec_module(mod) + return vars(mod) + + +class DataProxy: + """ + Helper class implementing nested dict+attr access for `.Config`. + + Specifically, is used both for `.Config` itself, and to wrap any other + dicts assigned as config values (recursively). + + .. warning:: + All methods (of this object or in subclasses) must take care to + initialize new attributes via ``self._set(name='value')``, or they'll + run into recursion errors! + + .. versionadded:: 1.0 + """ + + # Attributes which get proxied through to inner merged-dict config obj. + _proxies = ( + tuple( + """ + get + has_key + items + iteritems + iterkeys + itervalues + keys + values + """.split() + ) + + tuple( + "__{}__".format(x) + for x in """ + cmp + contains + iter + sizeof + """.split() + ) + ) + + @classmethod + def from_data( + cls, + data: Dict[str, Any], + root: Optional["DataProxy"] = None, + keypath: Tuple[str, ...] = tuple(), + ) -> "DataProxy": + """ + Alternate constructor for 'baby' DataProxies used as sub-dict values. + + Allows creating standalone DataProxy objects while also letting + subclasses like `.Config` define their own ``__init__`` without + muddling the two. + + :param dict data: + This particular DataProxy's personal data. Required, it's the Data + being Proxied. + + :param root: + Optional handle on a root DataProxy/Config which needs notification + on data updates. + + :param tuple keypath: + Optional tuple describing the path of keys leading to this + DataProxy's location inside the ``root`` structure. Required if + ``root`` was given (and vice versa.) + + .. versionadded:: 1.0 + """ + obj = cls() + obj._set(_config=data) + obj._set(_root=root) + obj._set(_keypath=keypath) + return obj + + def __getattr__(self, key: str) -> Any: + # NOTE: due to default Python attribute-lookup semantics, "real" + # attributes will always be yielded on attribute access and this method + # is skipped. That behavior is good for us (it's more intuitive than + # having a config key accidentally shadow a real attribute or method). + try: + return self._get(key) + except KeyError: + # Proxy most special vars to config for dict procotol. + if key in self._proxies: + return getattr(self._config, key) + # Otherwise, raise useful AttributeError to follow getattr proto. + err = "No attribute or config key found for {!r}".format(key) + attrs = [x for x in dir(self.__class__) if not x.startswith("_")] + err += "\n\nValid keys: {!r}".format( + sorted(list(self._config.keys())) + ) + err += "\n\nValid real attributes: {!r}".format(attrs) + raise AttributeError(err) + + def __setattr__(self, key: str, value: Any) -> None: + # Turn attribute-sets into config updates anytime we don't have a real + # attribute with the given name/key. + has_real_attr = key in dir(self) + if not has_real_attr: + # Make sure to trigger our own __setitem__ instead of going direct + # to our internal dict/cache + self[key] = value + else: + super().__setattr__(key, value) + + def __iter__(self) -> Iterator[Dict[str, Any]]: + # For some reason Python is ignoring our __hasattr__ when determining + # whether we support __iter__. BOO + return iter(self._config) + + def __eq__(self, other: object) -> bool: + # NOTE: Can't proxy __eq__ because the RHS will always be an obj of the + # current class, not the proxied-to class, and that causes + # NotImplemented. + # Try comparing to other objects like ourselves, falling back to a not + # very comparable value (None) so comparison fails. + other_val = getattr(other, "_config", None) + # But we can compare to vanilla dicts just fine, since our _config is + # itself just a dict. + if isinstance(other, dict): + other_val = other + return bool(self._config == other_val) + + def __len__(self) -> int: + return len(self._config) + + def __setitem__(self, key: str, value: str) -> None: + self._config[key] = value + self._track_modification_of(key, value) + + def __getitem__(self, key: str) -> Any: + return self._get(key) + + def _get(self, key: str) -> Any: + # Short-circuit if pickling/copying mechanisms are asking if we've got + # __setstate__ etc; they'll ask this w/o calling our __init__ first, so + # we'd be in a RecursionError-causing catch-22 otherwise. + if key in ("__setstate__",): + raise AttributeError(key) + # At this point we should be able to assume a self._config... + value = self._config[key] + if isinstance(value, dict): + # New object's keypath is simply the key, prepended with our own + # keypath if we've got one. + keypath = (key,) + if hasattr(self, "_keypath"): + keypath = self._keypath + keypath + # If we have no _root, we must be the root, so it's us. Otherwise, + # pass along our handle on the root. + root = getattr(self, "_root", self) + value = DataProxy.from_data(data=value, root=root, keypath=keypath) + return value + + def _set(self, *args: Any, **kwargs: Any) -> None: + """ + Convenience workaround of default 'attrs are config keys' behavior. + + Uses `object.__setattr__` to work around the class' normal proxying + behavior, but is less verbose than using that directly. + + Has two modes (which may be combined if you really want): + + - ``self._set('attrname', value)``, just like ``__setattr__`` + - ``self._set(attname=value)`` (i.e. kwargs), even less typing. + """ + if args: + object.__setattr__(self, *args) + for key, value in kwargs.items(): + object.__setattr__(self, key, value) + + def __repr__(self) -> str: + return "<{}: {}>".format(self.__class__.__name__, self._config) + + def __contains__(self, key: str) -> bool: + return key in self._config + + @property + def _is_leaf(self) -> bool: + return hasattr(self, "_root") + + @property + def _is_root(self) -> bool: + return hasattr(self, "_modify") + + def _track_removal_of(self, key: str) -> None: + # Grab the root object responsible for tracking removals; either the + # referenced root (if we're a leaf) or ourselves (if we're not). + # (Intermediate nodes never have anything but __getitem__ called on + # them, otherwise they're by definition being treated as a leaf.) + target = None + if self._is_leaf: + target = self._root + elif self._is_root: + target = self + if target is not None: + target._remove(getattr(self, "_keypath", tuple()), key) + + def _track_modification_of(self, key: str, value: str) -> None: + target = None + if self._is_leaf: + target = self._root + elif self._is_root: + target = self + if target is not None: + target._modify(getattr(self, "_keypath", tuple()), key, value) + + def __delitem__(self, key: str) -> None: + del self._config[key] + self._track_removal_of(key) + + def __delattr__(self, name: str) -> None: + # Make sure we don't screw up true attribute deletion for the + # situations that actually want it. (Uncommon, but not rare.) + if name in self: + del self[name] + else: + object.__delattr__(self, name) + + def clear(self) -> None: + keys = list(self.keys()) + for key in keys: + del self[key] + + def pop(self, *args: Any) -> Any: + # Must test this up front before (possibly) mutating self._config + key_existed = args and args[0] in self._config + # We always have a _config (whether it's a real dict or a cache of + # merged levels) so we can fall back to it for all the corner case + # handling re: args (arity, handling a default, raising KeyError, etc) + ret = self._config.pop(*args) + # If it looks like no popping occurred (key wasn't there), presumably + # user gave default, so we can short-circuit return here - no need to + # track a deletion that did not happen. + if not key_existed: + return ret + # Here, we can assume at least the 1st posarg (key) existed. + self._track_removal_of(args[0]) + # In all cases, return the popped value. + return ret + + def popitem(self) -> Any: + ret = self._config.popitem() + self._track_removal_of(ret[0]) + return ret + + def setdefault(self, *args: Any) -> Any: + # Must test up front whether the key existed beforehand + key_existed = args and args[0] in self._config + # Run locally + ret = self._config.setdefault(*args) + # Key already existed -> nothing was mutated, short-circuit + if key_existed: + return ret + # Here, we can assume the key did not exist and thus user must have + # supplied a 'default' (if they did not, the real setdefault() above + # would have excepted.) + key, default = args + self._track_modification_of(key, default) + return ret + + def update(self, *args: Any, **kwargs: Any) -> None: + if kwargs: + for key, value in kwargs.items(): + self[key] = value + elif args: + # TODO: complain if arity>1 + arg = args[0] + if isinstance(arg, dict): + for key in arg: + self[key] = arg[key] + else: + # TODO: be stricter about input in this case + for pair in arg: + self[pair[0]] = pair[1] + + +class Config(DataProxy): + """ + Invoke's primary configuration handling class. + + See :doc:`/concepts/configuration` for details on the configuration system + this class implements, including the :ref:`configuration hierarchy + `. The rest of this class' documentation assumes + familiarity with that document. + + **Access** + + Configuration values may be accessed and/or updated using dict syntax:: + + config['foo'] + + or attribute syntax:: + + config.foo + + Nesting works the same way - dict config values are turned into objects + which honor both the dictionary protocol and the attribute-access method:: + + config['foo']['bar'] + config.foo.bar + + **A note about attribute access and methods** + + This class implements the entire dictionary protocol: methods such as + ``keys``, ``values``, ``items``, ``pop`` and so forth should all function + as they do on regular dicts. It also implements new config-specific methods + such as `load_system`, `load_collection`, `merge`, `clone`, etc. + + .. warning:: + Accordingly, this means that if you have configuration options sharing + names with these methods, you **must** use dictionary syntax (e.g. + ``myconfig['keys']``) to access the configuration data. + + **Lifecycle** + + At initialization time, `.Config`: + + - creates per-level data structures; + - stores any levels supplied to `__init__`, such as defaults or overrides, + as well as the various config file paths/filename patterns; + - and loads config files, if found (though typically this just means system + and user-level files, as project and runtime files need more info before + they can be found and loaded.) + + - This step can be skipped by specifying ``lazy=True``. + + At this point, `.Config` is fully usable - and because it pre-emptively + loads some config files, those config files can affect anything that + comes after, like CLI parsing or loading of task collections. + + In the CLI use case, further processing is done after instantiation, using + the ``load_*`` methods such as `load_overrides`, `load_project`, etc: + + - the result of argument/option parsing is applied to the overrides level; + - a project-level config file is loaded, as it's dependent on a loaded + tasks collection; + - a runtime config file is loaded, if its flag was supplied; + - then, for each task being executed: + + - per-collection data is loaded (only possible now that we have + collection & task in hand); + - shell environment data is loaded (must be done at end of process due + to using the rest of the config as a guide for interpreting env var + names.) + + At this point, the config object is handed to the task being executed, as + part of its execution `.Context`. + + Any modifications made directly to the `.Config` itself after this point + end up stored in their own (topmost) config level, making it easier to + debug final values. + + Finally, any *deletions* made to the `.Config` (e.g. applications of + dict-style mutators like ``pop``, ``clear`` etc) are also tracked in their + own structure, allowing the config object to honor such method calls + without mutating the underlying source data. + + **Special class attributes** + + The following class-level attributes are used for low-level configuration + of the config system itself, such as which file paths to load. They are + primarily intended for overriding by subclasses. + + - ``prefix``: Supplies the default value for ``file_prefix`` (directly) and + ``env_prefix`` (uppercased). See their descriptions for details. Its + default value is ``"invoke"``. + - ``file_prefix``: The config file 'basename' default (though it is not a + literal basename; it can contain path parts if desired) which is appended + to the configured values of ``system_prefix``, ``user_prefix``, etc, to + arrive at the final (pre-extension) file paths. + + Thus, by default, a system-level config file path concatenates the + ``system_prefix`` of ``/etc/`` with the ``file_prefix`` of ``invoke`` to + arrive at paths like ``/etc/invoke.json``. + + Defaults to ``None``, meaning to use the value of ``prefix``. + + - ``env_prefix``: A prefix used (along with a joining underscore) to + determine which environment variables are loaded as the env var + configuration level. Since its default is the value of ``prefix`` + capitalized, this means env vars like ``INVOKE_RUN_ECHO`` are sought by + default. + + Defaults to ``None``, meaning to use the value of ``prefix``. + + .. versionadded:: 1.0 + """ + + prefix = "invoke" + file_prefix = None + env_prefix = None + + @staticmethod + def global_defaults() -> Dict[str, Any]: + """ + Return the core default settings for Invoke. + + Generally only for use by `.Config` internals. For descriptions of + these values, see :ref:`default-values`. + + Subclasses may choose to override this method, calling + ``Config.global_defaults`` and applying `.merge_dicts` to the result, + to add to or modify these values. + + .. versionadded:: 1.0 + """ + # On Windows, which won't have /bin/bash, check for a set COMSPEC env + # var (https://en.wikipedia.org/wiki/COMSPEC) or fallback to an + # unqualified cmd.exe otherwise. + if WINDOWS: + shell = os.environ.get("COMSPEC", "cmd.exe") + # Else, assume Unix, most distros of which have /bin/bash available. + # TODO: consider an automatic fallback to /bin/sh for systems lacking + # /bin/bash; however users may configure run.shell quite easily, so... + else: + shell = "/bin/bash" + + return { + # TODO: we document 'debug' but it's not truly implemented outside + # of env var and CLI flag. If we honor it, we have to go around and + # figure out at what points we might want to call + # `util.enable_logging`: + # - just using it as a fallback default for arg parsing isn't much + # use, as at that point the config holds nothing but defaults & CLI + # flag values + # - doing it at file load time might be somewhat useful, though + # where this happens may be subject to change soon + # - doing it at env var load time seems a bit silly given the + # existing support for at-startup testing for INVOKE_DEBUG + # 'debug': False, + # TODO: I feel like we want these to be more consistent re: default + # values stored here vs 'stored' as logic where they are + # referenced, there are probably some bits that are all "if None -> + # default" that could go here. Alternately, make _more_ of these + # default to None? + "run": { + "asynchronous": False, + "disown": False, + "dry": False, + "echo": False, + "echo_stdin": None, + "encoding": None, + "env": {}, + "err_stream": None, + "fallback": True, + "hide": None, + "in_stream": None, + "out_stream": None, + "echo_format": "\033[1;37m{command}\033[0m", + "pty": False, + "replace_env": False, + "shell": shell, + "warn": False, + "watchers": [], + }, + # This doesn't live inside the 'run' tree; otherwise it'd make it + # somewhat harder to extend/override in Fabric 2 which has a split + # local/remote runner situation. + "runners": {"local": Local}, + "sudo": { + "password": None, + "prompt": "[sudo] password: ", + "user": None, + }, + "tasks": { + "auto_dash_names": True, + "collection_name": "tasks", + "dedupe": True, + "executor_class": None, + "ignore_unknown_help": False, + "search_root": None, + }, + "timeouts": {"command": None}, + } + + def __init__( + self, + overrides: Optional[Dict[str, Any]] = None, + defaults: Optional[Dict[str, Any]] = None, + system_prefix: Optional[str] = None, + user_prefix: Optional[str] = None, + project_location: Optional[PathLike] = None, + runtime_path: Optional[PathLike] = None, + lazy: bool = False, + ): + """ + Creates a new config object. + + :param dict defaults: + A dict containing default (lowest level) config data. Default: + `global_defaults`. + + :param dict overrides: + A dict containing override-level config data. Default: ``{}``. + + :param str system_prefix: + Base path for the global config file location; combined with the + prefix and file suffixes to arrive at final file path candidates. + + Default: ``/etc/`` (thus e.g. ``/etc/invoke.yaml`` or + ``/etc/invoke.json``). + + :param str user_prefix: + Like ``system_prefix`` but for the per-user config file. These + variables are joined as strings, not via path-style joins, so they + may contain partial file paths; for the per-user config file this + often means a leading dot, to make the final result a hidden file + on most systems. + + Default: ``~/.`` (e.g. ``~/.invoke.yaml``). + + :param str project_location: + Optional directory path of the currently loaded `.Collection` (as + loaded by `.Loader`). When non-empty, will trigger seeking of + per-project config files in this directory. + + :param str runtime_path: + Optional file path to a runtime configuration file. + + Used to fill the penultimate slot in the config hierarchy. Should + be a full file path to an existing file, not a directory path or a + prefix. + + :param bool lazy: + Whether to automatically load some of the lower config levels. + + By default (``lazy=False``), ``__init__`` automatically calls + `load_system` and `load_user` to load system and user config files, + respectively. + + For more control over what is loaded when, you can say + ``lazy=True``, and no automatic loading is done. + + .. note:: + If you give ``defaults`` and/or ``overrides`` as ``__init__`` + kwargs instead of waiting to use `load_defaults` or + `load_overrides` afterwards, those *will* still end up 'loaded' + immediately. + """ + # Technically an implementation detail - do not expose in public API. + # Stores merged configs and is accessed via DataProxy. + self._set(_config={}) + + # Config file suffixes to search, in preference order. + self._set(_file_suffixes=("yaml", "yml", "json", "py")) + + # Default configuration values, typically a copy of `global_defaults`. + if defaults is None: + defaults = copy_dict(self.global_defaults()) + self._set(_defaults=defaults) + + # Collection-driven config data, gathered from the collection tree + # containing the currently executing task. + self._set(_collection={}) + + # Path prefix searched for the system config file. + # NOTE: There is no default system prefix on Windows. + if system_prefix is None and not WINDOWS: + system_prefix = "/etc/" + self._set(_system_prefix=system_prefix) + # Path to loaded system config file, if any. + self._set(_system_path=None) + # Whether the system config file has been loaded or not (or ``None`` if + # no loading has been attempted yet.) + self._set(_system_found=None) + # Data loaded from the system config file. + self._set(_system={}) + + # Path prefix searched for per-user config files. + if user_prefix is None: + user_prefix = "~/." + self._set(_user_prefix=user_prefix) + # Path to loaded user config file, if any. + self._set(_user_path=None) + # Whether the user config file has been loaded or not (or ``None`` if + # no loading has been attempted yet.) + self._set(_user_found=None) + # Data loaded from the per-user config file. + self._set(_user={}) + + # As it may want to be set post-init, project conf file related attrs + # get initialized or overwritten via a specific method. + self.set_project_location(project_location) + + # Environment variable name prefix + env_prefix = self.env_prefix + if env_prefix is None: + env_prefix = self.prefix + env_prefix = "{}_".format(env_prefix.upper()) + self._set(_env_prefix=env_prefix) + # Config data loaded from the shell environment. + self._set(_env={}) + + # As it may want to be set post-init, runtime conf file related attrs + # get initialized or overwritten via a specific method. + self.set_runtime_path(runtime_path) + + # Overrides - highest normal config level. Typically filled in from + # command-line flags. + if overrides is None: + overrides = {} + self._set(_overrides=overrides) + + # Absolute highest level: user modifications. + self._set(_modifications={}) + # And its sibling: user deletions. (stored as a flat dict of keypath + # keys and dummy values, for constant-time membership testing/removal + # w/ no messy recursion. TODO: maybe redo _everything_ that way? in + # _modifications and other levels, the values would of course be + # valuable and not just None) + self._set(_deletions={}) + + # Convenience loading of user and system files, since those require no + # other levels in order to function. + if not lazy: + self.load_base_conf_files() + # Always merge, otherwise defaults, etc are not usable until creator or + # a subroutine does so. + self.merge() + + def load_base_conf_files(self) -> None: + # Just a refactor of something done in unlazy init or in clone() + self.load_system(merge=False) + self.load_user(merge=False) + + def load_defaults(self, data: Dict[str, Any], merge: bool = True) -> None: + """ + Set or replace the 'defaults' configuration level, from ``data``. + + :param dict data: The config data to load as the defaults level. + + :param bool merge: + Whether to merge the loaded data into the central config. Default: + ``True``. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._set(_defaults=data) + if merge: + self.merge() + + def load_overrides(self, data: Dict[str, Any], merge: bool = True) -> None: + """ + Set or replace the 'overrides' configuration level, from ``data``. + + :param dict data: The config data to load as the overrides level. + + :param bool merge: + Whether to merge the loaded data into the central config. Default: + ``True``. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._set(_overrides=data) + if merge: + self.merge() + + def load_system(self, merge: bool = True) -> None: + """ + Load a system-level config file, if possible. + + Checks the configured ``_system_prefix`` path, which defaults to + ``/etc``, and will thus load files like ``/etc/invoke.yml``. + + :param bool merge: + Whether to merge the loaded data into the central config. Default: + ``True``. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._load_file(prefix="system", merge=merge) + + def load_user(self, merge: bool = True) -> None: + """ + Load a user-level config file, if possible. + + Checks the configured ``_user_prefix`` path, which defaults to ``~/.``, + and will thus load files like ``~/.invoke.yml``. + + :param bool merge: + Whether to merge the loaded data into the central config. Default: + ``True``. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._load_file(prefix="user", merge=merge) + + def load_project(self, merge: bool = True) -> None: + """ + Load a project-level config file, if possible. + + Checks the configured ``_project_prefix`` value derived from the path + given to `set_project_location`, which is typically set to the + directory containing the loaded task collection. + + Thus, if one were to run the CLI tool against a tasks collection + ``/home/myuser/code/tasks.py``, `load_project` would seek out files + like ``/home/myuser/code/invoke.yml``. + + :param bool merge: + Whether to merge the loaded data into the central config. Default: + ``True``. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._load_file(prefix="project", merge=merge) + + def set_runtime_path(self, path: Optional[PathLike]) -> None: + """ + Set the runtime config file path. + + .. versionadded:: 1.0 + """ + # Path to the user-specified runtime config file. + self._set(_runtime_path=path) + # Data loaded from the runtime config file. + self._set(_runtime={}) + # Whether the runtime config file has been loaded or not (or ``None`` + # if no loading has been attempted yet.) + self._set(_runtime_found=None) + + def load_runtime(self, merge: bool = True) -> None: + """ + Load a runtime-level config file, if one was specified. + + When the CLI framework creates a `Config`, it sets ``_runtime_path``, + which is a full path to the requested config file. This method attempts + to load that file. + + :param bool merge: + Whether to merge the loaded data into the central config. Default: + ``True``. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._load_file(prefix="runtime", absolute=True, merge=merge) + + def load_shell_env(self) -> None: + """ + Load values from the shell environment. + + `.load_shell_env` is intended for execution late in a `.Config` + object's lifecycle, once all other sources (such as a runtime config + file or per-collection configurations) have been loaded. Loading from + the shell is not terrifically expensive, but must be done at a specific + point in time to ensure the "only known config keys are loaded from the + env" behavior works correctly. + + See :ref:`env-vars` for details on this design decision and other info + re: how environment variables are scanned and loaded. + + .. versionadded:: 1.0 + """ + # Force merge of existing data to ensure we have an up to date picture + debug("Running pre-merge for shell env loading...") + self.merge() + debug("Done with pre-merge.") + loader = Environment(config=self._config, prefix=self._env_prefix) + self._set(_env=loader.load()) + debug("Loaded shell environment, triggering final merge") + self.merge() + + def load_collection( + self, data: Dict[str, Any], merge: bool = True + ) -> None: + """ + Update collection-driven config data. + + `.load_collection` is intended for use by the core task execution + machinery, which is responsible for obtaining collection-driven data. + See :ref:`collection-configuration` for details. + + .. versionadded:: 1.0 + """ + debug("Loading collection configuration") + self._set(_collection=data) + if merge: + self.merge() + + def set_project_location(self, path: Union[PathLike, str, None]) -> None: + """ + Set the directory path where a project-level config file may be found. + + Does not do any file loading on its own; for that, see `load_project`. + + .. versionadded:: 1.0 + """ + # 'Prefix' to match the other sets of attrs + project_prefix = None + if path is not None: + # Ensure the prefix is normalized to a directory-like path string + project_prefix = join(path, "") + self._set(_project_prefix=project_prefix) + # Path to loaded per-project config file, if any. + self._set(_project_path=None) + # Whether the project config file has been loaded or not (or ``None`` + # if no loading has been attempted yet.) + self._set(_project_found=None) + # Data loaded from the per-project config file. + self._set(_project={}) + + def _load_file( + self, prefix: str, absolute: bool = False, merge: bool = True + ) -> None: + # Setup + found = "_{}_found".format(prefix) + path = "_{}_path".format(prefix) + data = "_{}".format(prefix) + midfix = self.file_prefix + if midfix is None: + midfix = self.prefix + # Short-circuit if loading appears to have occurred already + if getattr(self, found) is not None: + return + # Moar setup + if absolute: + absolute_path = getattr(self, path) + # None -> expected absolute path but none set, short circuit + if absolute_path is None: + return + paths = [absolute_path] + else: + path_prefix = getattr(self, "_{}_prefix".format(prefix)) + # Short circuit if loading seems unnecessary (eg for project config + # files when not running out of a project) + if path_prefix is None: + return + paths = [ + ".".join((path_prefix + midfix, x)) + for x in self._file_suffixes + ] + # Poke 'em + for filepath in paths: + # Normalize + filepath = expanduser(filepath) + try: + try: + type_ = splitext(filepath)[1].lstrip(".") + loader = getattr(self, "_load_{}".format(type_)) + except AttributeError: + msg = "Config files of type {!r} (from file {!r}) are not supported! Please use one of: {!r}" # noqa + raise UnknownFileType( + msg.format(type_, filepath, self._file_suffixes) + ) + # Store data, the path it was found at, and fact that it was + # found + self._set(data, loader(filepath)) + self._set(path, filepath) + self._set(found, True) + break + # Typically means 'no such file', so just note & skip past. + except IOError as e: + if e.errno == 2: + err = "Didn't see any {}, skipping." + debug(err.format(filepath)) + else: + raise + # Still None -> no suffixed paths were found, record this fact + if getattr(self, path) is None: + self._set(found, False) + # Merge loaded data in if any was found + elif merge: + self.merge() + + def _load_yaml(self, path: PathLike) -> Any: + with open(path) as fd: + return yaml.safe_load(fd) + + _load_yml = _load_yaml + + def _load_json(self, path: PathLike) -> Any: + with open(path) as fd: + return json.load(fd) + + def _load_py(self, path: str) -> Dict[str, Any]: + data = {} + for key, value in (load_source("mod", path)).items(): + # Strip special members, as these are always going to be builtins + # and other special things a user will not want in their config. + if key.startswith("__"): + continue + # Raise exceptions on module values; they are unpicklable. + # TODO: suck it up and reimplement copy() without pickling? Then + # again, a user trying to stuff a module into their config is + # probably doing something better done in runtime/library level + # code and not in a "config file"...right? + if isinstance(value, types.ModuleType): + err = "'{}' is a module, which can't be used as a config value. (Are you perhaps giving a tasks file instead of a config file by mistake?)" # noqa + raise UnpicklableConfigMember(err.format(key)) + data[key] = value + return data + + def merge(self) -> None: + """ + Merge all config sources, in order. + + .. versionadded:: 1.0 + """ + debug("Merging config sources in order onto new empty _config...") + self._set(_config={}) + debug("Defaults: {!r}".format(self._defaults)) + merge_dicts(self._config, self._defaults) + debug("Collection-driven: {!r}".format(self._collection)) + merge_dicts(self._config, self._collection) + self._merge_file("system", "System-wide") + self._merge_file("user", "Per-user") + self._merge_file("project", "Per-project") + debug("Environment variable config: {!r}".format(self._env)) + merge_dicts(self._config, self._env) + self._merge_file("runtime", "Runtime") + debug("Overrides: {!r}".format(self._overrides)) + merge_dicts(self._config, self._overrides) + debug("Modifications: {!r}".format(self._modifications)) + merge_dicts(self._config, self._modifications) + debug("Deletions: {!r}".format(self._deletions)) + obliterate(self._config, self._deletions) + + def _merge_file(self, name: str, desc: str) -> None: + # Setup + desc += " config file" # yup + found = getattr(self, "_{}_found".format(name)) + path = getattr(self, "_{}_path".format(name)) + data = getattr(self, "_{}".format(name)) + # None -> no loading occurred yet + if found is None: + debug("{} has not been loaded yet, skipping".format(desc)) + # True -> hooray + elif found: + debug("{} ({}): {!r}".format(desc, path, data)) + merge_dicts(self._config, data) + # False -> did try, did not succeed + else: + # TODO: how to preserve what was tried for each case but only for + # the negative? Just a branch here based on 'name'? + debug("{} not found, skipping".format(desc)) + + def clone(self, into: Optional[Type["Config"]] = None) -> "Config": + """ + Return a copy of this configuration object. + + The new object will be identical in terms of configured sources and any + loaded (or user-manipulated) data, but will be a distinct object with + as little shared mutable state as possible. + + Specifically, all `dict` values within the config are recursively + recreated, with non-dict leaf values subjected to `copy.copy` (note: + *not* `copy.deepcopy`, as this can cause issues with various objects + such as compiled regexen or threading locks, often found buried deep + within rich aggregates like API or DB clients). + + The only remaining config values that may end up shared between a + config and its clone are thus those 'rich' objects that do not + `copy.copy` cleanly, or compound non-dict objects (such as lists or + tuples). + + :param into: + A `.Config` subclass that the new clone should be "upgraded" to. + + Used by client libraries which have their own `.Config` subclasses + that e.g. define additional defaults; cloning "into" one of these + subclasses ensures that any new keys/subtrees are added gracefully, + without overwriting anything that may have been pre-defined. + + Default: ``None`` (just clone into another regular `.Config`). + + :returns: + A `.Config`, or an instance of the class given to ``into``. + + .. versionadded:: 1.0 + """ + # Construct new object + klass = self.__class__ if into is None else into + # Also allow arbitrary constructor kwargs, for subclasses where passing + # (some) data in at init time is desired (vs post-init copying) + # TODO: probably want to pivot the whole class this way eventually...? + # No longer recall exactly why we went with the 'fresh init + attribute + # setting' approach originally...tho there's clearly some impedance + # mismatch going on between "I want stuff to happen in my config's + # instantiation" and "I want cloning to not trigger certain things like + # external data source loading". + # NOTE: this will include lazy=True, see end of method + new = klass(**self._clone_init_kwargs(into=into)) + # Copy/merge/etc all 'private' data sources and attributes + for name in """ + collection + system_prefix + system_path + system_found + system + user_prefix + user_path + user_found + user + project_prefix + project_path + project_found + project + env_prefix + env + runtime_path + runtime_found + runtime + overrides + modifications + """.split(): + name = "_{}".format(name) + my_data = getattr(self, name) + # Non-dict data gets carried over straight (via a copy()) + # NOTE: presumably someone could really screw up and change these + # values' types, but at that point it's on them... + if not isinstance(my_data, dict): + new._set(name, copy.copy(my_data)) + # Dict data gets merged (which also involves a copy.copy + # eventually) + else: + merge_dicts(getattr(new, name), my_data) + # Do what __init__ would've done if not lazy, i.e. load user/system + # conf files. + new.load_base_conf_files() + # Finally, merge() for reals (_load_base_conf_files doesn't do so + # internally, so that data wouldn't otherwise show up.) + new.merge() + return new + + def _clone_init_kwargs( + self, into: Optional[Type["Config"]] = None + ) -> Dict[str, Any]: + """ + Supply kwargs suitable for initializing a new clone of this object. + + Note that most of the `.clone` process involves copying data between + two instances instead of passing init kwargs; however, sometimes you + really do want init kwargs, which is why this method exists. + + :param into: The value of ``into`` as passed to the calling `.clone`. + + :returns: A `dict`. + """ + # NOTE: must pass in defaults fresh or otherwise global_defaults() gets + # used instead. Except when 'into' is in play, in which case we truly + # want the union of the two. + new_defaults = copy_dict(self._defaults) + if into is not None: + merge_dicts(new_defaults, into.global_defaults()) + # The kwargs. + return dict( + defaults=new_defaults, + # TODO: consider making this 'hardcoded' on the calling end (ie + # inside clone()) to make sure nobody accidentally nukes it via + # subclassing? + lazy=True, + ) + + def _modify(self, keypath: Tuple[str, ...], key: str, value: str) -> None: + """ + Update our user-modifications config level with new data. + + :param tuple keypath: + The key path identifying the sub-dict being updated. May be an + empty tuple if the update is occurring at the topmost level. + + :param str key: + The actual key receiving an update. + + :param value: + The value being written. + """ + # First, ensure we wipe the keypath from _deletions, in case it was + # previously deleted. + excise(self._deletions, keypath + (key,)) + # Now we can add it to the modifications structure. + data = self._modifications + keypath_list = list(keypath) + while keypath_list: + subkey = keypath_list.pop(0) + # TODO: could use defaultdict here, but...meh? + if subkey not in data: + # TODO: generify this and the subsequent 3 lines... + data[subkey] = {} + data = data[subkey] + data[key] = value + self.merge() + + def _remove(self, keypath: Tuple[str, ...], key: str) -> None: + """ + Like `._modify`, but for removal. + """ + # NOTE: because deletions are processed in merge() last, we do not need + # to remove things from _modifications on removal; but we *do* do the + # inverse - remove from _deletions on modification. + # TODO: may be sane to push this step up to callers? + data = self._deletions + keypath_list = list(keypath) + while keypath_list: + subkey = keypath_list.pop(0) + if subkey in data: + data = data[subkey] + # If we encounter None, it means something higher up than our + # requested keypath is already marked as deleted; so we don't + # have to do anything or go further. + if data is None: + return + # Otherwise it's presumably another dict, so keep looping... + else: + # Key not found -> nobody's marked anything along this part of + # the path for deletion, so we'll start building it out. + data[subkey] = {} + # Then prep for next iteration + data = data[subkey] + # Exited loop -> data must be the leafmost dict, so we can now set our + # deleted key to None + data[key] = None + self.merge() + + +class AmbiguousMergeError(ValueError): + pass + + +def merge_dicts( + base: Dict[str, Any], updates: Dict[str, Any] +) -> Dict[str, Any]: + """ + Recursively merge dict ``updates`` into dict ``base`` (mutating ``base``.) + + * Values which are themselves dicts will be recursed into. + * Values which are a dict in one input and *not* a dict in the other input + (e.g. if our inputs were ``{'foo': 5}`` and ``{'foo': {'bar': 5}}``) are + irreconciliable and will generate an exception. + * Non-dict leaf values are run through `copy.copy` to avoid state bleed. + + .. note:: + This is effectively a lightweight `copy.deepcopy` which offers + protection from mismatched types (dict vs non-dict) and avoids some + core deepcopy problems (such as how it explodes on certain object + types). + + :returns: + The value of ``base``, which is mostly useful for wrapper functions + like `copy_dict`. + + .. versionadded:: 1.0 + """ + # TODO: for chrissakes just make it return instead of mutating? + for key, value in (updates or {}).items(): + # Dict values whose keys also exist in 'base' -> recurse + # (But only if both types are dicts.) + if key in base: + if isinstance(value, dict): + if isinstance(base[key], dict): + merge_dicts(base[key], value) + else: + raise _merge_error(base[key], value) + else: + if isinstance(base[key], dict): + raise _merge_error(base[key], value) + # Fileno-bearing objects are probably 'real' files which do not + # copy well & must be passed by reference. Meh. + elif hasattr(value, "fileno"): + base[key] = value + else: + base[key] = copy.copy(value) + # New values get set anew + else: + # Dict values get reconstructed to avoid being references to the + # updates dict, which can lead to nasty state-bleed bugs otherwise + if isinstance(value, dict): + base[key] = copy_dict(value) + # Fileno-bearing objects are probably 'real' files which do not + # copy well & must be passed by reference. Meh. + elif hasattr(value, "fileno"): + base[key] = value + # Non-dict values just get set straight + else: + base[key] = copy.copy(value) + return base + + +def _merge_error(orig: object, new: object) -> AmbiguousMergeError: + return AmbiguousMergeError( + "Can't cleanly merge {} with {}".format( + _format_mismatch(orig), _format_mismatch(new) + ) + ) + + +def _format_mismatch(x: object) -> str: + return "{} ({!r})".format(type(x), x) + + +def copy_dict(source: Dict[str, Any]) -> Dict[str, Any]: + """ + Return a fresh copy of ``source`` with as little shared state as possible. + + Uses `merge_dicts` under the hood, with an empty ``base`` dict; see its + documentation for details on behavior. + + .. versionadded:: 1.0 + """ + return merge_dicts({}, source) + + +def excise(dict_: Dict[str, Any], keypath: Tuple[str, ...]) -> None: + """ + Remove key pointed at by ``keypath`` from nested dict ``dict_``, if exists. + + .. versionadded:: 1.0 + """ + data = dict_ + keypath_list = list(keypath) + leaf_key = keypath_list.pop() + while keypath_list: + key = keypath_list.pop(0) + if key not in data: + # Not there, nothing to excise + return + data = data[key] + if leaf_key in data: + del data[leaf_key] + + +def obliterate(base: Dict[str, Any], deletions: Dict[str, Any]) -> None: + """ + Remove all (nested) keys mentioned in ``deletions``, from ``base``. + + .. versionadded:: 1.0 + """ + for key, value in deletions.items(): + if isinstance(value, dict): + # NOTE: not testing for whether base[key] exists; if something's + # listed in a deletions structure, it must exist in some source + # somewhere, and thus also in the cache being obliterated. + obliterate(base[key], deletions[key]) + else: # implicitly None + del base[key] diff --git a/lib/invoke/context.py b/lib/invoke/context.py new file mode 100644 index 0000000..e9beaf4 --- /dev/null +++ b/lib/invoke/context.py @@ -0,0 +1,602 @@ +import os +import re +from contextlib import contextmanager +from itertools import cycle +from os import PathLike +from typing import ( + TYPE_CHECKING, + Any, + Generator, + Iterator, + List, + Optional, + Union, +) +from unittest.mock import Mock + +from .config import Config, DataProxy +from .exceptions import Failure, AuthFailure, ResponseNotAccepted +from .runners import Result +from .watchers import FailingResponder + +if TYPE_CHECKING: + from invoke.runners import Runner + + +class Context(DataProxy): + """ + Context-aware API wrapper & state-passing object. + + `.Context` objects are created during command-line parsing (or, if desired, + by hand) and used to share parser and configuration state with executed + tasks (see :ref:`why-context`). + + Specifically, the class offers wrappers for core API calls (such as `.run`) + which take into account CLI parser flags, configuration files, and/or + changes made at runtime. It also acts as a proxy for its `~.Context.config` + attribute - see that attribute's documentation for details. + + Instances of `.Context` may be shared between tasks when executing + sub-tasks - either the same context the caller was given, or an altered + copy thereof (or, theoretically, a brand new one). + + .. versionadded:: 1.0 + """ + + def __init__(self, config: Optional[Config] = None) -> None: + """ + :param config: + `.Config` object to use as the base configuration. + + Defaults to an anonymous/default `.Config` instance. + """ + #: The fully merged `.Config` object appropriate for this context. + #: + #: `.Config` settings (see their documentation for details) may be + #: accessed like dictionary keys (``c.config['foo']``) or object + #: attributes (``c.config.foo``). + #: + #: As a convenience shorthand, the `.Context` object proxies to its + #: ``config`` attribute in the same way - e.g. ``c['foo']`` or + #: ``c.foo`` returns the same value as ``c.config['foo']``. + config = config if config is not None else Config() + self._set(_config=config) + #: A list of commands to run (via "&&") before the main argument to any + #: `run` or `sudo` calls. Note that the primary API for manipulating + #: this list is `prefix`; see its docs for details. + command_prefixes: List[str] = list() + self._set(command_prefixes=command_prefixes) + #: A list of directories to 'cd' into before running commands with + #: `run` or `sudo`; intended for management via `cd`, please see its + #: docs for details. + command_cwds: List[str] = list() + self._set(command_cwds=command_cwds) + + @property + def config(self) -> Config: + # Allows Context to expose a .config attribute even though DataProxy + # otherwise considers it a config key. + return self._config + + @config.setter + def config(self, value: Config) -> None: + # NOTE: mostly used by client libraries needing to tweak a Context's + # config at execution time; i.e. a Context subclass that bears its own + # unique data may want to be stood up when parameterizing/expanding a + # call list at start of a session, with the final config filled in at + # runtime. + self._set(_config=value) + + def run(self, command: str, **kwargs: Any) -> Optional[Result]: + """ + Execute a local shell command, honoring config options. + + Specifically, this method instantiates a `.Runner` subclass (according + to the ``runner`` config option; default is `.Local`) and calls its + ``.run`` method with ``command`` and ``kwargs``. + + See `.Runner.run` for details on ``command`` and the available keyword + arguments. + + .. versionadded:: 1.0 + """ + runner = self.config.runners.local(self) + return self._run(runner, command, **kwargs) + + # NOTE: broken out of run() to allow for runner class injection in + # Fabric/etc, which needs to juggle multiple runner class types (local and + # remote). + def _run( + self, runner: "Runner", command: str, **kwargs: Any + ) -> Optional[Result]: + command = self._prefix_commands(command) + return runner.run(command, **kwargs) + + def sudo(self, command: str, **kwargs: Any) -> Optional[Result]: + """ + Execute a shell command via ``sudo`` with password auto-response. + + **Basics** + + This method is identical to `run` but adds a handful of + convenient behaviors around invoking the ``sudo`` program. It doesn't + do anything users could not do themselves by wrapping `run`, but the + use case is too common to make users reinvent these wheels themselves. + + .. note:: + If you intend to respond to sudo's password prompt by hand, just + use ``run("sudo command")`` instead! The autoresponding features in + this method will just get in your way. + + Specifically, `sudo`: + + * Places a `.FailingResponder` into the ``watchers`` kwarg (see + :doc:`/concepts/watchers`) which: + + * searches for the configured ``sudo`` password prompt; + * responds with the configured sudo password (``sudo.password`` + from the :doc:`configuration `); + * can tell when that response causes an authentication failure + (e.g. if the system requires a password and one was not + configured), and raises `.AuthFailure` if so. + + * Builds a ``sudo`` command string using the supplied ``command`` + argument, prefixed by various flags (see below); + * Executes that command via a call to `run`, returning the result. + + **Flags used** + + ``sudo`` flags used under the hood include: + + - ``-S`` to allow auto-responding of password via stdin; + - ``-p `` to explicitly state the prompt to use, so we can be + sure our auto-responder knows what to look for; + - ``-u `` if ``user`` is not ``None``, to execute the command as + a user other than ``root``; + - When ``-u`` is present, ``-H`` is also added, to ensure the + subprocess has the requested user's ``$HOME`` set properly. + + **Configuring behavior** + + There are a couple of ways to change how this method behaves: + + - Because it wraps `run`, it honors all `run` config parameters and + keyword arguments, in the same way that `run` does. + + - Thus, invocations such as ``c.sudo('command', echo=True)`` are + possible, and if a config layer (such as a config file or env + var) specifies that e.g. ``run.warn = True``, that too will take + effect under `sudo`. + + - `sudo` has its own set of keyword arguments (see below) and they are + also all controllable via the configuration system, under the + ``sudo.*`` tree. + + - Thus you could, for example, pre-set a sudo user in a config + file; such as an ``invoke.json`` containing ``{"sudo": {"user": + "someuser"}}``. + + :param str password: Runtime override for ``sudo.password``. + :param str user: Runtime override for ``sudo.user``. + + .. versionadded:: 1.0 + """ + runner = self.config.runners.local(self) + return self._sudo(runner, command, **kwargs) + + # NOTE: this is for runner injection; see NOTE above _run(). + def _sudo( + self, runner: "Runner", command: str, **kwargs: Any + ) -> Optional[Result]: + prompt = self.config.sudo.prompt + password = kwargs.pop("password", self.config.sudo.password) + user = kwargs.pop("user", self.config.sudo.user) + env = kwargs.get("env", {}) + # TODO: allow subclassing for 'get the password' so users who REALLY + # want lazy runtime prompting can have it easily implemented. + # TODO: want to print a "cleaner" echo with just 'sudo '; but + # hard to do as-is, obtaining config data from outside a Runner one + # holds is currently messy (could fix that), if instead we manually + # inspect the config ourselves that duplicates logic. NOTE: once we + # figure that out, there is an existing, would-fail-if-not-skipped test + # for this behavior in test/context.py. + # TODO: once that is done, though: how to handle "full debug" output + # exactly (display of actual, real full sudo command w/ -S and -p), in + # terms of API/config? Impl is easy, just go back to passing echo + # through to 'run'... + user_flags = "" + if user is not None: + user_flags = "-H -u {} ".format(user) + env_flags = "" + if env: + env_flags = "--preserve-env='{}' ".format(",".join(env.keys())) + command = self._prefix_commands(command) + cmd_str = "sudo -S -p '{}' {}{}{}".format( + prompt, env_flags, user_flags, command + ) + watcher = FailingResponder( + pattern=re.escape(prompt), + response="{}\n".format(password), + sentinel="Sorry, try again.\n", + ) + # Ensure we merge any user-specified watchers with our own. + # NOTE: If there are config-driven watchers, we pull those up to the + # kwarg level; that lets us merge cleanly without needing complex + # config-driven "override vs merge" semantics. + # TODO: if/when those semantics are implemented, use them instead. + # NOTE: config value for watchers defaults to an empty list; and we + # want to clone it to avoid actually mutating the config. + watchers = kwargs.pop("watchers", list(self.config.run.watchers)) + watchers.append(watcher) + try: + return runner.run(cmd_str, watchers=watchers, **kwargs) + except Failure as failure: + # Transmute failures driven by our FailingResponder, into auth + # failures - the command never even ran. + # TODO: wants to be a hook here for users that desire "override a + # bad config value for sudo.password" manual input + # NOTE: as noted in #294 comments, we MAY in future want to update + # this so run() is given ability to raise AuthFailure on its own. + # For now that has been judged unnecessary complexity. + if isinstance(failure.reason, ResponseNotAccepted): + # NOTE: not bothering with 'reason' here, it's pointless. + error = AuthFailure(result=failure.result, prompt=prompt) + raise error + # Reraise for any other error so it bubbles up normally. + else: + raise + + # TODO: wonder if it makes sense to move this part of things inside Runner, + # which would grow a `prefixes` and `cwd` init kwargs or similar. The less + # that's stuffed into Context, probably the better. + def _prefix_commands(self, command: str) -> str: + """ + Prefixes ``command`` with all prefixes found in ``command_prefixes``. + + ``command_prefixes`` is a list of strings which is modified by the + `prefix` context manager. + """ + prefixes = list(self.command_prefixes) + current_directory = self.cwd + if current_directory: + prefixes.insert(0, "cd {}".format(current_directory)) + + return " && ".join(prefixes + [command]) + + @contextmanager + def prefix(self, command: str) -> Generator[None, None, None]: + """ + Prefix all nested `run`/`sudo` commands with given command plus ``&&``. + + Most of the time, you'll want to be using this alongside a shell script + which alters shell state, such as ones which export or alter shell + environment variables. + + For example, one of the most common uses of this tool is with the + ``workon`` command from `virtualenvwrapper + `_:: + + with c.prefix('workon myvenv'): + c.run('./manage.py migrate') + + In the above snippet, the actual shell command run would be this:: + + $ workon myvenv && ./manage.py migrate + + This context manager is compatible with `cd`, so if your virtualenv + doesn't ``cd`` in its ``postactivate`` script, you could do the + following:: + + with c.cd('/path/to/app'): + with c.prefix('workon myvenv'): + c.run('./manage.py migrate') + c.run('./manage.py loaddata fixture') + + Which would result in executions like so:: + + $ cd /path/to/app && workon myvenv && ./manage.py migrate + $ cd /path/to/app && workon myvenv && ./manage.py loaddata fixture + + Finally, as alluded to above, `prefix` may be nested if desired, e.g.:: + + with c.prefix('workon myenv'): + c.run('ls') + with c.prefix('source /some/script'): + c.run('touch a_file') + + The result:: + + $ workon myenv && ls + $ workon myenv && source /some/script && touch a_file + + Contrived, but hopefully illustrative. + + .. versionadded:: 1.0 + """ + self.command_prefixes.append(command) + try: + yield + finally: + self.command_prefixes.pop() + + @property + def cwd(self) -> str: + """ + Return the current working directory, accounting for uses of `cd`. + + .. versionadded:: 1.0 + """ + if not self.command_cwds: + # TODO: should this be None? Feels cleaner, though there may be + # benefits to it being an empty string, such as relying on a no-arg + # `cd` typically being shorthand for "go to user's $HOME". + return "" + + # get the index for the subset of paths starting with the last / or ~ + for i, path in reversed(list(enumerate(self.command_cwds))): + if path.startswith("~") or path.startswith("/"): + break + + # TODO: see if there's a stronger "escape this path" function somewhere + # we can reuse. e.g., escaping tildes or slashes in filenames. + paths = [path.replace(" ", r"\ ") for path in self.command_cwds[i:]] + return str(os.path.join(*paths)) + + @contextmanager + def cd(self, path: Union[PathLike, str]) -> Generator[None, None, None]: + """ + Context manager that keeps directory state when executing commands. + + Any calls to `run`, `sudo`, within the wrapped block will implicitly + have a string similar to ``"cd && "`` prefixed in order to give + the sense that there is actually statefulness involved. + + Because use of `cd` affects all such invocations, any code making use + of the `cwd` property will also be affected by use of `cd`. + + Like the actual 'cd' shell builtin, `cd` may be called with relative + paths (keep in mind that your default starting directory is your user's + ``$HOME``) and may be nested as well. + + Below is a "normal" attempt at using the shell 'cd', which doesn't work + since all commands are executed in individual subprocesses -- state is + **not** kept between invocations of `run` or `sudo`:: + + c.run('cd /var/www') + c.run('ls') + + The above snippet will list the contents of the user's ``$HOME`` + instead of ``/var/www``. With `cd`, however, it will work as expected:: + + with c.cd('/var/www'): + c.run('ls') # Turns into "cd /var/www && ls" + + Finally, a demonstration (see inline comments) of nesting:: + + with c.cd('/var/www'): + c.run('ls') # cd /var/www && ls + with c.cd('website1'): + c.run('ls') # cd /var/www/website1 && ls + + .. note:: + Space characters will be escaped automatically to make dealing with + such directory names easier. + + .. versionadded:: 1.0 + .. versionchanged:: 1.5 + Explicitly cast the ``path`` argument (the only argument) to a + string; this allows any object defining ``__str__`` to be handed in + (such as the various ``Path`` objects out there), and not just + string literals. + """ + path = str(path) + self.command_cwds.append(path) + try: + yield + finally: + self.command_cwds.pop() + + +class MockContext(Context): + """ + A `.Context` whose methods' return values can be predetermined. + + Primarily useful for testing Invoke-using codebases. + + .. note:: + This class wraps its ``run``, etc methods in `unittest.mock.Mock` + objects. This allows you to easily assert that the methods (still + returning the values you prepare them with) were actually called. + + .. note:: + Methods not given `Results <.Result>` to yield will raise + ``NotImplementedError`` if called (since the alternative is to call the + real underlying method - typically undesirable when mocking.) + + .. versionadded:: 1.0 + .. versionchanged:: 1.5 + Added ``Mock`` wrapping of ``run`` and ``sudo``. + """ + + def __init__(self, config: Optional[Config] = None, **kwargs: Any) -> None: + """ + Create a ``Context``-like object whose methods yield `.Result` objects. + + :param config: + A Configuration object to use. Identical in behavior to `.Context`. + + :param run: + A data structure indicating what `.Result` objects to return from + calls to the instantiated object's `~.Context.run` method (instead + of actually executing the requested shell command). + + Specifically, this kwarg accepts: + + - A single `.Result` object. + - A boolean; if True, yields a `.Result` whose ``exited`` is ``0``, + and if False, ``1``. + - An iterable of the above values, which will be returned on each + subsequent call to ``.run`` (the first item on the first call, + the second on the second call, etc). + - A dict mapping command strings or compiled regexen to the above + values (including an iterable), allowing specific + call-and-response semantics instead of assuming a call order. + + :param sudo: + Identical to ``run``, but whose values are yielded from calls to + `~.Context.sudo`. + + :param bool repeat: + A flag determining whether results yielded by this class' methods + repeat or are consumed. + + For example, when a single result is indicated, it will normally + only be returned once, causing ``NotImplementedError`` afterwards. + But when ``repeat=True`` is given, that result is returned on + every call, forever. + + Similarly, iterable results are normally exhausted once, but when + this setting is enabled, they are wrapped in `itertools.cycle`. + + Default: ``True``. + + :raises: + ``TypeError``, if the values given to ``run`` or other kwargs + aren't of the expected types. + + .. versionchanged:: 1.5 + Added support for boolean and string result values. + .. versionchanged:: 1.5 + Added support for regex dict keys. + .. versionchanged:: 1.5 + Added the ``repeat`` keyword argument. + .. versionchanged:: 2.0 + Changed ``repeat`` default value from ``False`` to ``True``. + """ + # Set up like any other Context would, with the config + super().__init__(config) + # Pull out behavioral kwargs + self._set("__repeat", kwargs.pop("repeat", True)) + # The rest must be things like run/sudo - mock Context method info + for method, results in kwargs.items(): + # For each possible value type, normalize to iterable of Result + # objects (possibly repeating). + singletons = (Result, bool, str) + if isinstance(results, dict): + for key, value in results.items(): + results[key] = self._normalize(value) + elif isinstance(results, singletons) or hasattr( + results, "__iter__" + ): + results = self._normalize(results) + # Unknown input value: cry + else: + err = "Not sure how to yield results from a {!r}" + raise TypeError(err.format(type(results))) + # Save results for use by the method + self._set("__{}".format(method), results) + # Wrap the method in a Mock + self._set(method, Mock(wraps=getattr(self, method))) + + def _normalize(self, value: Any) -> Iterator[Any]: + # First turn everything into an iterable + if not hasattr(value, "__iter__") or isinstance(value, str): + value = [value] + # Then turn everything within into a Result + results = [] + for obj in value: + if isinstance(obj, bool): + obj = Result(exited=0 if obj else 1) + elif isinstance(obj, str): + obj = Result(obj) + results.append(obj) + # Finally, turn that iterable into an iteratOR, depending on repeat + return cycle(results) if getattr(self, "__repeat") else iter(results) + + # TODO: _maybe_ make this more metaprogrammy/flexible (using __call__ etc)? + # Pretty worried it'd cause more hard-to-debug issues than it's presently + # worth. Maybe in situations where Context grows a _lot_ of methods (e.g. + # in Fabric 2; though Fabric could do its own sub-subclass in that case...) + + def _yield_result(self, attname: str, command: str) -> Result: + try: + obj = getattr(self, attname) + # Dicts need to try direct lookup or regex matching + if isinstance(obj, dict): + try: + obj = obj[command] + except KeyError: + # TODO: could optimize by skipping this if not any regex + # objects in keys()? + for key, value in obj.items(): + if hasattr(key, "match") and key.match(command): + obj = value + break + else: + # Nope, nothing did match. + raise KeyError + # Here, the value was either never a dict or has been extracted + # from one, so we can assume it's an iterable of Result objects due + # to work done by __init__. + result: Result = next(obj) + # Populate Result's command string with what matched unless + # explicitly given + if not result.command: + result.command = command + return result + except (AttributeError, IndexError, KeyError, StopIteration): + # raise_from(NotImplementedError(command), None) + raise NotImplementedError(command) + + def run(self, command: str, *args: Any, **kwargs: Any) -> Result: + # TODO: perform more convenience stuff associating args/kwargs with the + # result? E.g. filling in .command, etc? Possibly useful for debugging + # if one hits unexpected-order problems with what they passed in to + # __init__. + return self._yield_result("__run", command) + + def sudo(self, command: str, *args: Any, **kwargs: Any) -> Result: + # TODO: this completely nukes the top-level behavior of sudo(), which + # could be good or bad, depending. Most of the time I think it's good. + # No need to supply dummy password config, etc. + # TODO: see the TODO from run() re: injecting arg/kwarg values + return self._yield_result("__sudo", command) + + def set_result_for( + self, attname: str, command: str, result: Result + ) -> None: + """ + Modify the stored mock results for given ``attname`` (e.g. ``run``). + + This is similar to how one instantiates `MockContext` with a ``run`` or + ``sudo`` dict kwarg. For example, this:: + + mc = MockContext(run={'mycommand': Result("mystdout")}) + assert mc.run('mycommand').stdout == "mystdout" + + is functionally equivalent to this:: + + mc = MockContext() + mc.set_result_for('run', 'mycommand', Result("mystdout")) + assert mc.run('mycommand').stdout == "mystdout" + + `set_result_for` is mostly useful for modifying an already-instantiated + `MockContext`, such as one created by test setup or helper methods. + + .. versionadded:: 1.0 + """ + attname = "__{}".format(attname) + heck = TypeError( + "Can't update results for non-dict or nonexistent mock results!" + ) + # Get value & complain if it's not a dict. + # TODO: should we allow this to set non-dict values too? Seems vaguely + # pointless, at that point, just make a new MockContext eh? + try: + value = getattr(self, attname) + except AttributeError: + raise heck + if not isinstance(value, dict): + raise heck + # OK, we're good to modify, so do so. + value[command] = self._normalize(result) diff --git a/lib/invoke/env.py b/lib/invoke/env.py new file mode 100644 index 0000000..2c7aaa6 --- /dev/null +++ b/lib/invoke/env.py @@ -0,0 +1,123 @@ +""" +Environment variable configuration loading class. + +Using a class here doesn't really model anything but makes state passing (in a +situation requiring it) more convenient. + +This module is currently considered private/an implementation detail and should +not be included in the Sphinx API documentation. +""" + +import os +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Sequence + +from .exceptions import UncastableEnvVar, AmbiguousEnvVar +from .util import debug + +if TYPE_CHECKING: + from .config import Config + + +class Environment: + def __init__(self, config: "Config", prefix: str) -> None: + self._config = config + self._prefix = prefix + self.data: Dict[str, Any] = {} # Accumulator + + def load(self) -> Dict[str, Any]: + """ + Return a nested dict containing values from `os.environ`. + + Specifically, values whose keys map to already-known configuration + settings, allowing us to perform basic typecasting. + + See :ref:`env-vars` for details. + """ + # Obtain allowed env var -> existing value map + env_vars = self._crawl(key_path=[], env_vars={}) + m = "Scanning for env vars according to prefix: {!r}, mapping: {!r}" + debug(m.format(self._prefix, env_vars)) + # Check for actual env var (honoring prefix) and try to set + for env_var, key_path in env_vars.items(): + real_var = (self._prefix or "") + env_var + if real_var in os.environ: + self._path_set(key_path, os.environ[real_var]) + debug("Obtained env var config: {!r}".format(self.data)) + return self.data + + def _crawl( + self, key_path: List[str], env_vars: Mapping[str, Sequence[str]] + ) -> Dict[str, Any]: + """ + Examine config at location ``key_path`` & return potential env vars. + + Uses ``env_vars`` dict to determine if a conflict exists, and raises an + exception if so. This dict is of the following form:: + + { + 'EXPECTED_ENV_VAR_HERE': ['actual', 'nested', 'key_path'], + ... + } + + Returns another dictionary of new keypairs as per above. + """ + new_vars: Dict[str, List[str]] = {} + obj = self._path_get(key_path) + # Sub-dict -> recurse + if ( + hasattr(obj, "keys") + and callable(obj.keys) + and hasattr(obj, "__getitem__") + ): + for key in obj.keys(): + merged_vars = dict(env_vars, **new_vars) + merged_path = key_path + [key] + crawled = self._crawl(merged_path, merged_vars) + # Handle conflicts + for key in crawled: + if key in new_vars: + err = "Found >1 source for {}" + raise AmbiguousEnvVar(err.format(key)) + # Merge and continue + new_vars.update(crawled) + # Other -> is leaf, no recursion + else: + new_vars[self._to_env_var(key_path)] = key_path + return new_vars + + def _to_env_var(self, key_path: Iterable[str]) -> str: + return "_".join(key_path).upper() + + def _path_get(self, key_path: Iterable[str]) -> "Config": + # Gets are from self._config because that's what determines valid env + # vars and/or values for typecasting. + obj = self._config + for key in key_path: + obj = obj[key] + return obj + + def _path_set(self, key_path: Sequence[str], value: str) -> None: + # Sets are to self.data since that's what we are presenting to the + # outer config object and debugging. + obj = self.data + for key in key_path[:-1]: + if key not in obj: + obj[key] = {} + obj = obj[key] + old = self._path_get(key_path) + new = self._cast(old, value) + obj[key_path[-1]] = new + + def _cast(self, old: Any, new: Any) -> Any: + if isinstance(old, bool): + return new not in ("0", "") + elif isinstance(old, str): + return new + elif old is None: + return new + elif isinstance(old, (list, tuple)): + err = "Can't adapt an environment string into a {}!" + err = err.format(type(old)) + raise UncastableEnvVar(err) + else: + return old.__class__(new) diff --git a/lib/invoke/exceptions.py b/lib/invoke/exceptions.py new file mode 100644 index 0000000..19ca563 --- /dev/null +++ b/lib/invoke/exceptions.py @@ -0,0 +1,425 @@ +""" +Custom exception classes. + +These vary in use case from "we needed a specific data structure layout in +exceptions used for message-passing" to simply "we needed to express an error +condition in a way easily told apart from other, truly unexpected errors". +""" + +from pprint import pformat +from traceback import format_exception +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +if TYPE_CHECKING: + from .parser import ParserContext + from .runners import Result + from .util import ExceptionWrapper + + +class CollectionNotFound(Exception): + def __init__(self, name: str, start: str) -> None: + self.name = name + self.start = start + + +class Failure(Exception): + """ + Exception subclass representing failure of a command execution. + + "Failure" may mean the command executed and the shell indicated an unusual + result (usually, a non-zero exit code), or it may mean something else, like + a ``sudo`` command which was aborted when the supplied password failed + authentication. + + Two attributes allow introspection to determine the nature of the problem: + + * ``result``: a `.Result` instance with info about the command being + executed and, if it ran to completion, how it exited. + * ``reason``: a wrapped exception instance if applicable (e.g. a + `.StreamWatcher` raised `WatcherError`) or ``None`` otherwise, in which + case, it's probably a `Failure` subclass indicating its own specific + nature, such as `UnexpectedExit` or `CommandTimedOut`. + + This class is only rarely raised by itself; most of the time `.Runner.run` + (or a wrapper of same, such as `.Context.sudo`) will raise a specific + subclass like `UnexpectedExit` or `AuthFailure`. + + .. versionadded:: 1.0 + """ + + def __init__( + self, result: "Result", reason: Optional["WatcherError"] = None + ) -> None: + self.result = result + self.reason = reason + + def streams_for_display(self) -> Tuple[str, str]: + """ + Return stdout/err streams as necessary for error display. + + Subject to the following rules: + + - If a given stream was *not* hidden during execution, a placeholder is + used instead, to avoid printing it twice. + - Only the last 10 lines of stream text is included. + - PTY-driven execution will lack stderr, and a specific message to this + effect is returned instead of a stderr dump. + + :returns: Two-tuple of stdout, stderr strings. + + .. versionadded:: 1.3 + """ + already_printed = " already printed" + if "stdout" not in self.result.hide: + stdout = already_printed + else: + stdout = self.result.tail("stdout") + if self.result.pty: + stderr = " n/a (PTYs have no stderr)" + else: + if "stderr" not in self.result.hide: + stderr = already_printed + else: + stderr = self.result.tail("stderr") + return stdout, stderr + + def __repr__(self) -> str: + return self._repr() + + def _repr(self, **kwargs: Any) -> str: + """ + Return ``__repr__``-like value from inner result + any kwargs. + """ + # TODO: expand? + # TODO: truncate command? + template = "<{}: cmd={!r}{}>" + rest = "" + if kwargs: + rest = " " + " ".join( + "{}={}".format(key, value) for key, value in kwargs.items() + ) + return template.format( + self.__class__.__name__, self.result.command, rest + ) + + +class UnexpectedExit(Failure): + """ + A shell command ran to completion but exited with an unexpected exit code. + + Its string representation displays the following: + + - Command executed; + - Exit code; + - The last 10 lines of stdout, if it was hidden; + - The last 10 lines of stderr, if it was hidden and non-empty (e.g. + pty=False; when pty=True, stderr never happens.) + + .. versionadded:: 1.0 + """ + + def __str__(self) -> str: + stdout, stderr = self.streams_for_display() + command = self.result.command + exited = self.result.exited + template = """Encountered a bad command exit code! + +Command: {!r} + +Exit code: {} + +Stdout:{} + +Stderr:{} + +""" + return template.format(command, exited, stdout, stderr) + + def _repr(self, **kwargs: Any) -> str: + kwargs.setdefault("exited", self.result.exited) + return super()._repr(**kwargs) + + +class CommandTimedOut(Failure): + """ + Raised when a subprocess did not exit within a desired timeframe. + """ + + def __init__(self, result: "Result", timeout: int) -> None: + super().__init__(result) + self.timeout = timeout + + def __repr__(self) -> str: + return self._repr(timeout=self.timeout) + + def __str__(self) -> str: + stdout, stderr = self.streams_for_display() + command = self.result.command + template = """Command did not complete within {} seconds! + +Command: {!r} + +Stdout:{} + +Stderr:{} + +""" + return template.format(self.timeout, command, stdout, stderr) + + +class AuthFailure(Failure): + """ + An authentication failure, e.g. due to an incorrect ``sudo`` password. + + .. note:: + `.Result` objects attached to these exceptions typically lack exit code + information, since the command was never fully executed - the exception + was raised instead. + + .. versionadded:: 1.0 + """ + + def __init__(self, result: "Result", prompt: str) -> None: + self.result = result + self.prompt = prompt + + def __str__(self) -> str: + err = "The password submitted to prompt {!r} was rejected." + return err.format(self.prompt) + + +class ParseError(Exception): + """ + An error arising from the parsing of command-line flags/arguments. + + Ambiguous input, invalid task names, invalid flags, etc. + + .. versionadded:: 1.0 + """ + + def __init__( + self, msg: str, context: Optional["ParserContext"] = None + ) -> None: + super().__init__(msg) + self.context = context + + +class Exit(Exception): + """ + Simple custom stand-in for SystemExit. + + Replaces scattered sys.exit calls, improves testability, allows one to + catch an exit request without intercepting real SystemExits (typically an + unfriendly thing to do, as most users calling `sys.exit` rather expect it + to truly exit.) + + Defaults to a non-printing, exit-0 friendly termination behavior if the + exception is uncaught. + + If ``code`` (an int) given, that code is used to exit. + + If ``message`` (a string) given, it is printed to standard error, and the + program exits with code ``1`` by default (unless overridden by also giving + ``code`` explicitly.) + + .. versionadded:: 1.0 + """ + + def __init__( + self, message: Optional[str] = None, code: Optional[int] = None + ) -> None: + self.message = message + self._code = code + + @property + def code(self) -> int: + if self._code is not None: + return self._code + return 1 if self.message else 0 + + +class PlatformError(Exception): + """ + Raised when an illegal operation occurs for the current platform. + + E.g. Windows users trying to use functionality requiring the ``pty`` + module. + + Typically used to present a clearer error message to the user. + + .. versionadded:: 1.0 + """ + + pass + + +class AmbiguousEnvVar(Exception): + """ + Raised when loading env var config keys has an ambiguous target. + + .. versionadded:: 1.0 + """ + + pass + + +class UncastableEnvVar(Exception): + """ + Raised on attempted env var loads whose default values are too rich. + + E.g. trying to stuff ``MY_VAR="foo"`` into ``{'my_var': ['uh', 'oh']}`` + doesn't make any sense until/if we implement some sort of transform option. + + .. versionadded:: 1.0 + """ + + pass + + +class UnknownFileType(Exception): + """ + A config file of an unknown type was specified and cannot be loaded. + + .. versionadded:: 1.0 + """ + + pass + + +class UnpicklableConfigMember(Exception): + """ + A config file contained module objects, which can't be pickled/copied. + + We raise this more easily catchable exception instead of letting the + (unclearly phrased) TypeError bubble out of the pickle module. (However, to + avoid our own fragile catching of that error, we head it off by explicitly + testing for module members.) + + .. versionadded:: 1.0.2 + """ + + pass + + +def _printable_kwargs(kwargs: Any) -> Dict[str, Any]: + """ + Return print-friendly version of a thread-related ``kwargs`` dict. + + Extra care is taken with ``args`` members which are very long iterables - + those need truncating to be useful. + """ + printable = {} + for key, value in kwargs.items(): + item = value + if key == "args": + item = [] + for arg in value: + new_arg = arg + if hasattr(arg, "__len__") and len(arg) > 10: + msg = "<... remainder truncated during error display ...>" + new_arg = arg[:10] + [msg] + item.append(new_arg) + printable[key] = item + return printable + + +class ThreadException(Exception): + """ + One or more exceptions were raised within background threads. + + The real underlying exceptions are stored in the `exceptions` attribute; + see its documentation for data structure details. + + .. note:: + Threads which did not encounter an exception, do not contribute to this + exception object and thus are not present inside `exceptions`. + + .. versionadded:: 1.0 + """ + + #: A tuple of `ExceptionWrappers ` containing + #: the initial thread constructor kwargs (because `threading.Thread` + #: subclasses should always be called with kwargs) and the caught exception + #: for that thread as seen by `sys.exc_info` (so: type, value, traceback). + #: + #: .. note:: + #: The ordering of this attribute is not well-defined. + #: + #: .. note:: + #: Thread kwargs which appear to be very long (e.g. IO + #: buffers) will be truncated when printed, to avoid huge + #: unreadable error display. + exceptions: Tuple["ExceptionWrapper", ...] = tuple() + + def __init__(self, exceptions: List["ExceptionWrapper"]) -> None: + self.exceptions = tuple(exceptions) + + def __str__(self) -> str: + details = [] + for x in self.exceptions: + # Build useful display + detail = "Thread args: {}\n\n{}" + details.append( + detail.format( + pformat(_printable_kwargs(x.kwargs)), + "\n".join(format_exception(x.type, x.value, x.traceback)), + ) + ) + args = ( + len(self.exceptions), + ", ".join(x.type.__name__ for x in self.exceptions), + "\n\n".join(details), + ) + return """ +Saw {} exceptions within threads ({}): + + +{} +""".format( + *args + ) + + +class WatcherError(Exception): + """ + Generic parent exception class for `.StreamWatcher`-related errors. + + Typically, one of these exceptions indicates a `.StreamWatcher` noticed + something anomalous in an output stream, such as an authentication response + failure. + + `.Runner` catches these and attaches them to `.Failure` exceptions so they + can be referenced by intermediate code and/or act as extra info for end + users. + + .. versionadded:: 1.0 + """ + + pass + + +class ResponseNotAccepted(WatcherError): + """ + A responder/watcher class noticed a 'bad' response to its submission. + + Mostly used by `.FailingResponder` and subclasses, e.g. "oh dear I + autosubmitted a sudo password and it was incorrect." + + .. versionadded:: 1.0 + """ + + pass + + +class SubprocessPipeError(Exception): + """ + Some problem was encountered handling subprocess pipes (stdout/err/in). + + Typically only for corner cases; most of the time, errors in this area are + raised by the interpreter or the operating system, and end up wrapped in a + `.ThreadException`. + + .. versionadded:: 1.3 + """ + + pass diff --git a/lib/invoke/executor.py b/lib/invoke/executor.py new file mode 100644 index 0000000..08aa74e --- /dev/null +++ b/lib/invoke/executor.py @@ -0,0 +1,229 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +from .config import Config +from .parser import ParserContext +from .util import debug +from .tasks import Call, Task + +if TYPE_CHECKING: + from .collection import Collection + from .runners import Result + from .parser import ParseResult + + +class Executor: + """ + An execution strategy for Task objects. + + Subclasses may override various extension points to change, add or remove + behavior. + + .. versionadded:: 1.0 + """ + + def __init__( + self, + collection: "Collection", + config: Optional["Config"] = None, + core: Optional["ParseResult"] = None, + ) -> None: + """ + Initialize executor with handles to necessary data structures. + + :param collection: + A `.Collection` used to look up requested tasks (and their default + config data, if any) by name during execution. + + :param config: + An optional `.Config` holding configuration state. Defaults to an + empty `.Config` if not given. + + :param core: + An optional `.ParseResult` holding parsed core program arguments. + Defaults to ``None``. + """ + self.collection = collection + self.config = config if config is not None else Config() + self.core = core + + def execute( + self, *tasks: Union[str, Tuple[str, Dict[str, Any]], ParserContext] + ) -> Dict["Task", "Result"]: + """ + Execute one or more ``tasks`` in sequence. + + :param tasks: + An all-purpose iterable of "tasks to execute", each member of which + may take one of the following forms: + + **A string** naming a task from the Executor's `.Collection`. This + name may contain dotted syntax appropriate for calling namespaced + tasks, e.g. ``subcollection.taskname``. Such tasks are executed + without arguments. + + **A two-tuple** whose first element is a task name string (as + above) and whose second element is a dict suitable for use as + ``**kwargs`` when calling the named task. E.g.:: + + [ + ('task1', {}), + ('task2', {'arg1': 'val1'}), + ... + ] + + is equivalent, roughly, to:: + + task1() + task2(arg1='val1') + + **A `.ParserContext`** instance, whose ``.name`` attribute is used + as the task name and whose ``.as_kwargs`` attribute is used as the + task kwargs (again following the above specifications). + + .. note:: + When called without any arguments at all (i.e. when ``*tasks`` + is empty), the default task from ``self.collection`` is used + instead, if defined. + + :returns: + A dict mapping task objects to their return values. + + This dict may include pre- and post-tasks if any were executed. For + example, in a collection with a ``build`` task depending on another + task named ``setup``, executing ``build`` will result in a dict + with two keys, one for ``build`` and one for ``setup``. + + .. versionadded:: 1.0 + """ + # Normalize input + debug("Examining top level tasks {!r}".format([x for x in tasks])) + calls = self.normalize(tasks) + debug("Tasks (now Calls) with kwargs: {!r}".format(calls)) + # Obtain copy of directly-given tasks since they should sometimes + # behave differently + direct = list(calls) + # Expand pre/post tasks + # TODO: may make sense to bundle expansion & deduping now eh? + expanded = self.expand_calls(calls) + # Get some good value for dedupe option, even if config doesn't have + # the tree we expect. (This is a concession to testing.) + try: + dedupe = self.config.tasks.dedupe + except AttributeError: + dedupe = True + # Dedupe across entire run now that we know about all calls in order + calls = self.dedupe(expanded) if dedupe else expanded + # Execute + results = {} + # TODO: maybe clone initial config here? Probably not necessary, + # especially given Executor is not designed to execute() >1 time at the + # moment... + for call in calls: + autoprint = call in direct and call.autoprint + debug("Executing {!r}".format(call)) + # Hand in reference to our config, which will preserve user + # modifications across the lifetime of the session. + config = self.config + # But make sure we reset its task-sensitive levels each time + # (collection & shell env) + # TODO: load_collection needs to be skipped if task is anonymous + # (Fabric 2 or other subclassing libs only) + collection_config = self.collection.configuration(call.called_as) + config.load_collection(collection_config) + config.load_shell_env() + debug("Finished loading collection & shell env configs") + # Get final context from the Call (which will know how to generate + # an appropriate one; e.g. subclasses might use extra data from + # being parameterized), handing in this config for use there. + context = call.make_context(config) + args = (context, *call.args) + result = call.task(*args, **call.kwargs) + if autoprint: + print(result) + # TODO: handle the non-dedupe case / the same-task-different-args + # case, wherein one task obj maps to >1 result. + results[call.task] = result + return results + + def normalize( + self, + tasks: Tuple[ + Union[str, Tuple[str, Dict[str, Any]], ParserContext], ... + ], + ) -> List["Call"]: + """ + Transform arbitrary task list w/ various types, into `.Call` objects. + + See docstring for `~.Executor.execute` for details. + + .. versionadded:: 1.0 + """ + calls = [] + for task in tasks: + name: Optional[str] + if isinstance(task, str): + name = task + kwargs = {} + elif isinstance(task, ParserContext): + name = task.name + kwargs = task.as_kwargs + else: + name, kwargs = task + c = Call(self.collection[name], kwargs=kwargs, called_as=name) + calls.append(c) + if not tasks and self.collection.default is not None: + calls = [Call(self.collection[self.collection.default])] + return calls + + def dedupe(self, calls: List["Call"]) -> List["Call"]: + """ + Deduplicate a list of `tasks <.Call>`. + + :param calls: An iterable of `.Call` objects representing tasks. + + :returns: A list of `.Call` objects. + + .. versionadded:: 1.0 + """ + deduped = [] + debug("Deduplicating tasks...") + for call in calls: + if call not in deduped: + debug("{!r}: no duplicates found, ok".format(call)) + deduped.append(call) + else: + debug("{!r}: found in list already, skipping".format(call)) + return deduped + + def expand_calls(self, calls: List["Call"]) -> List["Call"]: + """ + Expand a list of `.Call` objects into a near-final list of same. + + The default implementation of this method simply adds a task's + pre/post-task list before/after the task itself, as necessary. + + Subclasses may wish to do other things in addition (or instead of) the + above, such as multiplying the `calls <.Call>` by argument vectors or + similar. + + .. versionadded:: 1.0 + """ + ret = [] + for call in calls: + # Normalize to Call (this method is sometimes called with pre/post + # task lists, which may contain 'raw' Task objects) + if isinstance(call, Task): + call = Call(call) + debug("Expanding task-call {!r}".format(call)) + # TODO: this is where we _used_ to call Executor.config_for(call, + # config)... + # TODO: now we may need to preserve more info like where the call + # came from, etc, but I feel like that shit should go _on the call + # itself_ right??? + # TODO: we _probably_ don't even want the config in here anymore, + # we want this to _just_ be about the recursion across pre/post + # tasks or parameterization...? + ret.extend(self.expand_calls(call.pre)) + ret.append(call) + ret.extend(self.expand_calls(call.post)) + return ret diff --git a/lib/invoke/loader.py b/lib/invoke/loader.py new file mode 100644 index 0000000..801d163 --- /dev/null +++ b/lib/invoke/loader.py @@ -0,0 +1,154 @@ +import os +import sys +from importlib.machinery import ModuleSpec +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from types import ModuleType +from typing import Any, Optional, Tuple + +from . import Config +from .exceptions import CollectionNotFound +from .util import debug + + +class Loader: + """ + Abstract class defining how to find/import a session's base `.Collection`. + + .. versionadded:: 1.0 + """ + + def __init__(self, config: Optional["Config"] = None) -> None: + """ + Set up a new loader with some `.Config`. + + :param config: + An explicit `.Config` to use; it is referenced for loading-related + config options. Defaults to an anonymous ``Config()`` if none is + given. + """ + if config is None: + config = Config() + self.config = config + + def find(self, name: str) -> Optional[ModuleSpec]: + """ + Implementation-specific finder method seeking collection ``name``. + + Must return a ModuleSpec valid for use by `importlib`, which is + typically a name string followed by the contents of the 3-tuple + returned by `importlib.module_from_spec` (``name``, ``loader``, + ``origin``.) + + For a sample implementation, see `.FilesystemLoader`. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def load(self, name: Optional[str] = None) -> Tuple[ModuleType, str]: + """ + Load and return collection module identified by ``name``. + + This method requires a working implementation of `.find` in order to + function. + + In addition to importing the named module, it will add the module's + parent directory to the front of `sys.path` to provide normal Python + import behavior (i.e. so the loaded module may load local-to-it modules + or packages.) + + :returns: + Two-tuple of ``(module, directory)`` where ``module`` is the + collection-containing Python module object, and ``directory`` is + the string path to the directory the module was found in. + + .. versionadded:: 1.0 + """ + if name is None: + name = self.config.tasks.collection_name + spec = self.find(name) + if spec and spec.loader and spec.origin: + # Typically either tasks.py or tasks/__init__.py + source_file = Path(spec.origin) + # Will be 'the dir tasks.py is in', or 'tasks/', in both cases this + # is what wants to be in sys.path for "from . import sibling" + enclosing_dir = source_file.parent + # Will be "the directory above the spot that 'import tasks' found", + # namely the parent of "your task tree", i.e. "where project level + # config files are looked for". So, same as enclosing_dir for + # tasks.py, but one more level up for tasks/__init__.py... + module_parent = enclosing_dir + if spec.parent: # it's a package, so we have to go up again + module_parent = module_parent.parent + # Get the enclosing dir on the path + enclosing_str = str(enclosing_dir) + if enclosing_str not in sys.path: + sys.path.insert(0, enclosing_str) + # Actual import + module = module_from_spec(spec) + sys.modules[spec.name] = module # so 'from . import xxx' works + spec.loader.exec_module(module) + # Return the module and the folder it was found in + return module, str(module_parent) + msg = "ImportError loading {!r}, raising ImportError" + debug(msg.format(name)) + raise ImportError + + +class FilesystemLoader(Loader): + """ + Loads Python files from the filesystem (e.g. ``tasks.py``.) + + Searches recursively towards filesystem root from a given start point. + + .. versionadded:: 1.0 + """ + + # TODO: could introduce config obj here for transmission to Collection + # TODO: otherwise Loader has to know about specific bits to transmit, such + # as auto-dashes, and has to grow one of those for every bit Collection + # ever needs to know + def __init__(self, start: Optional[str] = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + if start is None: + start = self.config.tasks.search_root + self._start = start + + @property + def start(self) -> str: + # Lazily determine default CWD if configured value is falsey + return self._start or os.getcwd() + + def find(self, name: str) -> Optional[ModuleSpec]: + debug("FilesystemLoader find starting at {!r}".format(self.start)) + spec = None + module = "{}.py".format(name) + paths = self.start.split(os.sep) + try: + # walk the path upwards to check for dynamic import + for x in reversed(range(len(paths) + 1)): + path = os.sep.join(paths[0:x]) + if module in os.listdir(path): + spec = spec_from_file_location( + name, os.path.join(path, module) + ) + break + elif name in os.listdir(path) and os.path.exists( + os.path.join(path, name, "__init__.py") + ): + basepath = os.path.join(path, name) + spec = spec_from_file_location( + name, + os.path.join(basepath, "__init__.py"), + submodule_search_locations=[basepath], + ) + break + if spec: + debug("Found module: {!r}".format(spec)) + return spec + except (FileNotFoundError, ModuleNotFoundError): + msg = "ImportError loading {!r}, raising CollectionNotFound" + debug(msg.format(name)) + raise CollectionNotFound(name=name, start=self.start) + return None diff --git a/lib/invoke/main.py b/lib/invoke/main.py new file mode 100644 index 0000000..3576b5a --- /dev/null +++ b/lib/invoke/main.py @@ -0,0 +1,14 @@ +""" +Invoke's own 'binary' entrypoint. + +Dogfoods the `program` module. +""" + +from . import __version__, Program + +program = Program( + name="Invoke", + binary="inv[oke]", + binary_names=["invoke", "inv"], + version=__version__, +) diff --git a/lib/invoke/parser/__init__.py b/lib/invoke/parser/__init__.py new file mode 100644 index 0000000..02aa026 --- /dev/null +++ b/lib/invoke/parser/__init__.py @@ -0,0 +1,5 @@ +# flake8: noqa +from .parser import * +from .context import ParserContext +from .context import ParserContext as Context, to_flag, translate_underscores +from .argument import Argument diff --git a/lib/invoke/parser/__pycache__/__init__.cpython-314.pyc b/lib/invoke/parser/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..b450e4e Binary files /dev/null and b/lib/invoke/parser/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/parser/__pycache__/argument.cpython-314.pyc b/lib/invoke/parser/__pycache__/argument.cpython-314.pyc new file mode 100644 index 0000000..cfe4dae Binary files /dev/null and b/lib/invoke/parser/__pycache__/argument.cpython-314.pyc differ diff --git a/lib/invoke/parser/__pycache__/context.cpython-314.pyc b/lib/invoke/parser/__pycache__/context.cpython-314.pyc new file mode 100644 index 0000000..d372a56 Binary files /dev/null and b/lib/invoke/parser/__pycache__/context.cpython-314.pyc differ diff --git a/lib/invoke/parser/__pycache__/parser.cpython-314.pyc b/lib/invoke/parser/__pycache__/parser.cpython-314.pyc new file mode 100644 index 0000000..c2b4d01 Binary files /dev/null and b/lib/invoke/parser/__pycache__/parser.cpython-314.pyc differ diff --git a/lib/invoke/parser/argument.py b/lib/invoke/parser/argument.py new file mode 100644 index 0000000..761eb60 --- /dev/null +++ b/lib/invoke/parser/argument.py @@ -0,0 +1,178 @@ +from typing import Any, Iterable, Optional, Tuple + +# TODO: dynamic type for kind +# T = TypeVar('T') + + +class Argument: + """ + A command-line argument/flag. + + :param name: + Syntactic sugar for ``names=[]``. Giving both ``name`` and + ``names`` is invalid. + :param names: + List of valid identifiers for this argument. For example, a "help" + argument may be defined with a name list of ``['-h', '--help']``. + :param kind: + Type factory & parser hint. E.g. ``int`` will turn the default text + value parsed, into a Python integer; and ``bool`` will tell the + parser not to expect an actual value but to treat the argument as a + toggle/flag. + :param default: + Default value made available to the parser if no value is given on the + command line. + :param help: + Help text, intended for use with ``--help``. + :param positional: + Whether or not this argument's value may be given positionally. When + ``False`` (default) arguments must be explicitly named. + :param optional: + Whether or not this (non-``bool``) argument requires a value. + :param incrementable: + Whether or not this (``int``) argument is to be incremented instead of + overwritten/assigned to. + :param attr_name: + A Python identifier/attribute friendly name, typically filled in with + the underscored version when ``name``/``names`` contain dashes. + + .. versionadded:: 1.0 + """ + + def __init__( + self, + name: Optional[str] = None, + names: Iterable[str] = (), + kind: Any = str, + default: Optional[Any] = None, + help: Optional[str] = None, + positional: bool = False, + optional: bool = False, + incrementable: bool = False, + attr_name: Optional[str] = None, + ) -> None: + if name and names: + raise TypeError( + "Cannot give both 'name' and 'names' arguments! Pick one." + ) + if not (name or names): + raise TypeError("An Argument must have at least one name.") + if names: + self.names = tuple(names) + elif name and not names: + self.names = (name,) + self.kind = kind + initial_value: Optional[Any] = None + # Special case: list-type args start out as empty list, not None. + if kind is list: + initial_value = [] + # Another: incrementable args start out as their default value. + if incrementable: + initial_value = default + self.raw_value = self._value = initial_value + self.default = default + self.help = help + self.positional = positional + self.optional = optional + self.incrementable = incrementable + self.attr_name = attr_name + + def __repr__(self) -> str: + nicks = "" + if self.nicknames: + nicks = " ({})".format(", ".join(self.nicknames)) + flags = "" + if self.positional or self.optional: + flags = " " + if self.positional: + flags += "*" + if self.optional: + flags += "?" + # TODO: store this default value somewhere other than signature of + # Argument.__init__? + kind = "" + if self.kind != str: + kind = " [{}]".format(self.kind.__name__) + return "<{}: {}{}{}{}>".format( + self.__class__.__name__, self.name, nicks, kind, flags + ) + + @property + def name(self) -> Optional[str]: + """ + The canonical attribute-friendly name for this argument. + + Will be ``attr_name`` (if given to constructor) or the first name in + ``names`` otherwise. + + .. versionadded:: 1.0 + """ + return self.attr_name or self.names[0] + + @property + def nicknames(self) -> Tuple[str, ...]: + return self.names[1:] + + @property + def takes_value(self) -> bool: + if self.kind is bool: + return False + if self.incrementable: + return False + return True + + @property + def value(self) -> Any: + # TODO: should probably be optional instead + return self._value if self._value is not None else self.default + + @value.setter + def value(self, arg: str) -> None: + self.set_value(arg, cast=True) + + def set_value(self, value: Any, cast: bool = True) -> None: + """ + Actual explicit value-setting API call. + + Sets ``self.raw_value`` to ``value`` directly. + + Sets ``self.value`` to ``self.kind(value)``, unless: + + - ``cast=False``, in which case the raw value is also used. + - ``self.kind==list``, in which case the value is appended to + ``self.value`` instead of cast & overwritten. + - ``self.incrementable==True``, in which case the value is ignored and + the current (assumed int) value is simply incremented. + + .. versionadded:: 1.0 + """ + self.raw_value = value + # Default to do-nothing/identity function + func = lambda x: x + # If cast, set to self.kind, which should be str/int/etc + if cast: + func = self.kind + # If self.kind is a list, append instead of using cast func. + if self.kind is list: + func = lambda x: self.value + [x] + # If incrementable, just increment. + if self.incrementable: + # TODO: explode nicely if self.value was not an int to start + # with + func = lambda x: self.value + 1 + self._value = func(value) + + @property + def got_value(self) -> bool: + """ + Returns whether the argument was ever given a (non-default) value. + + For most argument kinds, this simply checks whether the internally + stored value is non-``None``; for others, such as ``list`` kinds, + different checks may be used. + + .. versionadded:: 1.3 + """ + if self.kind is list: + return bool(self._value) + return self._value is not None diff --git a/lib/invoke/parser/context.py b/lib/invoke/parser/context.py new file mode 100644 index 0000000..359e9f9 --- /dev/null +++ b/lib/invoke/parser/context.py @@ -0,0 +1,266 @@ +import itertools +from typing import Any, Dict, List, Iterable, Optional, Tuple, Union + +try: + from ..vendor.lexicon import Lexicon +except ImportError: + from lexicon import Lexicon # type: ignore[no-redef] + +from .argument import Argument + + +def translate_underscores(name: str) -> str: + return name.lstrip("_").rstrip("_").replace("_", "-") + + +def to_flag(name: str) -> str: + name = translate_underscores(name) + if len(name) == 1: + return "-" + name + return "--" + name + + +def sort_candidate(arg: Argument) -> str: + names = arg.names + # TODO: is there no "split into two buckets on predicate" builtin? + shorts = {x for x in names if len(x.strip("-")) == 1} + longs = {x for x in names if x not in shorts} + return str(sorted(shorts if shorts else longs)[0]) + + +def flag_key(arg: Argument) -> List[Union[int, str]]: + """ + Obtain useful key list-of-ints for sorting CLI flags. + + .. versionadded:: 1.0 + """ + # Setup + ret: List[Union[int, str]] = [] + x = sort_candidate(arg) + # Long-style flags win over short-style ones, so the first item of + # comparison is simply whether the flag is a single character long (with + # non-length-1 flags coming "first" [lower number]) + ret.append(1 if len(x) == 1 else 0) + # Next item of comparison is simply the strings themselves, + # case-insensitive. They will compare alphabetically if compared at this + # stage. + ret.append(x.lower()) + # Finally, if the case-insensitive test also matched, compare + # case-sensitive, but inverse (with lowercase letters coming first) + inversed = "" + for char in x: + inversed += char.lower() if char.isupper() else char.upper() + ret.append(inversed) + return ret + + +# Named slightly more verbose so Sphinx references can be unambiguous. +# Got real sick of fully qualified paths. +class ParserContext: + """ + Parsing context with knowledge of flags & their format. + + Generally associated with the core program or a task. + + When run through a parser, will also hold runtime values filled in by the + parser. + + .. versionadded:: 1.0 + """ + + def __init__( + self, + name: Optional[str] = None, + aliases: Iterable[str] = (), + args: Iterable[Argument] = (), + ) -> None: + """ + Create a new ``ParserContext`` named ``name``, with ``aliases``. + + ``name`` is optional, and should be a string if given. It's used to + tell ParserContext objects apart, and for use in a Parser when + determining what chunk of input might belong to a given ParserContext. + + ``aliases`` is also optional and should be an iterable containing + strings. Parsing will honor any aliases when trying to "find" a given + context in its input. + + May give one or more ``args``, which is a quick alternative to calling + ``for arg in args: self.add_arg(arg)`` after initialization. + """ + self.args = Lexicon() + self.positional_args: List[Argument] = [] + self.flags = Lexicon() + self.inverse_flags: Dict[str, str] = {} # No need for Lexicon here + self.name = name + self.aliases = aliases + for arg in args: + self.add_arg(arg) + + def __repr__(self) -> str: + aliases = "" + if self.aliases: + aliases = " ({})".format(", ".join(self.aliases)) + name = (" {!r}{}".format(self.name, aliases)) if self.name else "" + args = (": {!r}".format(self.args)) if self.args else "" + return "".format(name, args) + + def add_arg(self, *args: Any, **kwargs: Any) -> None: + """ + Adds given ``Argument`` (or constructor args for one) to this context. + + The Argument in question is added to the following dict attributes: + + * ``args``: "normal" access, i.e. the given names are directly exposed + as keys. + * ``flags``: "flaglike" access, i.e. the given names are translated + into CLI flags, e.g. ``"foo"`` is accessible via ``flags['--foo']``. + * ``inverse_flags``: similar to ``flags`` but containing only the + "inverse" versions of boolean flags which default to True. This + allows the parser to track e.g. ``--no-myflag`` and turn it into a + False value for the ``myflag`` Argument. + + .. versionadded:: 1.0 + """ + # Normalize + if len(args) == 1 and isinstance(args[0], Argument): + arg = args[0] + else: + arg = Argument(*args, **kwargs) + # Uniqueness constraint: no name collisions + for name in arg.names: + if name in self.args: + msg = "Tried to add an argument named {!r} but one already exists!" # noqa + raise ValueError(msg.format(name)) + # First name used as "main" name for purposes of aliasing + main = arg.names[0] # NOT arg.name + self.args[main] = arg + # Note positionals in distinct, ordered list attribute + if arg.positional: + self.positional_args.append(arg) + # Add names & nicknames to flags, args + self.flags[to_flag(main)] = arg + for name in arg.nicknames: + self.args.alias(name, to=main) + self.flags.alias(to_flag(name), to=to_flag(main)) + # Add attr_name to args, but not flags + if arg.attr_name: + self.args.alias(arg.attr_name, to=main) + # Add to inverse_flags if required + if arg.kind == bool and arg.default is True: + # Invert the 'main' flag name here, which will be a dashed version + # of the primary argument name if underscore-to-dash transformation + # occurred. + inverse_name = to_flag("no-{}".format(main)) + self.inverse_flags[inverse_name] = to_flag(main) + + @property + def missing_positional_args(self) -> List[Argument]: + return [x for x in self.positional_args if x.value is None] + + @property + def as_kwargs(self) -> Dict[str, Any]: + """ + This context's arguments' values keyed by their ``.name`` attribute. + + Results in a dict suitable for use in Python contexts, where e.g. an + arg named ``foo-bar`` becomes accessible as ``foo_bar``. + + .. versionadded:: 1.0 + """ + ret = {} + for arg in self.args.values(): + ret[arg.name] = arg.value + return ret + + def names_for(self, flag: str) -> List[str]: + # TODO: should probably be a method on Lexicon/AliasDict + return list(set([flag] + self.flags.aliases_of(flag))) + + def help_for(self, flag: str) -> Tuple[str, str]: + """ + Return 2-tuple of ``(flag-spec, help-string)`` for given ``flag``. + + .. versionadded:: 1.0 + """ + # Obtain arg obj + if flag not in self.flags: + err = "{!r} is not a valid flag for this context! Valid flags are: {!r}" # noqa + raise ValueError(err.format(flag, self.flags.keys())) + arg = self.flags[flag] + # Determine expected value type, if any + value = {str: "STRING", int: "INT"}.get(arg.kind) + # Format & go + full_names = [] + for name in self.names_for(flag): + if value: + # Short flags are -f VAL, long are --foo=VAL + # When optional, also, -f [VAL] and --foo[=VAL] + if len(name.strip("-")) == 1: + value_ = ("[{}]".format(value)) if arg.optional else value + valuestr = " {}".format(value_) + else: + valuestr = "={}".format(value) + if arg.optional: + valuestr = "[{}]".format(valuestr) + else: + # no value => boolean + # check for inverse + if name in self.inverse_flags.values(): + name = "--[no-]{}".format(name[2:]) + + valuestr = "" + # Tack together + full_names.append(name + valuestr) + namestr = ", ".join(sorted(full_names, key=len)) + helpstr = arg.help or "" + return namestr, helpstr + + def help_tuples(self) -> List[Tuple[str, Optional[str]]]: + """ + Return sorted iterable of help tuples for all member Arguments. + + Sorts like so: + + * General sort is alphanumerically + * Short flags win over long flags + * Arguments with *only* long flags and *no* short flags will come + first. + * When an Argument has multiple long or short flags, it will sort using + the most favorable (lowest alphabetically) candidate. + + This will result in a help list like so:: + + --alpha, --zeta # 'alpha' wins + --beta + -a, --query # short flag wins + -b, --argh + -c + + .. versionadded:: 1.0 + """ + # TODO: argument/flag API must change :( + # having to call to_flag on 1st name of an Argument is just dumb. + # To pass in an Argument object to help_for may require moderate + # changes? + return list( + map( + lambda x: self.help_for(to_flag(x.name)), + sorted(self.flags.values(), key=flag_key), + ) + ) + + def flag_names(self) -> Tuple[str, ...]: + """ + Similar to `help_tuples` but returns flag names only, no helpstrs. + + Specifically, all flag names, flattened, in rough order. + + .. versionadded:: 1.0 + """ + # Regular flag names + flags = sorted(self.flags.values(), key=flag_key) + names = [self.names_for(to_flag(x.name)) for x in flags] + # Inverse flag names sold separately + names.append(list(self.inverse_flags.keys())) + return tuple(itertools.chain.from_iterable(names)) diff --git a/lib/invoke/parser/parser.py b/lib/invoke/parser/parser.py new file mode 100644 index 0000000..43e95df --- /dev/null +++ b/lib/invoke/parser/parser.py @@ -0,0 +1,455 @@ +import copy +from typing import TYPE_CHECKING, Any, Iterable, List, Optional + +try: + from ..vendor.lexicon import Lexicon + from ..vendor.fluidity import StateMachine, state, transition +except ImportError: + from lexicon import Lexicon # type: ignore[no-redef] + from fluidity import ( # type: ignore[no-redef] + StateMachine, + state, + transition, + ) + +from ..exceptions import ParseError +from ..util import debug + +if TYPE_CHECKING: + from .context import ParserContext + + +def is_flag(value: str) -> bool: + return value.startswith("-") + + +def is_long_flag(value: str) -> bool: + return value.startswith("--") + + +class ParseResult(List["ParserContext"]): + """ + List-like object with some extra parse-related attributes. + + Specifically, a ``.remainder`` attribute, which is the string found after a + ``--`` in any parsed argv list; and an ``.unparsed`` attribute, a list of + tokens that were unable to be parsed. + + .. versionadded:: 1.0 + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.remainder = "" + self.unparsed: List[str] = [] + + +class Parser: + """ + Create parser conscious of ``contexts`` and optional ``initial`` context. + + ``contexts`` should be an iterable of ``Context`` instances which will be + searched when new context names are encountered during a parse. These + Contexts determine what flags may follow them, as well as whether given + flags take values. + + ``initial`` is optional and will be used to determine validity of "core" + options/flags at the start of the parse run, if any are encountered. + + ``ignore_unknown`` determines what to do when contexts are found which do + not map to any members of ``contexts``. By default it is ``False``, meaning + any unknown contexts result in a parse error exception. If ``True``, + encountering an unknown context halts parsing and populates the return + value's ``.unparsed`` attribute with the remaining parse tokens. + + .. versionadded:: 1.0 + """ + + def __init__( + self, + contexts: Iterable["ParserContext"] = (), + initial: Optional["ParserContext"] = None, + ignore_unknown: bool = False, + ) -> None: + self.initial = initial + self.contexts = Lexicon() + self.ignore_unknown = ignore_unknown + for context in contexts: + debug("Adding {}".format(context)) + if not context.name: + raise ValueError("Non-initial contexts must have names.") + exists = "A context named/aliased {!r} is already in this parser!" + if context.name in self.contexts: + raise ValueError(exists.format(context.name)) + self.contexts[context.name] = context + for alias in context.aliases: + if alias in self.contexts: + raise ValueError(exists.format(alias)) + self.contexts.alias(alias, to=context.name) + + def parse_argv(self, argv: List[str]) -> ParseResult: + """ + Parse an argv-style token list ``argv``. + + Returns a list (actually a subclass, `.ParseResult`) of + `.ParserContext` objects matching the order they were found in the + ``argv`` and containing `.Argument` objects with updated values based + on any flags given. + + Assumes any program name has already been stripped out. Good:: + + Parser(...).parse_argv(['--core-opt', 'task', '--task-opt']) + + Bad:: + + Parser(...).parse_argv(['invoke', '--core-opt', ...]) + + :param argv: List of argument string tokens. + :returns: + A `.ParseResult` (a ``list`` subclass containing some number of + `.ParserContext` objects). + + .. versionadded:: 1.0 + """ + machine = ParseMachine( + # FIXME: initial should not be none + initial=self.initial, # type: ignore[arg-type] + contexts=self.contexts, + ignore_unknown=self.ignore_unknown, + ) + # FIXME: Why isn't there str.partition for lists? There must be a + # better way to do this. Split argv around the double-dash remainder + # sentinel. + debug("Starting argv: {!r}".format(argv)) + try: + ddash = argv.index("--") + except ValueError: + ddash = len(argv) # No remainder == body gets all + body = argv[:ddash] + remainder = argv[ddash:][1:] # [1:] to strip off remainder itself + if remainder: + debug( + "Remainder: argv[{!r}:][1:] => {!r}".format(ddash, remainder) + ) + for index, token in enumerate(body): + # Handle non-space-delimited forms, if not currently expecting a + # flag value and still in valid parsing territory (i.e. not in + # "unknown" state which implies store-only) + # NOTE: we do this in a few steps so we can + # split-then-check-validity; necessary for things like when the + # previously seen flag optionally takes a value. + mutations = [] + orig = token + if is_flag(token) and not machine.result.unparsed: + # Equals-sign-delimited flags, eg --foo=bar or -f=bar + if "=" in token: + token, _, value = token.partition("=") + msg = "Splitting x=y expr {!r} into tokens {!r} and {!r}" + debug(msg.format(orig, token, value)) + mutations.append((index + 1, value)) + # Contiguous boolean short flags, e.g. -qv + elif not is_long_flag(token) and len(token) > 2: + full_token = token[:] + rest, token = token[2:], token[:2] + err = "Splitting {!r} into token {!r} and rest {!r}" + debug(err.format(full_token, token, rest)) + # Handle boolean flag block vs short-flag + value. Make + # sure not to test the token as a context flag if we've + # passed into 'storing unknown stuff' territory (e.g. on a + # core-args pass, handling what are going to be task args) + have_flag = ( + token in machine.context.flags + and machine.current_state != "unknown" + ) + if have_flag and machine.context.flags[token].takes_value: + msg = "{!r} is a flag for current context & it takes a value, giving it {!r}" # noqa + debug(msg.format(token, rest)) + mutations.append((index + 1, rest)) + else: + _rest = ["-{}".format(x) for x in rest] + msg = "Splitting multi-flag glob {!r} into {!r} and {!r}" # noqa + debug(msg.format(orig, token, _rest)) + for item in reversed(_rest): + mutations.append((index + 1, item)) + # Here, we've got some possible mutations queued up, and 'token' + # may have been overwritten as well. Whether we apply those and + # continue as-is, or roll it back, depends: + # - If the parser wasn't waiting for a flag value, we're already on + # the right track, so apply mutations and move along to the + # handle() step. + # - If we ARE waiting for a value, and the flag expecting it ALWAYS + # wants a value (it's not optional), we go back to using the + # original token. (TODO: could reorganize this to avoid the + # sub-parsing in this case, but optimizing for human-facing + # execution isn't critical.) + # - Finally, if we are waiting for a value AND it's optional, we + # inspect the first sub-token/mutation to see if it would otherwise + # have been a valid flag, and let that determine what we do (if + # valid, we apply the mutations; if invalid, we reinstate the + # original token.) + if machine.waiting_for_flag_value: + optional = machine.flag and machine.flag.optional + subtoken_is_valid_flag = token in machine.context.flags + if not (optional and subtoken_is_valid_flag): + token = orig + mutations = [] + for index, value in mutations: + body.insert(index, value) + machine.handle(token) + machine.finish() + result = machine.result + result.remainder = " ".join(remainder) + return result + + +class ParseMachine(StateMachine): + initial_state = "context" + + state("context", enter=["complete_flag", "complete_context"]) + state("unknown", enter=["complete_flag", "complete_context"]) + state("end", enter=["complete_flag", "complete_context"]) + + transition(from_=("context", "unknown"), event="finish", to="end") + transition( + from_="context", + event="see_context", + action="switch_to_context", + to="context", + ) + transition( + from_=("context", "unknown"), + event="see_unknown", + action="store_only", + to="unknown", + ) + + def changing_state(self, from_: str, to: str) -> None: + debug("ParseMachine: {!r} => {!r}".format(from_, to)) + + def __init__( + self, + initial: "ParserContext", + contexts: Lexicon, + ignore_unknown: bool, + ) -> None: + # Initialize + self.ignore_unknown = ignore_unknown + self.initial = self.context = copy.deepcopy(initial) + debug("Initialized with context: {!r}".format(self.context)) + self.flag = None + self.flag_got_value = False + self.result = ParseResult() + self.contexts = copy.deepcopy(contexts) + debug("Available contexts: {!r}".format(self.contexts)) + # In case StateMachine does anything in __init__ + super().__init__() + + @property + def waiting_for_flag_value(self) -> bool: + # Do we have a current flag, and does it expect a value (vs being a + # bool/toggle)? + takes_value = self.flag and self.flag.takes_value + if not takes_value: + return False + # OK, this flag is one that takes values. + # Is it a list type (which has only just been switched to)? Then it'll + # always accept more values. + # TODO: how to handle somebody wanting it to be some other iterable + # like tuple or custom class? Or do we just say unsupported? + if self.flag.kind is list and not self.flag_got_value: + return True + # Not a list, okay. Does it already have a value? + has_value = self.flag.raw_value is not None + # If it doesn't have one, we're waiting for one (which tells the parser + # how to proceed and typically to store the next token.) + # TODO: in the negative case here, we should do something else instead: + # - Except, "hey you screwed up, you already gave that flag!" + # - Overwrite, "oh you changed your mind?" - which requires more work + # elsewhere too, unfortunately. (Perhaps additional properties on + # Argument that can be queried, e.g. "arg.is_iterable"?) + return not has_value + + def handle(self, token: str) -> None: + debug("Handling token: {!r}".format(token)) + # Handle unknown state at the top: we don't care about even + # possibly-valid input if we've encountered unknown input. + if self.current_state == "unknown": + debug("Top-of-handle() see_unknown({!r})".format(token)) + self.see_unknown(token) + return + # Flag + if self.context and token in self.context.flags: + debug("Saw flag {!r}".format(token)) + self.switch_to_flag(token) + elif self.context and token in self.context.inverse_flags: + debug("Saw inverse flag {!r}".format(token)) + self.switch_to_flag(token, inverse=True) + # Value for current flag + elif self.waiting_for_flag_value: + debug( + "We're waiting for a flag value so {!r} must be it?".format( + token + ) + ) # noqa + self.see_value(token) + # Positional args (must come above context-name check in case we still + # need a posarg and the user legitimately wants to give it a value that + # just happens to be a valid context name.) + elif self.context and self.context.missing_positional_args: + msg = "Context {!r} requires positional args, eating {!r}" + debug(msg.format(self.context, token)) + self.see_positional_arg(token) + # New context + elif token in self.contexts: + self.see_context(token) + # Initial-context flag being given as per-task flag (e.g. --help) + elif self.initial and token in self.initial.flags: + debug("Saw (initial-context) flag {!r}".format(token)) + flag = self.initial.flags[token] + # Special-case for core --help flag: context name is used as value. + if flag.name == "help": + flag.value = self.context.name + msg = "Saw --help in a per-task context, setting task name ({!r}) as its value" # noqa + debug(msg.format(flag.value)) + # All others: just enter the 'switch to flag' parser state + else: + # TODO: handle inverse core flags too? There are none at the + # moment (e.g. --no-dedupe is actually 'no_dedupe', not a + # default-False 'dedupe') and it's up to us whether we actually + # put any in place. + self.switch_to_flag(token) + # Unknown + else: + if not self.ignore_unknown: + debug("Can't find context named {!r}, erroring".format(token)) + self.error("No idea what {!r} is!".format(token)) + else: + debug("Bottom-of-handle() see_unknown({!r})".format(token)) + self.see_unknown(token) + + def store_only(self, token: str) -> None: + # Start off the unparsed list + debug("Storing unknown token {!r}".format(token)) + self.result.unparsed.append(token) + + def complete_context(self) -> None: + debug( + "Wrapping up context {!r}".format( + self.context.name if self.context else self.context + ) + ) + # Ensure all of context's positional args have been given. + if self.context and self.context.missing_positional_args: + err = "'{}' did not receive required positional arguments: {}" + names = ", ".join( + "'{}'".format(x.name) + for x in self.context.missing_positional_args + ) + self.error(err.format(self.context.name, names)) + if self.context and self.context not in self.result: + self.result.append(self.context) + + def switch_to_context(self, name: str) -> None: + self.context = copy.deepcopy(self.contexts[name]) + debug("Moving to context {!r}".format(name)) + debug("Context args: {!r}".format(self.context.args)) + debug("Context flags: {!r}".format(self.context.flags)) + debug("Context inverse_flags: {!r}".format(self.context.inverse_flags)) + + def complete_flag(self) -> None: + if self.flag: + msg = "Completing current flag {} before moving on" + debug(msg.format(self.flag)) + # Barf if we needed a value and didn't get one + if ( + self.flag + and self.flag.takes_value + and self.flag.raw_value is None + and not self.flag.optional + ): + err = "Flag {!r} needed value and was not given one!" + self.error(err.format(self.flag)) + # Handle optional-value flags; at this point they were not given an + # explicit value, but they were seen, ergo they should get treated like + # bools. + if self.flag and self.flag.raw_value is None and self.flag.optional: + msg = "Saw optional flag {!r} go by w/ no value; setting to True" + debug(msg.format(self.flag.name)) + # Skip casting so the bool gets preserved + self.flag.set_value(True, cast=False) + + def check_ambiguity(self, value: Any) -> bool: + """ + Guard against ambiguity when current flag takes an optional value. + + .. versionadded:: 1.0 + """ + # No flag is currently being examined, or one is but it doesn't take an + # optional value? Ambiguity isn't possible. + if not (self.flag and self.flag.optional): + return False + # We *are* dealing with an optional-value flag, but it's already + # received a value? There can't be ambiguity here either. + if self.flag.raw_value is not None: + return False + # Otherwise, there *may* be ambiguity if 1 or more of the below tests + # fail. + tests = [] + # Unfilled posargs still exist? + tests.append(self.context and self.context.missing_positional_args) + # Value matches another valid task/context name? + tests.append(value in self.contexts) + if any(tests): + msg = "{!r} is ambiguous when given after an optional-value flag" + raise ParseError(msg.format(value)) + + def switch_to_flag(self, flag: str, inverse: bool = False) -> None: + # Sanity check for ambiguity w/ prior optional-value flag + self.check_ambiguity(flag) + # Also tie it off, in case prior had optional value or etc. Seems to be + # harmless for other kinds of flags. (TODO: this is a serious indicator + # that we need to move some of this flag-by-flag bookkeeping into the + # state machine bits, if possible - as-is it was REAL confusing re: why + # this was manually required!) + self.complete_flag() + # Set flag/arg obj + flag = self.context.inverse_flags[flag] if inverse else flag + # Update state + try: + self.flag = self.context.flags[flag] + except KeyError as e: + # Try fallback to initial/core flag + try: + self.flag = self.initial.flags[flag] + except KeyError: + # If it wasn't in either, raise the original context's + # exception, as that's more useful / correct. + raise e + debug("Moving to flag {!r}".format(self.flag)) + # Bookkeeping for iterable-type flags (where the typical 'value + # non-empty/nondefault -> clearly it got its value already' test is + # insufficient) + self.flag_got_value = False + # Handle boolean flags (which can immediately be updated) + if self.flag and not self.flag.takes_value: + val = not inverse + debug("Marking seen flag {!r} as {}".format(self.flag, val)) + self.flag.value = val + + def see_value(self, value: Any) -> None: + self.check_ambiguity(value) + if self.flag and self.flag.takes_value: + debug("Setting flag {!r} to value {!r}".format(self.flag, value)) + self.flag.value = value + self.flag_got_value = True + else: + self.error("Flag {!r} doesn't take any value!".format(self.flag)) + + def see_positional_arg(self, value: Any) -> None: + for arg in self.context.positional_args: + if arg.value is None: + arg.value = value + break + + def error(self, msg: str) -> None: + raise ParseError(msg, self.context) diff --git a/lib/invoke/program.py b/lib/invoke/program.py new file mode 100644 index 0000000..c7e5cd0 --- /dev/null +++ b/lib/invoke/program.py @@ -0,0 +1,987 @@ +import getpass +import inspect +import json +import os +import sys +import textwrap +from importlib import import_module # buffalo buffalo +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Sequence, + Tuple, + Type, +) + +from . import Collection, Config, Executor, FilesystemLoader +from .completion.complete import complete, print_completion_script +from .parser import Parser, ParserContext, Argument +from .exceptions import UnexpectedExit, CollectionNotFound, ParseError, Exit +from .terminals import pty_size +from .util import debug, enable_logging, helpline + +if TYPE_CHECKING: + from .loader import Loader + from .parser import ParseResult + from .util import Lexicon + + +class Program: + """ + Manages top-level CLI invocation, typically via ``setup.py`` entrypoints. + + Designed for distributing Invoke task collections as standalone programs, + but also used internally to implement the ``invoke`` program itself. + + .. seealso:: + :ref:`reusing-as-a-binary` for a tutorial/walkthrough of this + functionality. + + .. versionadded:: 1.0 + """ + + core: "ParseResult" + + def core_args(self) -> List["Argument"]: + """ + Return default core `.Argument` objects, as a list. + + .. versionadded:: 1.0 + """ + # Arguments present always, even when wrapped as a different binary + return [ + Argument( + names=("command-timeout", "T"), + kind=int, + help="Specify a global command execution timeout, in seconds.", + ), + Argument( + names=("complete",), + kind=bool, + default=False, + help="Print tab-completion candidates for given parse remainder.", # noqa + ), + Argument( + names=("config", "f"), + help="Runtime configuration file to use.", + ), + Argument( + names=("debug", "d"), + kind=bool, + default=False, + help="Enable debug output.", + ), + Argument( + names=("dry", "R"), + kind=bool, + default=False, + help="Echo commands instead of running.", + ), + Argument( + names=("echo", "e"), + kind=bool, + default=False, + help="Echo executed commands before running.", + ), + Argument( + names=("help", "h"), + optional=True, + help="Show core or per-task help and exit.", + ), + Argument( + names=("hide",), + help="Set default value of run()'s 'hide' kwarg.", + ), + Argument( + names=("list", "l"), + optional=True, + help="List available tasks, optionally limited to a namespace.", # noqa + ), + Argument( + names=("list-depth", "D"), + kind=int, + default=0, + help="When listing tasks, only show the first INT levels.", + ), + Argument( + names=("list-format", "F"), + help="Change the display format used when listing tasks. Should be one of: flat (default), nested, json.", # noqa + default="flat", + ), + Argument( + names=("print-completion-script",), + kind=str, + default="", + help="Print the tab-completion script for your preferred shell (bash|zsh|fish).", # noqa + ), + Argument( + names=("prompt-for-sudo-password",), + kind=bool, + default=False, + help="Prompt user at start of session for the sudo.password config value.", # noqa + ), + Argument( + names=("pty", "p"), + kind=bool, + default=False, + help="Use a pty when executing shell commands.", + ), + Argument( + names=("version", "V"), + kind=bool, + default=False, + help="Show version and exit.", + ), + Argument( + names=("warn-only", "w"), + kind=bool, + default=False, + help="Warn, instead of failing, when shell commands fail.", + ), + Argument( + names=("write-pyc",), + kind=bool, + default=False, + help="Enable creation of .pyc files.", + ), + ] + + def task_args(self) -> List["Argument"]: + """ + Return default task-related `.Argument` objects, as a list. + + These are only added to the core args in "task runner" mode (the + default for ``invoke`` itself) - they are omitted when the constructor + is given a non-empty ``namespace`` argument ("bundled namespace" mode). + + .. versionadded:: 1.0 + """ + # Arguments pertaining specifically to invocation as 'invoke' itself + # (or as other arbitrary-task-executing programs, like 'fab') + return [ + Argument( + names=("collection", "c"), + help="Specify collection name to load.", + ), + Argument( + names=("no-dedupe",), + kind=bool, + default=False, + help="Disable task deduplication.", + ), + Argument( + names=("search-root", "r"), + help="Change root directory used for finding task modules.", + ), + ] + + argv: List[str] + # Other class-level global variables a subclass might override sometime + # maybe? + leading_indent_width = 2 + leading_indent = " " * leading_indent_width + indent_width = 4 + indent = " " * indent_width + col_padding = 3 + + def __init__( + self, + version: Optional[str] = None, + namespace: Optional["Collection"] = None, + name: Optional[str] = None, + binary: Optional[str] = None, + loader_class: Optional[Type["Loader"]] = None, + executor_class: Optional[Type["Executor"]] = None, + config_class: Optional[Type["Config"]] = None, + binary_names: Optional[List[str]] = None, + ) -> None: + """ + Create a new, parameterized `.Program` instance. + + :param str version: + The program's version, e.g. ``"0.1.0"``. Defaults to ``"unknown"``. + + :param namespace: + A `.Collection` to use as this program's subcommands. + + If ``None`` (the default), the program will behave like ``invoke``, + seeking a nearby task namespace with a `.Loader` and exposing + arguments such as :option:`--list` and :option:`--collection` for + inspecting or selecting specific namespaces. + + If given a `.Collection` object, will use it as if it had been + handed to :option:`--collection`. Will also update the parser to + remove references to tasks and task-related options, and display + the subcommands in ``--help`` output. The result will be a program + that has a static set of subcommands. + + :param str name: + The program's name, as displayed in ``--version`` output. + + If ``None`` (default), is a capitalized version of the first word + in the ``argv`` handed to `.run`. For example, when invoked from a + binstub installed as ``foobar``, it will default to ``Foobar``. + + :param str binary: + Descriptive lowercase binary name string used in help text. + + For example, Invoke's own internal value for this is ``inv[oke]``, + denoting that it is installed as both ``inv`` and ``invoke``. As + this is purely text intended for help display, it may be in any + format you wish, though it should match whatever you've put into + your ``setup.py``'s ``console_scripts`` entry. + + If ``None`` (default), uses the first word in ``argv`` verbatim (as + with ``name`` above, except not capitalized). + + :param binary_names: + List of binary name strings, for use in completion scripts. + + This list ensures that the shell completion scripts generated by + :option:`--print-completion-script` instruct the shell to use + that completion for all of this program's installed names. + + For example, Invoke's internal default for this is ``["inv", + "invoke"]``. + + If ``None`` (the default), the first word in ``argv`` (in the + invocation of :option:`--print-completion-script`) is used in a + single-item list. + + :param loader_class: + The `.Loader` subclass to use when loading task collections. + + Defaults to `.FilesystemLoader`. + + :param executor_class: + The `.Executor` subclass to use when executing tasks. + + Defaults to `.Executor`; may also be overridden at runtime by the + :ref:`configuration system ` and its + ``tasks.executor_class`` setting (anytime that setting is not + ``None``). + + :param config_class: + The `.Config` subclass to use for the base config object. + + Defaults to `.Config`. + + .. versionchanged:: 1.2 + Added the ``binary_names`` argument. + """ + self.version = "unknown" if version is None else version + self.namespace = namespace + self._name = name + # TODO 3.0: rename binary to binary_help_name or similar. (Or write + # code to autogenerate it from binary_names.) + self._binary = binary + self._binary_names = binary_names + self.argv = [] + self.loader_class = loader_class or FilesystemLoader + self.executor_class = executor_class or Executor + self.config_class = config_class or Config + + def create_config(self) -> None: + """ + Instantiate a `.Config` (or subclass, depending) for use in task exec. + + This Config is fully usable but will lack runtime-derived data like + project & runtime config files, CLI arg overrides, etc. That data is + added later in `update_config`. See `.Config` docstring for lifecycle + details. + + :returns: ``None``; sets ``self.config`` instead. + + .. versionadded:: 1.0 + """ + self.config = self.config_class() + + def update_config(self, merge: bool = True) -> None: + """ + Update the previously instantiated `.Config` with parsed data. + + For example, this is how ``--echo`` is able to override the default + config value for ``run.echo``. + + :param bool merge: + Whether to merge at the end, or defer. Primarily useful for + subclassers. Default: ``True``. + + .. versionadded:: 1.0 + """ + # Now that we have parse results handy, we can grab the remaining + # config bits: + # - runtime config, as it is dependent on the runtime flag/env var + # - the overrides config level, as it is composed of runtime flag data + # NOTE: only fill in values that would alter behavior, otherwise we + # want the defaults to come through. + run = {} + if self.args["warn-only"].value: + run["warn"] = True + if self.args.pty.value: + run["pty"] = True + if self.args.hide.value: + run["hide"] = self.args.hide.value + if self.args.echo.value: + run["echo"] = True + if self.args.dry.value: + run["dry"] = True + tasks = {} + if "no-dedupe" in self.args and self.args["no-dedupe"].value: + tasks["dedupe"] = False + timeouts = {} + command = self.args["command-timeout"].value + if command: + timeouts["command"] = command + # Handle "fill in config values at start of runtime", which for now is + # just sudo password + sudo = {} + if self.args["prompt-for-sudo-password"].value: + prompt = "Desired 'sudo.password' config value: " + sudo["password"] = getpass.getpass(prompt) + overrides = dict(run=run, tasks=tasks, sudo=sudo, timeouts=timeouts) + self.config.load_overrides(overrides, merge=False) + runtime_path = self.args.config.value + if runtime_path is None: + runtime_path = os.environ.get("INVOKE_RUNTIME_CONFIG", None) + self.config.set_runtime_path(runtime_path) + self.config.load_runtime(merge=False) + if merge: + self.config.merge() + + def run(self, argv: Optional[List[str]] = None, exit: bool = True) -> None: + """ + Execute main CLI logic, based on ``argv``. + + :param argv: + The arguments to execute against. May be ``None``, a list of + strings, or a string. See `.normalize_argv` for details. + + :param bool exit: + When ``False`` (default: ``True``), will ignore `.ParseError`, + `.Exit` and `.Failure` exceptions, which otherwise trigger calls to + `sys.exit`. + + .. note:: + This is mostly a concession to testing. If you're setting this + to ``False`` in a production setting, you should probably be + using `.Executor` and friends directly instead! + + .. versionadded:: 1.0 + """ + try: + # Create an initial config, which will hold defaults & values from + # most config file locations (all but runtime.) Used to inform + # loading & parsing behavior. + self.create_config() + # Parse the given ARGV with our CLI parsing machinery, resulting in + # things like self.args (core args/flags), self.collection (the + # loaded namespace, which may be affected by the core flags) and + # self.tasks (the tasks requested for exec and their own + # args/flags) + self.parse_core(argv) + # Handle collection concerns including project config + self.parse_collection() + # Parse remainder of argv as task-related input + self.parse_tasks() + # End of parsing (typically bailout stuff like --list, --help) + self.parse_cleanup() + # Update the earlier Config with new values from the parse step - + # runtime config file contents and flag-derived overrides (e.g. for + # run()'s echo, warn, etc options.) + self.update_config() + # Create an Executor, passing in the data resulting from the prior + # steps, then tell it to execute the tasks. + self.execute() + except (UnexpectedExit, Exit, ParseError) as e: + debug("Received a possibly-skippable exception: {!r}".format(e)) + # Print error messages from parser, runner, etc if necessary; + # prevents messy traceback but still clues interactive user into + # problems. + if isinstance(e, ParseError): + print(e, file=sys.stderr) + if isinstance(e, Exit) and e.message: + print(e.message, file=sys.stderr) + if isinstance(e, UnexpectedExit) and e.result.hide: + print(e, file=sys.stderr, end="") + # Terminate execution unless we were told not to. + if exit: + if isinstance(e, UnexpectedExit): + code = e.result.exited + elif isinstance(e, Exit): + code = e.code + elif isinstance(e, ParseError): + code = 1 + sys.exit(code) + else: + debug("Invoked as run(..., exit=False), ignoring exception") + except KeyboardInterrupt: + sys.exit(1) # Same behavior as Python itself outside of REPL + + def parse_core(self, argv: Optional[List[str]]) -> None: + debug("argv given to Program.run: {!r}".format(argv)) + self.normalize_argv(argv) + + # Obtain core args (sets self.core) + self.parse_core_args() + debug("Finished parsing core args") + + # Set interpreter bytecode-writing flag + sys.dont_write_bytecode = not self.args["write-pyc"].value + + # Enable debugging from here on out, if debug flag was given. + # (Prior to this point, debugging requires setting INVOKE_DEBUG). + if self.args.debug.value: + enable_logging() + + # Short-circuit if --version + if self.args.version.value: + debug("Saw --version, printing version & exiting") + self.print_version() + raise Exit + + # Print (dynamic, no tasks required) completion script if requested + if self.args["print-completion-script"].value: + print_completion_script( + shell=self.args["print-completion-script"].value, + names=self.binary_names, + ) + raise Exit + + def parse_collection(self) -> None: + """ + Load a tasks collection & project-level config. + + .. versionadded:: 1.0 + """ + # Load a collection of tasks unless one was already set. + if self.namespace is not None: + debug( + "Program was given default namespace, not loading collection" + ) + self.collection = self.namespace + else: + debug( + "No default namespace provided, trying to load one from disk" + ) # noqa + # If no bundled namespace & --help was given, just print it and + # exit. (If we did have a bundled namespace, core --help will be + # handled *after* the collection is loaded & parsing is done.) + if self.args.help.value is True: + debug( + "No bundled namespace & bare --help given; printing help." + ) + self.print_help() + raise Exit + self.load_collection() + # Set these up for potential use later when listing tasks + # TODO: be nice if these came from the config...! Users would love to + # say they default to nested for example. Easy 2.x feature-add. + self.list_root: Optional[str] = None + self.list_depth: Optional[int] = None + self.list_format = "flat" + self.scoped_collection = self.collection + + # TODO: load project conf, if possible, gracefully + + def parse_cleanup(self) -> None: + """ + Post-parsing, pre-execution steps such as --help, --list, etc. + + .. versionadded:: 1.0 + """ + halp = self.args.help.value + + # Core (no value given) --help output (only when bundled namespace) + if halp is True: + debug("Saw bare --help, printing help & exiting") + self.print_help() + raise Exit + + # Print per-task help, if necessary + if halp: + if halp in self.parser.contexts: + msg = "Saw --help , printing per-task help & exiting" + debug(msg) + self.print_task_help(halp) + raise Exit + else: + # TODO: feels real dumb to factor this out of Parser, but...we + # should? + raise ParseError("No idea what '{}' is!".format(halp)) + + # Print discovered tasks if necessary + list_root = self.args.list.value # will be True or string + self.list_format = self.args["list-format"].value + self.list_depth = self.args["list-depth"].value + if list_root: + # Not just --list, but --list some-root - do moar work + if isinstance(list_root, str): + self.list_root = list_root + try: + sub = self.collection.subcollection_from_path(list_root) + self.scoped_collection = sub + except KeyError: + msg = "Sub-collection '{}' not found!" + raise Exit(msg.format(list_root)) + self.list_tasks() + raise Exit + + # Print completion helpers if necessary + if self.args.complete.value: + complete( + names=self.binary_names, + core=self.core, + initial_context=self.initial_context, + collection=self.collection, + # NOTE: can't reuse self.parser as it has likely been mutated + # between when it was set and now. + parser=self._make_parser(), + ) + + # Fallback behavior if no tasks were given & no default specified + # (mostly a subroutine for overriding purposes) + # NOTE: when there is a default task, Executor will select it when no + # tasks were found in CLI parsing. + if not self.tasks and not self.collection.default: + self.no_tasks_given() + + def no_tasks_given(self) -> None: + debug( + "No tasks specified for execution and no default task; printing global help as fallback" # noqa + ) + self.print_help() + raise Exit + + def execute(self) -> None: + """ + Hand off data & tasks-to-execute specification to an `.Executor`. + + .. note:: + Client code just wanting a different `.Executor` subclass can just + set ``executor_class`` in `.__init__`, or override + ``tasks.executor_class`` anywhere in the :ref:`config system + ` (which may allow you to avoid using a custom + Program entirely). + + .. versionadded:: 1.0 + """ + klass = self.executor_class + config_path = self.config.tasks.executor_class + if config_path is not None: + # TODO: why the heck is this not builtin to importlib? + module_path, _, class_name = config_path.rpartition(".") + # TODO: worth trying to wrap both of these and raising ImportError + # for cases where module exists but class name does not? More + # "normal" but also its own possible source of bugs/confusion... + module = import_module(module_path) + klass = getattr(module, class_name) + executor = klass(self.collection, self.config, self.core) + executor.execute(*self.tasks) + + def normalize_argv(self, argv: Optional[List[str]]) -> None: + """ + Massages ``argv`` into a useful list of strings. + + **If None** (the default), uses `sys.argv`. + + **If a non-string iterable**, uses that in place of `sys.argv`. + + **If a string**, performs a `str.split` and then executes with the + result. (This is mostly a convenience; when in doubt, use a list.) + + Sets ``self.argv`` to the result. + + .. versionadded:: 1.0 + """ + if argv is None: + argv = sys.argv + debug("argv was None; using sys.argv: {!r}".format(argv)) + elif isinstance(argv, str): + argv = argv.split() + debug("argv was string-like; splitting: {!r}".format(argv)) + self.argv = argv + + @property + def name(self) -> str: + """ + Derive program's human-readable name based on `.binary`. + + .. versionadded:: 1.0 + """ + return self._name or self.binary.capitalize() + + @property + def called_as(self) -> str: + """ + Returns the program name we were actually called as. + + Specifically, this is the (Python's os module's concept of a) basename + of the first argument in the parsed argument vector. + + .. versionadded:: 1.2 + """ + # XXX: defaults to empty string if 'argv' is '[]' or 'None' + return os.path.basename(self.argv[0]) if self.argv else "" + + @property + def binary(self) -> str: + """ + Derive program's help-oriented binary name(s) from init args & argv. + + .. versionadded:: 1.0 + """ + return self._binary or self.called_as + + @property + def binary_names(self) -> List[str]: + """ + Derive program's completion-oriented binary name(s) from args & argv. + + .. versionadded:: 1.2 + """ + return self._binary_names or [self.called_as] + + # TODO 3.0: ugh rename this or core_args, they are too confusing + @property + def args(self) -> "Lexicon": + """ + Obtain core program args from ``self.core`` parse result. + + .. versionadded:: 1.0 + """ + return self.core[0].args + + @property + def initial_context(self) -> ParserContext: + """ + The initial parser context, aka core program flags. + + The specific arguments contained therein will differ depending on + whether a bundled namespace was specified in `.__init__`. + + .. versionadded:: 1.0 + """ + args = self.core_args() + if self.namespace is None: + args += self.task_args() + return ParserContext(args=args) + + def print_version(self) -> None: + print("{} {}".format(self.name, self.version or "unknown")) + + def print_help(self) -> None: + usage_suffix = "task1 [--task1-opts] ... taskN [--taskN-opts]" + if self.namespace is not None: + usage_suffix = " [--subcommand-opts] ..." + print("Usage: {} [--core-opts] {}".format(self.binary, usage_suffix)) + print("") + print("Core options:") + print("") + self.print_columns(self.initial_context.help_tuples()) + if self.namespace is not None: + self.list_tasks() + + def parse_core_args(self) -> None: + """ + Filter out core args, leaving any tasks or their args for later. + + Sets ``self.core`` to the `.ParseResult` from this step. + + .. versionadded:: 1.0 + """ + debug("Parsing initial context (core args)") + parser = Parser(initial=self.initial_context, ignore_unknown=True) + self.core = parser.parse_argv(self.argv[1:]) + msg = "Core-args parse result: {!r} & unparsed: {!r}" + debug(msg.format(self.core, self.core.unparsed)) + + def load_collection(self) -> None: + """ + Load a task collection based on parsed core args, or die trying. + + .. versionadded:: 1.0 + """ + # NOTE: start, coll_name both fall back to configuration values within + # Loader (which may, however, get them from our config.) + start = self.args["search-root"].value + loader = self.loader_class( # type: ignore + config=self.config, start=start + ) + coll_name = self.args.collection.value + try: + module, parent = loader.load(coll_name) + # This is the earliest we can load project config, so we should - + # allows project config to affect the task parsing step! + # TODO: is it worth merging these set- and load- methods? May + # require more tweaking of how things behave in/after __init__. + self.config.set_project_location(parent) + self.config.load_project() + self.collection = Collection.from_module( + module, + loaded_from=parent, + auto_dash_names=self.config.tasks.auto_dash_names, + ) + except CollectionNotFound as e: + raise Exit("Can't find any collection named {!r}!".format(e.name)) + + def _update_core_context( + self, context: ParserContext, new_args: Dict[str, Any] + ) -> None: + # Update core context w/ core_via_task args, if and only if the + # via-task version of the arg was truly given a value. + # TODO: push this into an Argument-aware Lexicon subclass and + # .update()? + for key, arg in new_args.items(): + if arg.got_value: + context.args[key]._value = arg._value + + def _make_parser(self) -> Parser: + return Parser( + initial=self.initial_context, + contexts=self.collection.to_contexts( + ignore_unknown_help=self.config.tasks.ignore_unknown_help + ), + ) + + def parse_tasks(self) -> None: + """ + Parse leftover args, which are typically tasks & per-task args. + + Sets ``self.parser`` to the parser used, ``self.tasks`` to the + parsed per-task contexts, and ``self.core_via_tasks`` to a context + holding any core flags seen within the task contexts. + + Also modifies ``self.core`` to include the data from ``core_via_tasks`` + (so that it correctly reflects any supplied core flags regardless of + where they appeared). + + .. versionadded:: 1.0 + """ + self.parser = self._make_parser() + debug("Parsing tasks against {!r}".format(self.collection)) + result = self.parser.parse_argv(self.core.unparsed) + self.core_via_tasks = result.pop(0) + self._update_core_context( + context=self.core[0], new_args=self.core_via_tasks.args + ) + self.tasks = result + debug("Resulting task contexts: {!r}".format(self.tasks)) + + def print_task_help(self, name: str) -> None: + """ + Print help for a specific task, e.g. ``inv --help ``. + + .. versionadded:: 1.0 + """ + # Setup + ctx = self.parser.contexts[name] + tuples = ctx.help_tuples() + docstring = inspect.getdoc(self.collection[name]) + header = "Usage: {} [--core-opts] {} {}[other tasks here ...]" + opts = "[--options] " if tuples else "" + print(header.format(self.binary, name, opts)) + print("") + print("Docstring:") + if docstring: + # Really wish textwrap worked better for this. + for line in docstring.splitlines(): + if line.strip(): + print(self.leading_indent + line) + else: + print("") + print("") + else: + print(self.leading_indent + "none") + print("") + print("Options:") + if tuples: + self.print_columns(tuples) + else: + print(self.leading_indent + "none") + print("") + + def list_tasks(self) -> None: + # Short circuit if no tasks to show (Collection now implements bool) + focus = self.scoped_collection + if not focus: + msg = "No tasks found in collection '{}'!" + raise Exit(msg.format(focus.name)) + # TODO: now that flat/nested are almost 100% unified, maybe rethink + # this a bit? + getattr(self, "list_{}".format(self.list_format))() + + def list_flat(self) -> None: + pairs = self._make_pairs(self.scoped_collection) + self.display_with_columns(pairs=pairs) + + def list_nested(self) -> None: + pairs = self._make_pairs(self.scoped_collection) + extra = "'*' denotes collection defaults" + self.display_with_columns(pairs=pairs, extra=extra) + + def _make_pairs( + self, + coll: "Collection", + ancestors: Optional[List[str]] = None, + ) -> List[Tuple[str, Optional[str]]]: + if ancestors is None: + ancestors = [] + pairs = [] + indent = len(ancestors) * self.indent + ancestor_path = ".".join(x for x in ancestors) + for name, task in sorted(coll.tasks.items()): + is_default = name == coll.default + # Start with just the name and just the aliases, no prefixes or + # dots. + displayname = name + aliases = list(map(coll.transform, sorted(task.aliases))) + # If displaying a sub-collection (or if we are displaying a given + # namespace/root), tack on some dots to make it clear these names + # require dotted paths to invoke. + if ancestors or self.list_root: + displayname = ".{}".format(displayname) + aliases = [".{}".format(x) for x in aliases] + # Nested? Indent, and add asterisks to default-tasks. + if self.list_format == "nested": + prefix = indent + if is_default: + displayname += "*" + # Flat? Prefix names and aliases with ancestor names to get full + # dotted path; and give default-tasks their collection name as the + # first alias. + if self.list_format == "flat": + prefix = ancestor_path + # Make sure leading dots are present for subcollections if + # scoped display + if prefix and self.list_root: + prefix = "." + prefix + aliases = [prefix + alias for alias in aliases] + if is_default and ancestors: + aliases.insert(0, prefix) + # Generate full name and help columns and add to pairs. + alias_str = " ({})".format(", ".join(aliases)) if aliases else "" + full = prefix + displayname + alias_str + pairs.append((full, helpline(task))) + # Determine whether we're at max-depth or not + truncate = self.list_depth and (len(ancestors) + 1) >= self.list_depth + for name, subcoll in sorted(coll.collections.items()): + displayname = name + if ancestors or self.list_root: + displayname = ".{}".format(displayname) + if truncate: + tallies = [ + "{} {}".format(len(getattr(subcoll, attr)), attr) + for attr in ("tasks", "collections") + if getattr(subcoll, attr) + ] + displayname += " [{}]".format(", ".join(tallies)) + if self.list_format == "nested": + pairs.append((indent + displayname, helpline(subcoll))) + elif self.list_format == "flat" and truncate: + # NOTE: only adding coll-oriented pair if limiting by depth + pairs.append((ancestor_path + displayname, helpline(subcoll))) + # Recurse, if not already at max depth + if not truncate: + recursed_pairs = self._make_pairs( + coll=subcoll, ancestors=ancestors + [name] + ) + pairs.extend(recursed_pairs) + return pairs + + def list_json(self) -> None: + # Sanity: we can't cleanly honor the --list-depth argument without + # changing the data schema or otherwise acting strangely; and it also + # doesn't make a ton of sense to limit depth when the output is for a + # script to handle. So we just refuse, for now. TODO: find better way + if self.list_depth: + raise Exit( + "The --list-depth option is not supported with JSON format!" + ) # noqa + # TODO: consider using something more formal re: the format this emits, + # eg json-schema or whatever. Would simplify the + # relatively-concise-but-only-human docs that currently describe this. + coll = self.scoped_collection + data = coll.serialized() + print(json.dumps(data)) + + def task_list_opener(self, extra: str = "") -> str: + root = self.list_root + depth = self.list_depth + specifier = " '{}'".format(root) if root else "" + tail = "" + if depth or extra: + depthstr = "depth={}".format(depth) if depth else "" + joiner = "; " if (depth and extra) else "" + tail = " ({}{}{})".format(depthstr, joiner, extra) + text = "Available{} tasks{}".format(specifier, tail) + # TODO: do use cases w/ bundled namespace want to display things like + # root and depth too? Leaving off for now... + if self.namespace is not None: + text = "Subcommands" + return text + + def display_with_columns( + self, pairs: Sequence[Tuple[str, Optional[str]]], extra: str = "" + ) -> None: + root = self.list_root + print("{}:\n".format(self.task_list_opener(extra=extra))) + self.print_columns(pairs) + # TODO: worth stripping this out for nested? since it's signified with + # asterisk there? ugggh + default = self.scoped_collection.default + if default: + specific = "" + if root: + specific = " '{}'".format(root) + default = ".{}".format(default) + # TODO: trim/prefix dots + print("Default{} task: {}\n".format(specific, default)) + + def print_columns( + self, tuples: Sequence[Tuple[str, Optional[str]]] + ) -> None: + """ + Print tabbed columns from (name, help) ``tuples``. + + Useful for listing tasks + docstrings, flags + help strings, etc. + + .. versionadded:: 1.0 + """ + # Calculate column sizes: don't wrap flag specs, give what's left over + # to the descriptions. + name_width = max(len(x[0]) for x in tuples) + desc_width = ( + pty_size()[0] + - name_width + - self.leading_indent_width + - self.col_padding + - 1 + ) + wrapper = textwrap.TextWrapper(width=desc_width) + for name, help_str in tuples: + if help_str is None: + help_str = "" + # Wrap descriptions/help text + help_chunks = wrapper.wrap(help_str) + # Print flag spec + padding + name_padding = name_width - len(name) + spec = "".join( + ( + self.leading_indent, + name, + name_padding * " ", + self.col_padding * " ", + ) + ) + # Print help text as needed + if help_chunks: + print(spec + help_chunks[0]) + for chunk in help_chunks[1:]: + print((" " * len(spec)) + chunk) + else: + print(spec.rstrip()) + print("") diff --git a/lib/invoke/py.typed b/lib/invoke/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lib/invoke/runners.py b/lib/invoke/runners.py new file mode 100644 index 0000000..f1c888f --- /dev/null +++ b/lib/invoke/runners.py @@ -0,0 +1,1675 @@ +import errno +import locale +import os +import struct +import sys +import threading +import time +import signal +from subprocess import Popen, PIPE +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + IO, + List, + Optional, + Tuple, + Type, +) + +# Import some platform-specific things at top level so they can be mocked for +# tests. +try: + import pty +except ImportError: + pty = None # type: ignore[assignment] +try: + import fcntl +except ImportError: + fcntl = None # type: ignore[assignment] +try: + import termios +except ImportError: + termios = None # type: ignore[assignment] + +from .exceptions import ( + UnexpectedExit, + Failure, + ThreadException, + WatcherError, + SubprocessPipeError, + CommandTimedOut, +) +from .terminals import ( + WINDOWS, + pty_size, + character_buffered, + ready_for_reading, + bytes_to_read, +) +from .util import has_fileno, isatty, ExceptionHandlingThread + +if TYPE_CHECKING: + from .context import Context + from .watchers import StreamWatcher + + +class Runner: + """ + Partially-abstract core command-running API. + + This class is not usable by itself and must be subclassed, implementing a + number of methods such as `start`, `wait` and `returncode`. For a subclass + implementation example, see the source code for `.Local`. + + .. versionadded:: 1.0 + """ + + opts: Dict[str, Any] + using_pty: bool + read_chunk_size = 1000 + input_sleep = 0.01 + + def __init__(self, context: "Context") -> None: + """ + Create a new runner with a handle on some `.Context`. + + :param context: + a `.Context` instance, used to transmit default options and provide + access to other contextualized information (e.g. a remote-oriented + `.Runner` might want a `.Context` subclass holding info about + hostnames and ports.) + + .. note:: + The `.Context` given to `.Runner` instances **must** contain + default config values for the `.Runner` class in question. At a + minimum, this means values for each of the default + `.Runner.run` keyword arguments such as ``echo`` and ``warn``. + + :raises exceptions.ValueError: + if not all expected default values are found in ``context``. + """ + #: The `.Context` given to the same-named argument of `__init__`. + self.context = context + #: A `threading.Event` signaling program completion. + #: + #: Typically set after `wait` returns. Some IO mechanisms rely on this + #: to know when to exit an infinite read loop. + self.program_finished = threading.Event() + # I wish Sphinx would organize all class/instance attrs in the same + # place. If I don't do this here, it goes 'class vars -> __init__ + # docstring -> instance vars' :( TODO: consider just merging class and + # __init__ docstrings, though that's annoying too. + #: How many bytes (at maximum) to read per iteration of stream reads. + self.read_chunk_size = self.__class__.read_chunk_size + # Ditto re: declaring this in 2 places for doc reasons. + #: How many seconds to sleep on each iteration of the stdin read loop + #: and other otherwise-fast loops. + self.input_sleep = self.__class__.input_sleep + #: Whether pty fallback warning has been emitted. + self.warned_about_pty_fallback = False + #: A list of `.StreamWatcher` instances for use by `respond`. Is filled + #: in at runtime by `run`. + self.watchers: List["StreamWatcher"] = [] + # Optional timeout timer placeholder + self._timer: Optional[threading.Timer] = None + # Async flags (initialized for 'finally' referencing in case something + # goes REAL bad during options parsing) + self._asynchronous = False + self._disowned = False + + def run(self, command: str, **kwargs: Any) -> Optional["Result"]: + """ + Execute ``command``, returning an instance of `Result` once complete. + + By default, this method is synchronous (it only returns once the + subprocess has completed), and allows interactive keyboard + communication with the subprocess. + + It can instead behave asynchronously (returning early & requiring + interaction with the resulting object to manage subprocess lifecycle) + if you specify ``asynchronous=True``. Furthermore, you can completely + disassociate the subprocess from Invoke's control (allowing it to + persist on its own after Python exits) by saying ``disown=True``. See + the per-kwarg docs below for details on both of these. + + .. note:: + All kwargs will default to the values found in this instance's + `~.Runner.context` attribute, specifically in its configuration's + ``run`` subtree (e.g. ``run.echo`` provides the default value for + the ``echo`` keyword, etc). The base default values are described + in the parameter list below. + + :param str command: The shell command to execute. + + :param bool asynchronous: + When set to ``True`` (default ``False``), enables asynchronous + behavior, as follows: + + - Connections to the controlling terminal are disabled, meaning you + will not see the subprocess output and it will not respond to + your keyboard input - similar to ``hide=True`` and + ``in_stream=False`` (though explicitly given + ``(out|err|in)_stream`` file-like objects will still be honored + as normal). + - `.run` returns immediately after starting the subprocess, and its + return value becomes an instance of `Promise` instead of + `Result`. + - `Promise` objects are primarily useful for their `~Promise.join` + method, which blocks until the subprocess exits (similar to + threading APIs) and either returns a final `~Result` or raises an + exception, just as a synchronous ``run`` would. + + - As with threading and similar APIs, users of + ``asynchronous=True`` should make sure to ``join`` their + `Promise` objects to prevent issues with interpreter + shutdown. + - One easy way to handle such cleanup is to use the `Promise` + as a context manager - it will automatically ``join`` at the + exit of the context block. + + .. versionadded:: 1.4 + + :param bool disown: + When set to ``True`` (default ``False``), returns immediately like + ``asynchronous=True``, but does not perform any background work + related to that subprocess (it is completely ignored). This allows + subprocesses using shell backgrounding or similar techniques (e.g. + trailing ``&``, ``nohup``) to persist beyond the lifetime of the + Python process running Invoke. + + .. note:: + If you're unsure whether you want this or ``asynchronous``, you + probably want ``asynchronous``! + + Specifically, ``disown=True`` has the following behaviors: + + - The return value is ``None`` instead of a `Result` or subclass. + - No I/O worker threads are spun up, so you will have no access to + the subprocess' stdout/stderr, your stdin will not be forwarded, + ``(out|err|in)_stream`` will be ignored, and features like + ``watchers`` will not function. + - No exit code is checked for, so you will not receive any errors + if the subprocess fails to exit cleanly. + - ``pty=True`` may not function correctly (subprocesses may not run + at all; this seems to be a potential bug in Python's + ``pty.fork``) unless your command line includes tools such as + ``nohup`` or (the shell builtin) ``disown``. + + .. versionadded:: 1.4 + + :param bool dry: + Whether to dry-run instead of truly invoking the given command. See + :option:`--dry` (which flips this on globally) for details on this + behavior. + + .. versionadded:: 1.3 + + :param bool echo: + Controls whether `.run` prints the command string to local stdout + prior to executing it. Default: ``False``. + + .. note:: + ``hide=True`` will override ``echo=True`` if both are given. + + :param echo_format: + A string, which when passed to Python's inbuilt ``.format`` method, + will change the format of the output when ``run.echo`` is set to + true. + + Currently, only ``{command}`` is supported as a parameter. + + Defaults to printing the full command string in ANSI-escaped bold. + + :param bool echo_stdin: + Whether to write data from ``in_stream`` back to ``out_stream``. + + In other words, in normal interactive usage, this parameter + controls whether Invoke mirrors what you type back to your + terminal. + + By default (when ``None``), this behavior is triggered by the + following: + + * Not using a pty to run the subcommand (i.e. ``pty=False``), + as ptys natively echo stdin to stdout on their own; + * And when the controlling terminal of Invoke itself (as per + ``in_stream``) appears to be a valid terminal device or TTY. + (Specifically, when `~invoke.util.isatty` yields a ``True`` + result when given ``in_stream``.) + + .. note:: + This property tends to be ``False`` when piping another + program's output into an Invoke session, or when running + Invoke within another program (e.g. running Invoke from + itself). + + If both of those properties are true, echoing will occur; if either + is false, no echoing will be performed. + + When not ``None``, this parameter will override that auto-detection + and force, or disable, echoing. + + :param str encoding: + Override auto-detection of which encoding the subprocess is using + for its stdout/stderr streams (which defaults to the return value + of `default_encoding`). + + :param err_stream: + Same as ``out_stream``, except for standard error, and defaulting + to ``sys.stderr``. + + :param dict env: + By default, subprocesses receive a copy of Invoke's own environment + (i.e. ``os.environ``). Supply a dict here to update that child + environment. + + For example, ``run('command', env={'PYTHONPATH': + '/some/virtual/env/maybe'})`` would modify the ``PYTHONPATH`` env + var, with the rest of the child's env looking identical to the + parent. + + .. seealso:: ``replace_env`` for changing 'update' to 'replace'. + + :param bool fallback: + Controls auto-fallback behavior re: problems offering a pty when + ``pty=True``. Whether this has any effect depends on the specific + `Runner` subclass being invoked. Default: ``True``. + + :param hide: + Allows the caller to disable ``run``'s default behavior of copying + the subprocess' stdout and stderr to the controlling terminal. + Specify ``hide='out'`` (or ``'stdout'``) to hide only the stdout + stream, ``hide='err'`` (or ``'stderr'``) to hide only stderr, or + ``hide='both'`` (or ``True``) to hide both streams. + + The default value is ``None``, meaning to print everything; + ``False`` will also disable hiding. + + .. note:: + Stdout and stderr are always captured and stored in the + ``Result`` object, regardless of ``hide``'s value. + + .. note:: + ``hide=True`` will also override ``echo=True`` if both are + given (either as kwargs or via config/CLI). + + :param in_stream: + A file-like stream object to used as the subprocess' standard + input. If ``None`` (the default), ``sys.stdin`` will be used. + + If ``False``, will disable stdin mirroring entirely (though other + functionality which writes to the subprocess' stdin, such as + autoresponding, will still function.) Disabling stdin mirroring can + help when ``sys.stdin`` is a misbehaving non-stream object, such as + under test harnesses or headless command runners. + + :param out_stream: + A file-like stream object to which the subprocess' standard output + should be written. If ``None`` (the default), ``sys.stdout`` will + be used. + + :param bool pty: + By default, ``run`` connects directly to the invoked process and + reads its stdout/stderr streams. Some programs will buffer (or even + behave) differently in this situation compared to using an actual + terminal or pseudoterminal (pty). To use a pty instead of the + default behavior, specify ``pty=True``. + + .. warning:: + Due to their nature, ptys have a single output stream, so the + ability to tell stdout apart from stderr is **not possible** + when ``pty=True``. As such, all output will appear on + ``out_stream`` (see below) and be captured into the ``stdout`` + result attribute. ``err_stream`` and ``stderr`` will always be + empty when ``pty=True``. + + :param bool replace_env: + When ``True``, causes the subprocess to receive the dictionary + given to ``env`` as its entire shell environment, instead of + updating a copy of ``os.environ`` (which is the default behavior). + Default: ``False``. + + :param str shell: + Which shell binary to use. Default: ``/bin/bash`` (on Unix; + ``COMSPEC`` or ``cmd.exe`` on Windows.) + + :param timeout: + Cause the runner to submit an interrupt to the subprocess and raise + `.CommandTimedOut`, if the command takes longer than ``timeout`` + seconds to execute. Defaults to ``None``, meaning no timeout. + + .. versionadded:: 1.3 + + :param bool warn: + Whether to warn and continue, instead of raising + `.UnexpectedExit`, when the executed command exits with a + nonzero status. Default: ``False``. + + .. note:: + This setting has no effect on exceptions, which will still be + raised, typically bundled in `.ThreadException` objects if they + were raised by the IO worker threads. + + Similarly, `.WatcherError` exceptions raised by + `.StreamWatcher` instances will also ignore this setting, and + will usually be bundled inside `.Failure` objects (in order to + preserve the execution context). + + Ditto `.CommandTimedOut` - basically, anything that prevents a + command from actually getting to "exited with an exit code" + ignores this flag. + + :param watchers: + A list of `.StreamWatcher` instances which will be used to scan the + program's ``stdout`` or ``stderr`` and may write into its ``stdin`` + (typically ``bytes`` objects) in response to patterns or other + heuristics. + + See :doc:`/concepts/watchers` for details on this functionality. + + Default: ``[]``. + + :returns: + `Result`, or a subclass thereof. + + :raises: + `.UnexpectedExit`, if the command exited nonzero and + ``warn`` was ``False``. + + :raises: + `.Failure`, if the command didn't even exit cleanly, e.g. if a + `.StreamWatcher` raised `.WatcherError`. + + :raises: + `.ThreadException` (if the background I/O threads encountered + exceptions other than `.WatcherError`). + + .. versionadded:: 1.0 + """ + try: + return self._run_body(command, **kwargs) + finally: + if not (self._asynchronous or self._disowned): + self.stop() + + def echo(self, command: str) -> None: + print(self.opts["echo_format"].format(command=command)) + + def _setup(self, command: str, kwargs: Any) -> None: + """ + Prepare data on ``self`` so we're ready to start running. + """ + # Normalize kwargs w/ config; sets self.opts, self.streams + self._unify_kwargs_with_config(kwargs) + # Environment setup + self.env = self.generate_env( + self.opts["env"], self.opts["replace_env"] + ) + # Arrive at final encoding if neither config nor kwargs had one + self.encoding = self.opts["encoding"] or self.default_encoding() + # Echo running command (wants to be early to be included in dry-run) + if self.opts["echo"]: + self.echo(command) + # Prepare common result args. + # TODO: I hate this. Needs a deeper separate think about tweaking + # Runner.generate_result in a way that isn't literally just this same + # two-step process, and which also works w/ downstream. + self.result_kwargs = dict( + command=command, + shell=self.opts["shell"], + env=self.env, + pty=self.using_pty, + hide=self.opts["hide"], + encoding=self.encoding, + ) + + def _run_body(self, command: str, **kwargs: Any) -> Optional["Result"]: + # Prepare all the bits n bobs. + self._setup(command, kwargs) + # If dry-run, stop here. + if self.opts["dry"]: + return self.generate_result( + **dict(self.result_kwargs, stdout="", stderr="", exited=0) + ) + # Start executing the actual command (runs in background) + self.start(command, self.opts["shell"], self.env) + # If disowned, we just stop here - no threads, no timer, no error + # checking, nada. + if self._disowned: + return None + # Stand up & kick off IO, timer threads + self.start_timer(self.opts["timeout"]) + self.threads, self.stdout, self.stderr = self.create_io_threads() + for thread in self.threads.values(): + thread.start() + # Wrap up or promise that we will, depending + return self.make_promise() if self._asynchronous else self._finish() + + def make_promise(self) -> "Promise": + """ + Return a `Promise` allowing async control of the rest of lifecycle. + + .. versionadded:: 1.4 + """ + return Promise(self) + + def _finish(self) -> "Result": + # Wait for subprocess to run, forwarding signals as we get them. + try: + while True: + try: + self.wait() + break # done waiting! + # Don't locally stop on ^C, only forward it: + # - if remote end really stops, we'll naturally stop after + # - if remote end does not stop (eg REPL, editor) we don't want + # to stop prematurely + except KeyboardInterrupt as e: + self.send_interrupt(e) + # TODO: honor other signals sent to our own process and + # transmit them to the subprocess before handling 'normally'. + # Make sure we tie off our worker threads, even if something exploded. + # Any exceptions that raised during self.wait() above will appear after + # this block. + finally: + # Inform stdin-mirroring worker to stop its eternal looping + self.program_finished.set() + # Join threads, storing inner exceptions, & set a timeout if + # necessary. (Segregate WatcherErrors as they are "anticipated + # errors" that want to show up at the end during creation of + # Failure objects.) + watcher_errors = [] + thread_exceptions = [] + for target, thread in self.threads.items(): + thread.join(self._thread_join_timeout(target)) + exception = thread.exception() + if exception is not None: + real = exception.value + if isinstance(real, WatcherError): + watcher_errors.append(real) + else: + thread_exceptions.append(exception) + # If any exceptions appeared inside the threads, raise them now as an + # aggregate exception object. + # NOTE: this is kept outside the 'finally' so that main-thread + # exceptions are raised before worker-thread exceptions; they're more + # likely to be Big Serious Problems. + if thread_exceptions: + raise ThreadException(thread_exceptions) + # Collate stdout/err, calculate exited, and get final result obj + result = self._collate_result(watcher_errors) + # Any presence of WatcherError from the threads indicates a watcher was + # upset and aborted execution; make a generic Failure out of it and + # raise that. + if watcher_errors: + # TODO: ambiguity exists if we somehow get WatcherError in *both* + # threads...as unlikely as that would normally be. + raise Failure(result, reason=watcher_errors[0]) + # If a timeout was requested and the subprocess did time out, shout. + timeout = self.opts["timeout"] + if timeout is not None and self.timed_out: + raise CommandTimedOut(result, timeout=timeout) + if not (result or self.opts["warn"]): + raise UnexpectedExit(result) + return result + + def _unify_kwargs_with_config(self, kwargs: Any) -> None: + """ + Unify `run` kwargs with config options to arrive at local options. + + Sets: + + - ``self.opts`` - opts dict + - ``self.streams`` - map of stream names to stream target values + """ + opts = {} + for key, value in self.context.config.run.items(): + runtime = kwargs.pop(key, None) + opts[key] = value if runtime is None else runtime + # Pull in command execution timeout, which stores config elsewhere, + # but only use it if it's actually set (backwards compat) + config_timeout = self.context.config.timeouts.command + opts["timeout"] = kwargs.pop("timeout", config_timeout) + # Handle invalid kwarg keys (anything left in kwargs). + # Act like a normal function would, i.e. TypeError + if kwargs: + err = "run() got an unexpected keyword argument '{}'" + raise TypeError(err.format(list(kwargs.keys())[0])) + # Update disowned, async flags + self._asynchronous = opts["asynchronous"] + self._disowned = opts["disown"] + if self._asynchronous and self._disowned: + err = "Cannot give both 'asynchronous' and 'disown' at the same time!" # noqa + raise ValueError(err) + # If hide was True, turn off echoing + if opts["hide"] is True: + opts["echo"] = False + # Conversely, ensure echoing is always on when dry-running + if opts["dry"] is True: + opts["echo"] = True + # Always hide if async + if self._asynchronous: + opts["hide"] = True + # Then normalize 'hide' from one of the various valid input values, + # into a stream-names tuple. Also account for the streams. + out_stream, err_stream = opts["out_stream"], opts["err_stream"] + opts["hide"] = normalize_hide(opts["hide"], out_stream, err_stream) + # Derive stream objects + if out_stream is None: + out_stream = sys.stdout + if err_stream is None: + err_stream = sys.stderr + in_stream = opts["in_stream"] + if in_stream is None: + # If in_stream hasn't been overridden, and we're async, we don't + # want to read from sys.stdin (otherwise the default) - so set + # False instead. + in_stream = False if self._asynchronous else sys.stdin + # Determine pty or no + self.using_pty = self.should_use_pty(opts["pty"], opts["fallback"]) + if opts["watchers"]: + self.watchers = opts["watchers"] + # Set data + self.opts = opts + self.streams = {"out": out_stream, "err": err_stream, "in": in_stream} + + def _collate_result(self, watcher_errors: List[WatcherError]) -> "Result": + # At this point, we had enough success that we want to be returning or + # raising detailed info about our execution; so we generate a Result. + stdout = "".join(self.stdout) + stderr = "".join(self.stderr) + if WINDOWS: + # "Universal newlines" - replace all standard forms of + # newline with \n. This is not technically Windows related + # (\r as newline is an old Mac convention) but we only apply + # the translation for Windows as that's the only platform + # it is likely to matter for these days. + stdout = stdout.replace("\r\n", "\n").replace("\r", "\n") + stderr = stderr.replace("\r\n", "\n").replace("\r", "\n") + # Get return/exit code, unless there were WatcherErrors to handle. + # NOTE: In that case, returncode() may block waiting on the process + # (which may be waiting for user input). Since most WatcherError + # situations lack a useful exit code anyways, skipping this doesn't + # really hurt any. + exited = None if watcher_errors else self.returncode() + # TODO: as noted elsewhere, I kinda hate this. Consider changing + # generate_result()'s API in next major rev so we can tidy up. + result = self.generate_result( + **dict( + self.result_kwargs, stdout=stdout, stderr=stderr, exited=exited + ) + ) + return result + + def _thread_join_timeout(self, target: Callable) -> Optional[int]: + # Add a timeout to out/err thread joins when it looks like they're not + # dead but their counterpart is dead; this indicates issue #351 (fixed + # by #432) where the subproc may hang because its stdout (or stderr) is + # no longer being consumed by the dead thread (and a pipe is filling + # up.) In that case, the non-dead thread is likely to block forever on + # a `recv` unless we add this timeout. + if target == self.handle_stdin: + return None + opposite = self.handle_stderr + if target == self.handle_stderr: + opposite = self.handle_stdout + if opposite in self.threads and self.threads[opposite].is_dead: + return 1 + return None + + def create_io_threads( + self, + ) -> Tuple[Dict[Callable, ExceptionHandlingThread], List[str], List[str]]: + """ + Create and return a dictionary of IO thread worker objects. + + Caller is expected to handle persisting and/or starting the wrapped + threads. + """ + stdout: List[str] = [] + stderr: List[str] = [] + # Set up IO thread parameters (format - body_func: {kwargs}) + thread_args: Dict[Callable, Any] = { + self.handle_stdout: { + "buffer_": stdout, + "hide": "stdout" in self.opts["hide"], + "output": self.streams["out"], + } + } + # After opt processing above, in_stream will be a real stream obj or + # False, so we can truth-test it. We don't even create a stdin-handling + # thread if it's False, meaning user indicated stdin is nonexistent or + # problematic. + if self.streams["in"]: + thread_args[self.handle_stdin] = { + "input_": self.streams["in"], + "output": self.streams["out"], + "echo": self.opts["echo_stdin"], + } + if not self.using_pty: + thread_args[self.handle_stderr] = { + "buffer_": stderr, + "hide": "stderr" in self.opts["hide"], + "output": self.streams["err"], + } + # Kick off IO threads + threads = {} + for target, kwargs in thread_args.items(): + t = ExceptionHandlingThread(target=target, kwargs=kwargs) + threads[target] = t + return threads, stdout, stderr + + def generate_result(self, **kwargs: Any) -> "Result": + """ + Create & return a suitable `Result` instance from the given ``kwargs``. + + Subclasses may wish to override this in order to manipulate things or + generate a `Result` subclass (e.g. ones containing additional metadata + besides the default). + + .. versionadded:: 1.0 + """ + return Result(**kwargs) + + def read_proc_output(self, reader: Callable) -> Generator[str, None, None]: + """ + Iteratively read & decode bytes from a subprocess' out/err stream. + + :param reader: + A literal reader function/partial, wrapping the actual stream + object in question, which takes a number of bytes to read, and + returns that many bytes (or ``None``). + + ``reader`` should be a reference to either `read_proc_stdout` or + `read_proc_stderr`, which perform the actual, platform/library + specific read calls. + + :returns: + A generator yielding strings. + + Specifically, each resulting string is the result of decoding + `read_chunk_size` bytes read from the subprocess' out/err stream. + + .. versionadded:: 1.0 + """ + # NOTE: Typically, reading from any stdout/err (local, remote or + # otherwise) can be thought of as "read until you get nothing back". + # This is preferable over "wait until an out-of-band signal claims the + # process is done running" because sometimes that signal will appear + # before we've actually read all the data in the stream (i.e.: a race + # condition). + while True: + data = reader(self.read_chunk_size) + if not data: + break + yield self.decode(data) + + def write_our_output(self, stream: IO, string: str) -> None: + """ + Write ``string`` to ``stream``. + + Also calls ``.flush()`` on ``stream`` to ensure that real terminal + streams don't buffer. + + :param stream: + A file-like stream object, mapping to the ``out_stream`` or + ``err_stream`` parameters of `run`. + + :param string: A Unicode string object. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + stream.write(string) + stream.flush() + + def _handle_output( + self, + buffer_: List[str], + hide: bool, + output: IO, + reader: Callable, + ) -> None: + # TODO: store un-decoded/raw bytes somewhere as well... + for data in self.read_proc_output(reader): + # Echo to local stdout if necessary + # TODO: should we rephrase this as "if you want to hide, give me a + # dummy output stream, e.g. something like /dev/null"? Otherwise, a + # combo of 'hide=stdout' + 'here is an explicit out_stream' means + # out_stream is never written to, and that seems...odd. + if not hide: + self.write_our_output(stream=output, string=data) + # Store in shared buffer so main thread can do things with the + # result after execution completes. + # NOTE: this is threadsafe insofar as no reading occurs until after + # the thread is join()'d. + buffer_.append(data) + # Run our specific buffer through the autoresponder framework + self.respond(buffer_) + + def handle_stdout( + self, buffer_: List[str], hide: bool, output: IO + ) -> None: + """ + Read process' stdout, storing into a buffer & printing/parsing. + + Intended for use as a thread target. Only terminates when all stdout + from the subprocess has been read. + + :param buffer_: The capture buffer shared with the main thread. + :param bool hide: Whether or not to replay data into ``output``. + :param output: + Output stream (file-like object) to write data into when not + hiding. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self._handle_output( + buffer_, hide, output, reader=self.read_proc_stdout + ) + + def handle_stderr( + self, buffer_: List[str], hide: bool, output: IO + ) -> None: + """ + Read process' stderr, storing into a buffer & printing/parsing. + + Identical to `handle_stdout` except for the stream read from; see its + docstring for API details. + + .. versionadded:: 1.0 + """ + self._handle_output( + buffer_, hide, output, reader=self.read_proc_stderr + ) + + def read_our_stdin(self, input_: IO) -> Optional[str]: + """ + Read & decode bytes from a local stdin stream. + + :param input_: + Actual stream object to read from. Maps to ``in_stream`` in `run`, + so will often be ``sys.stdin``, but might be any stream-like + object. + + :returns: + A Unicode string, the result of decoding the read bytes (this might + be the empty string if the pipe has closed/reached EOF); or + ``None`` if stdin wasn't ready for reading yet. + + .. versionadded:: 1.0 + """ + # TODO: consider moving the character_buffered contextmanager call in + # here? Downside is it would be flipping those switches for every byte + # read instead of once per session, which could be costly (?). + bytes_ = None + if ready_for_reading(input_): + try: + bytes_ = input_.read(bytes_to_read(input_)) + except OSError as e: + # Assume EBADF in this situation implies running under nohup or + # similar, where: + # - we cannot reliably detect a bad FD up front + # - trying to read it would explode + # - user almost surely doesn't care about stdin anyways + # and ignore it (but not other OSErrors!) + if e.errno != errno.EBADF: + raise + # Decode if it appears to be binary-type. (From real terminal + # streams, usually yes; from file-like objects, often no.) + if bytes_ and isinstance(bytes_, bytes): + # TODO: will decoding 1 byte at a time break multibyte + # character encodings? How to square interactivity with that? + bytes_ = self.decode(bytes_) + return bytes_ + + def handle_stdin( + self, + input_: IO, + output: IO, + echo: bool = False, + ) -> None: + """ + Read local stdin, copying into process' stdin as necessary. + + Intended for use as a thread target. + + .. note:: + Because real terminal stdin streams have no well-defined "end", if + such a stream is detected (based on existence of a callable + ``.fileno()``) this method will wait until `program_finished` is + set, before terminating. + + When the stream doesn't appear to be from a terminal, the same + semantics as `handle_stdout` are used - the stream is simply + ``read()`` from until it returns an empty value. + + :param input_: Stream (file-like object) from which to read. + :param output: Stream (file-like object) to which echoing may occur. + :param bool echo: User override option for stdin-stdout echoing. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + # TODO: reinstate lock/whatever thread logic from fab v1 which prevents + # reading from stdin while other parts of the code are prompting for + # runtime passwords? (search for 'input_enabled') + # TODO: fabric#1339 is strongly related to this, if it's not literally + # exposing some regression in Fabric 1.x itself. + closed_stdin = False + with character_buffered(input_): + while True: + data = self.read_our_stdin(input_) + if data: + # Mirror what we just read to process' stdin. + # We encode to ensure bytes, but skip the decode step since + # there's presumably no need (nobody's interacting with + # this data programmatically). + self.write_proc_stdin(data) + # Also echo it back to local stdout (or whatever + # out_stream is set to) when necessary. + if echo is None: + echo = self.should_echo_stdin(input_, output) + if echo: + self.write_our_output(stream=output, string=data) + # Empty string/char/byte != None. Can't just use 'else' here. + elif data is not None: + # When reading from file-like objects that aren't "real" + # terminal streams, an empty byte signals EOF. + if not self.using_pty and not closed_stdin: + self.close_proc_stdin() + closed_stdin = True + # Dual all-done signals: program being executed is done + # running, *and* we don't seem to be reading anything out of + # stdin. (NOTE: If we only test the former, we may encounter + # race conditions re: unread stdin.) + if self.program_finished.is_set() and not data: + break + # Take a nap so we're not chewing CPU. + time.sleep(self.input_sleep) + + def should_echo_stdin(self, input_: IO, output: IO) -> bool: + """ + Determine whether data read from ``input_`` should echo to ``output``. + + Used by `handle_stdin`; tests attributes of ``input_`` and ``output``. + + :param input_: Input stream (file-like object). + :param output: Output stream (file-like object). + :returns: A ``bool``. + + .. versionadded:: 1.0 + """ + return (not self.using_pty) and isatty(input_) + + def respond(self, buffer_: List[str]) -> None: + """ + Write to the program's stdin in response to patterns in ``buffer_``. + + The patterns and responses are driven by the `.StreamWatcher` instances + from the ``watchers`` kwarg of `run` - see :doc:`/concepts/watchers` + for a conceptual overview. + + :param buffer: + The capture buffer for this thread's particular IO stream. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + # Join buffer contents into a single string; without this, + # StreamWatcher subclasses can't do things like iteratively scan for + # pattern matches. + # NOTE: using string.join should be "efficient enough" for now, re: + # speed and memory use. Should that become false, consider using + # StringIO or cStringIO (tho the latter doesn't do Unicode well?) which + # is apparently even more efficient. + stream = "".join(buffer_) + for watcher in self.watchers: + for response in watcher.submit(stream): + self.write_proc_stdin(response) + + def generate_env( + self, env: Dict[str, Any], replace_env: bool + ) -> Dict[str, Any]: + """ + Return a suitable environment dict based on user input & behavior. + + :param dict env: Dict supplying overrides or full env, depending. + :param bool replace_env: + Whether ``env`` updates, or is used in place of, the value of + `os.environ`. + + :returns: A dictionary of shell environment vars. + + .. versionadded:: 1.0 + """ + return env if replace_env else dict(os.environ, **env) + + def should_use_pty(self, pty: bool, fallback: bool) -> bool: + """ + Should execution attempt to use a pseudo-terminal? + + :param bool pty: + Whether the user explicitly asked for a pty. + :param bool fallback: + Whether falling back to non-pty execution should be allowed, in + situations where ``pty=True`` but a pty could not be allocated. + + .. versionadded:: 1.0 + """ + # NOTE: fallback not used: no falling back implemented by default. + return pty + + @property + def has_dead_threads(self) -> bool: + """ + Detect whether any IO threads appear to have terminated unexpectedly. + + Used during process-completion waiting (in `wait`) to ensure we don't + deadlock our child process if our IO processing threads have + errored/died. + + :returns: + ``True`` if any threads appear to have terminated with an + exception, ``False`` otherwise. + + .. versionadded:: 1.0 + """ + return any(x.is_dead for x in self.threads.values()) + + def wait(self) -> None: + """ + Block until the running command appears to have exited. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + while True: + proc_finished = self.process_is_finished + dead_threads = self.has_dead_threads + if proc_finished or dead_threads: + break + time.sleep(self.input_sleep) + + def write_proc_stdin(self, data: str) -> None: + """ + Write encoded ``data`` to the running process' stdin. + + :param data: A Unicode string. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + # Encode always, then request implementing subclass to perform the + # actual write to subprocess' stdin. + self._write_proc_stdin(data.encode(self.encoding)) + + def decode(self, data: bytes) -> str: + """ + Decode some ``data`` bytes, returning Unicode. + + .. versionadded:: 1.0 + """ + # NOTE: yes, this is a 1-liner. The point is to make it much harder to + # forget to use 'replace' when decoding :) + return data.decode(self.encoding, "replace") + + @property + def process_is_finished(self) -> bool: + """ + Determine whether our subprocess has terminated. + + .. note:: + The implementation of this method should be nonblocking, as it is + used within a query/poll loop. + + :returns: + ``True`` if the subprocess has finished running, ``False`` + otherwise. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def start(self, command: str, shell: str, env: Dict[str, Any]) -> None: + """ + Initiate execution of ``command`` (via ``shell``, with ``env``). + + Typically this means use of a forked subprocess or requesting start of + execution on a remote system. + + In most cases, this method will also set subclass-specific member + variables used in other methods such as `wait` and/or `returncode`. + + :param str command: + Command string to execute. + + :param str shell: + Shell to use when executing ``command``. + + :param dict env: + Environment dict used to prep shell environment. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def start_timer(self, timeout: int) -> None: + """ + Start a timer to `kill` our subprocess after ``timeout`` seconds. + """ + if timeout is not None: + self._timer = threading.Timer(timeout, self.kill) + self._timer.start() + + def read_proc_stdout(self, num_bytes: int) -> Optional[bytes]: + """ + Read ``num_bytes`` from the running process' stdout stream. + + :param int num_bytes: Number of bytes to read at maximum. + + :returns: A string/bytes object. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def read_proc_stderr(self, num_bytes: int) -> Optional[bytes]: + """ + Read ``num_bytes`` from the running process' stderr stream. + + :param int num_bytes: Number of bytes to read at maximum. + + :returns: A string/bytes object. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def _write_proc_stdin(self, data: bytes) -> None: + """ + Write ``data`` to running process' stdin. + + This should never be called directly; it's for subclasses to implement. + See `write_proc_stdin` for the public API call. + + :param data: Already-encoded byte data suitable for writing. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def close_proc_stdin(self) -> None: + """ + Close running process' stdin. + + :returns: ``None``. + + .. versionadded:: 1.3 + """ + raise NotImplementedError + + def default_encoding(self) -> str: + """ + Return a string naming the expected encoding of subprocess streams. + + This return value should be suitable for use by encode/decode methods. + + .. versionadded:: 1.0 + """ + # TODO: probably wants to be 2 methods, one for local and one for + # subprocess. For now, good enough to assume both are the same. + return default_encoding() + + def send_interrupt(self, interrupt: "KeyboardInterrupt") -> None: + """ + Submit an interrupt signal to the running subprocess. + + In almost all implementations, the default behavior is what will be + desired: submit ``\x03`` to the subprocess' stdin pipe. However, we + leave this as a public method in case this default needs to be + augmented or replaced. + + :param interrupt: + The locally-sourced ``KeyboardInterrupt`` causing the method call. + + :returns: ``None``. + + .. versionadded:: 1.0 + """ + self.write_proc_stdin("\x03") + + def returncode(self) -> Optional[int]: + """ + Return the numeric return/exit code resulting from command execution. + + :returns: + `int`, if any reasonable return code could be determined, or + ``None`` in corner cases where that was not possible. + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + def stop(self) -> None: + """ + Perform final cleanup, if necessary. + + This method is called within a ``finally`` clause inside the main `run` + method. Depending on the subclass, it may be a no-op, or it may do + things such as close network connections or open files. + + :returns: ``None`` + + .. versionadded:: 1.0 + """ + if self._timer: + self._timer.cancel() + + def kill(self) -> None: + """ + Forcibly terminate the subprocess. + + Typically only used by the timeout functionality. + + This is often a "best-effort" attempt, e.g. remote subprocesses often + must settle for simply shutting down the local side of the network + connection and hoping the remote end eventually gets the message. + """ + raise NotImplementedError + + @property + def timed_out(self) -> bool: + """ + Returns ``True`` if the subprocess stopped because it timed out. + + .. versionadded:: 1.3 + """ + # Timer expiry implies we did time out. (The timer itself will have + # killed the subprocess, allowing us to even get to this point.) + return bool(self._timer and not self._timer.is_alive()) + + +class Local(Runner): + """ + Execute a command on the local system in a subprocess. + + .. note:: + When Invoke itself is executed without a controlling terminal (e.g. + when ``sys.stdin`` lacks a useful ``fileno``), it's not possible to + present a handle on our PTY to local subprocesses. In such situations, + `Local` will fallback to behaving as if ``pty=False`` (on the theory + that degraded execution is better than none at all) as well as printing + a warning to stderr. + + To disable this behavior, say ``fallback=False``. + + .. versionadded:: 1.0 + """ + + def __init__(self, context: "Context") -> None: + super().__init__(context) + # Bookkeeping var for pty use case + self.status = 0 + + def should_use_pty(self, pty: bool = False, fallback: bool = True) -> bool: + use_pty = False + if pty: + use_pty = True + # TODO: pass in & test in_stream, not sys.stdin + if not has_fileno(sys.stdin) and fallback: + if not self.warned_about_pty_fallback: + err = "WARNING: stdin has no fileno; falling back to non-pty execution!\n" # noqa + sys.stderr.write(err) + self.warned_about_pty_fallback = True + use_pty = False + return use_pty + + def read_proc_stdout(self, num_bytes: int) -> Optional[bytes]: + # Obtain useful read-some-bytes function + if self.using_pty: + # Need to handle spurious OSErrors on some Linux platforms. + try: + data = os.read(self.parent_fd, num_bytes) + except OSError as e: + # Only eat I/O specific OSErrors so we don't hide others + stringified = str(e) + io_errors = ( + # The typical default + "Input/output error", + # Some less common platforms phrase it this way + "I/O error", + ) + if not any(error in stringified for error in io_errors): + raise + # The bad OSErrors happen after all expected output has + # appeared, so we return a falsey value, which triggers the + # "end of output" logic in code using reader functions. + data = None + elif self.process and self.process.stdout: + data = os.read(self.process.stdout.fileno(), num_bytes) + else: + data = None + return data + + def read_proc_stderr(self, num_bytes: int) -> Optional[bytes]: + # NOTE: when using a pty, this will never be called. + # TODO: do we ever get those OSErrors on stderr? Feels like we could? + if self.process and self.process.stderr: + return os.read(self.process.stderr.fileno(), num_bytes) + return None + + def _write_proc_stdin(self, data: bytes) -> None: + # NOTE: parent_fd from os.fork() is a read/write pipe attached to our + # forked process' stdout/stdin, respectively. + if self.using_pty: + fd = self.parent_fd + elif self.process and self.process.stdin: + fd = self.process.stdin.fileno() + else: + raise SubprocessPipeError( + "Unable to write to missing subprocess or stdin!" + ) + # Try to write, ignoring broken pipes if encountered (implies child + # process exited before the process piping stdin to us finished; + # there's nothing we can do about that!) + try: + os.write(fd, data) + except OSError as e: + if "Broken pipe" not in str(e): + raise + + def close_proc_stdin(self) -> None: + if self.using_pty: + # there is no working scenario to tell the process that stdin + # closed when using pty + raise SubprocessPipeError("Cannot close stdin when pty=True") + elif self.process and self.process.stdin: + self.process.stdin.close() + else: + raise SubprocessPipeError( + "Unable to close missing subprocess or stdin!" + ) + + def start(self, command: str, shell: str, env: Dict[str, Any]) -> None: + if self.using_pty: + if pty is None: # Encountered ImportError + err = "You indicated pty=True, but your platform doesn't support the 'pty' module!" # noqa + sys.exit(err) + cols, rows = pty_size() + self.pid, self.parent_fd = pty.fork() + # If we're the child process, load up the actual command in a + # shell, just as subprocess does; this replaces our process - whose + # pipes are all hooked up to the PTY - with the "real" one. + if self.pid == 0: + # TODO: both pty.spawn() and pexpect.spawn() do a lot of + # setup/teardown involving tty.setraw, getrlimit, signal. + # Ostensibly we'll want some of that eventually, but if + # possible write tests - integration-level if necessary - + # before adding it! + # + # Set pty window size based on what our own controlling + # terminal's window size appears to be. + # TODO: make subroutine? + winsize = struct.pack("HHHH", rows, cols, 0, 0) + fcntl.ioctl(sys.stdout.fileno(), termios.TIOCSWINSZ, winsize) + # Use execve for bare-minimum "exec w/ variable # args + env" + # behavior. No need for the 'p' (use PATH to find executable) + # for now. + # NOTE: stdlib subprocess (actually its posix flavor, which is + # written in C) uses either execve or execv, depending. + os.execve(shell, [shell, "-c", command], env) + else: + self.process = Popen( + command, + shell=True, + executable=shell, + env=env, + stdout=PIPE, + stderr=PIPE, + stdin=PIPE, + ) + + def kill(self) -> None: + pid = self.pid if self.using_pty else self.process.pid + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + # In odd situations where our subprocess is already dead, don't + # throw this upwards. + pass + + @property + def process_is_finished(self) -> bool: + if self.using_pty: + # NOTE: + # https://github.com/pexpect/ptyprocess/blob/4058faa05e2940662ab6da1330aa0586c6f9cd9c/ptyprocess/ptyprocess.py#L680-L687 + # implies that Linux "requires" use of the blocking, non-WNOHANG + # version of this call. Our testing doesn't verify this, however, + # so... + # NOTE: It does appear to be totally blocking on Windows, so our + # issue #351 may be totally unsolvable there. Unclear. + pid_val, self.status = os.waitpid(self.pid, os.WNOHANG) + return pid_val != 0 + else: + return self.process.poll() is not None + + def returncode(self) -> Optional[int]: + if self.using_pty: + # No subprocess.returncode available; use WIFEXITED/WIFSIGNALED to + # determine whch of WEXITSTATUS / WTERMSIG to use. + # TODO: is it safe to just say "call all WEXITSTATUS/WTERMSIG and + # return whichever one of them is nondefault"? Probably not? + # NOTE: doing this in an arbitrary order should be safe since only + # one of the WIF* methods ought to ever return True. + code = None + if os.WIFEXITED(self.status): + code = os.WEXITSTATUS(self.status) + elif os.WIFSIGNALED(self.status): + code = os.WTERMSIG(self.status) + # Match subprocess.returncode by turning signals into negative + # 'exit code' integers. + code = -1 * code + return code + # TODO: do we care about WIFSTOPPED? Maybe someday? + else: + return self.process.returncode + + def stop(self) -> None: + super().stop() + # If we opened a PTY for child communications, make sure to close() it, + # otherwise long-running Invoke-using processes exhaust their file + # descriptors eventually. + if self.using_pty: + try: + os.close(self.parent_fd) + except Exception: + # If something weird happened preventing the close, there's + # nothing to be done about it now... + pass + + +class Result: + """ + A container for information about the result of a command execution. + + All params are exposed as attributes of the same name and type. + + :param str stdout: + The subprocess' standard output. + + :param str stderr: + Same as ``stdout`` but containing standard error (unless the process + was invoked via a pty, in which case it will be empty; see + `.Runner.run`.) + + :param str encoding: + The string encoding used by the local shell environment. + + :param str command: + The command which was executed. + + :param str shell: + The shell binary used for execution. + + :param dict env: + The shell environment used for execution. (Default is the empty dict, + ``{}``, not ``None`` as displayed in the signature.) + + :param int exited: + An integer representing the subprocess' exit/return code. + + .. note:: + This may be ``None`` in situations where the subprocess did not run + to completion, such as when auto-responding failed or a timeout was + reached. + + :param bool pty: + A boolean describing whether the subprocess was invoked with a pty or + not; see `.Runner.run`. + + :param tuple hide: + A tuple of stream names (none, one or both of ``('stdout', 'stderr')``) + which were hidden from the user when the generating command executed; + this is a normalized value derived from the ``hide`` parameter of + `.Runner.run`. + + For example, ``run('command', hide='stdout')`` will yield a `Result` + where ``result.hide == ('stdout',)``; ``hide=True`` or ``hide='both'`` + results in ``result.hide == ('stdout', 'stderr')``; and ``hide=False`` + (the default) generates ``result.hide == ()`` (the empty tuple.) + + .. note:: + `Result` objects' truth evaluation is equivalent to their `.ok` + attribute's value. Therefore, quick-and-dirty expressions like the + following are possible:: + + if run("some shell command"): + do_something() + else: + handle_problem() + + However, remember `Zen of Python #2 + `_. + + .. versionadded:: 1.0 + """ + + # TODO: inherit from namedtuple instead? heh (or: use attrs from pypi) + def __init__( + self, + stdout: str = "", + stderr: str = "", + encoding: Optional[str] = None, + command: str = "", + shell: str = "", + env: Optional[Dict[str, Any]] = None, + exited: int = 0, + pty: bool = False, + hide: Tuple[str, ...] = tuple(), + ): + self.stdout = stdout + self.stderr = stderr + if encoding is None: + encoding = default_encoding() + self.encoding = encoding + self.command = command + self.shell = shell + self.env = {} if env is None else env + self.exited = exited + self.pty = pty + self.hide = hide + + @property + def return_code(self) -> int: + """ + An alias for ``.exited``. + + .. versionadded:: 1.0 + """ + return self.exited + + def __bool__(self) -> bool: + return self.ok + + def __str__(self) -> str: + if self.exited is not None: + desc = "Command exited with status {}.".format(self.exited) + else: + desc = "Command was not fully executed due to watcher error." + ret = [desc] + for x in ("stdout", "stderr"): + val = getattr(self, x) + ret.append( + """=== {} === +{} +""".format( + x, val.rstrip() + ) + if val + else "(no {})".format(x) + ) + return "\n".join(ret) + + def __repr__(self) -> str: + # TODO: more? e.g. len of stdout/err? (how to represent cleanly in a + # 'x=y' format like this? e.g. '4b' is ambiguous as to what it + # represents + template = "" + return template.format(self.command, self.exited) + + @property + def ok(self) -> bool: + """ + A boolean equivalent to ``exited == 0``. + + .. versionadded:: 1.0 + """ + return bool(self.exited == 0) + + @property + def failed(self) -> bool: + """ + The inverse of ``ok``. + + I.e., ``True`` if the program exited with a nonzero return code, and + ``False`` otherwise. + + .. versionadded:: 1.0 + """ + return not self.ok + + def tail(self, stream: str, count: int = 10) -> str: + """ + Return the last ``count`` lines of ``stream``, plus leading whitespace. + + :param str stream: + Name of some captured stream attribute, eg ``"stdout"``. + :param int count: + Number of lines to preserve. + + .. versionadded:: 1.3 + """ + # TODO: preserve alternate line endings? Mehhhh + # NOTE: no trailing \n preservation; easier for below display if + # normalized + return "\n\n" + "\n".join(getattr(self, stream).splitlines()[-count:]) + + +class Promise(Result): + """ + A promise of some future `Result`, yielded from asynchronous execution. + + This class' primary API member is `join`; instances may also be used as + context managers, which will automatically call `join` when the block + exits. In such cases, the context manager yields ``self``. + + `Promise` also exposes copies of many `Result` attributes, specifically + those that derive from `~Runner.run` kwargs and not the result of command + execution. For example, ``command`` is replicated here, but ``stdout`` is + not. + + .. versionadded:: 1.4 + """ + + def __init__(self, runner: "Runner") -> None: + """ + Create a new promise. + + :param runner: + An in-flight `Runner` instance making this promise. + + Must already have started the subprocess and spun up IO threads. + """ + self.runner = runner + # Basically just want exactly this (recently refactored) kwargs dict. + # TODO: consider proxying vs copying, but prob wait for refactor + for key, value in self.runner.result_kwargs.items(): + setattr(self, key, value) + + def join(self) -> Result: + """ + Block until associated subprocess exits, returning/raising the result. + + This acts identically to the end of a synchronously executed ``run``, + namely that: + + - various background threads (such as IO workers) are themselves + joined; + - if the subprocess exited normally, a `Result` is returned; + - in any other case (unforeseen exceptions, IO sub-thread + `.ThreadException`, `.Failure`, `.WatcherError`) the relevant + exception is raised here. + + See `~Runner.run` docs, or those of the relevant classes, for further + details. + """ + try: + return self.runner._finish() + finally: + self.runner.stop() + + def __enter__(self) -> "Promise": + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: BaseException, + exc_tb: Optional[TracebackType], + ) -> None: + self.join() + + +def normalize_hide( + val: Any, + out_stream: Optional[str] = None, + err_stream: Optional[str] = None, +) -> Tuple[str, ...]: + # Normalize to list-of-stream-names + hide_vals = (None, False, "out", "stdout", "err", "stderr", "both", True) + if val not in hide_vals: + err = "'hide' got {!r} which is not in {!r}" + raise ValueError(err.format(val, hide_vals)) + if val in (None, False): + hide = [] + elif val in ("both", True): + hide = ["stdout", "stderr"] + elif val == "out": + hide = ["stdout"] + elif val == "err": + hide = ["stderr"] + else: + hide = [val] + # Revert any streams that have been overridden from the default value + if out_stream is not None and "stdout" in hide: + hide.remove("stdout") + if err_stream is not None and "stderr" in hide: + hide.remove("stderr") + return tuple(hide) + + +def default_encoding() -> str: + """ + Obtain apparent interpreter-local default text encoding. + + Often used as a baseline in situations where we must use SOME encoding for + unknown-but-presumably-text bytes, and the user has not specified an + override. + """ + encoding = locale.getpreferredencoding(False) + return encoding diff --git a/lib/invoke/tasks.py b/lib/invoke/tasks.py new file mode 100644 index 0000000..cd3075e --- /dev/null +++ b/lib/invoke/tasks.py @@ -0,0 +1,519 @@ +""" +This module contains the core `.Task` class & convenience decorators used to +generate new tasks. +""" + +import inspect +import types +from copy import deepcopy +from functools import update_wrapper +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Generic, + Iterable, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, +) + +from .context import Context +from .parser import Argument, translate_underscores + +if TYPE_CHECKING: + from inspect import Signature + from .config import Config + +T = TypeVar("T", bound=Callable) + + +class Task(Generic[T]): + """ + Core object representing an executable task & its argument specification. + + For the most part, this object is a clearinghouse for all of the data that + may be supplied to the `@task ` decorator, such as + ``name``, ``aliases``, ``positional`` etc, which appear as attributes. + + In addition, instantiation copies some introspection/documentation friendly + metadata off of the supplied ``body`` object, such as ``__doc__``, + ``__name__`` and ``__module__``, allowing it to "appear as" ``body`` for + most intents and purposes. + + .. versionadded:: 1.0 + """ + + # TODO: store these kwarg defaults central, refer to those values both here + # and in @task. + # TODO: allow central per-session / per-taskmodule control over some of + # them, e.g. (auto_)positional, auto_shortflags. + # NOTE: we shadow __builtins__.help here on purpose - obfuscating to avoid + # it feels bad, given the builtin will never actually be in play anywhere + # except a debug shell whose frame is exactly inside this class. + def __init__( + self, + body: Callable, + name: Optional[str] = None, + aliases: Iterable[str] = (), + positional: Optional[Iterable[str]] = None, + optional: Iterable[str] = (), + default: bool = False, + auto_shortflags: bool = True, + help: Optional[Dict[str, Any]] = None, + pre: Optional[Union[List[str], str]] = None, + post: Optional[Union[List[str], str]] = None, + autoprint: bool = False, + iterable: Optional[Iterable[str]] = None, + incrementable: Optional[Iterable[str]] = None, + ) -> None: + # Real callable + self.body = body + update_wrapper(self, self.body) + # Copy a bunch of special properties from the body for the benefit of + # Sphinx autodoc or other introspectors. + self.__doc__ = getattr(body, "__doc__", "") + self.__name__ = getattr(body, "__name__", "") + self.__module__ = getattr(body, "__module__", "") + # Default name, alternate names, and whether it should act as the + # default for its parent collection + self._name = name + self.aliases = aliases + self.is_default = default + # Arg/flag/parser hints + self.positional = self.fill_implicit_positionals(positional) + self.optional = tuple(optional) + self.iterable = iterable or [] + self.incrementable = incrementable or [] + self.auto_shortflags = auto_shortflags + self.help = (help or {}).copy() + # Call chain bidness + self.pre = pre or [] + self.post = post or [] + self.times_called = 0 + # Whether to print return value post-execution + self.autoprint = autoprint + + @property + def name(self) -> str: + return self._name or self.__name__ + + def __repr__(self) -> str: + aliases = "" + if self.aliases: + aliases = " ({})".format(", ".join(self.aliases)) + return "".format(self.name, aliases) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Task) or self.name != other.name: + return False + # Functions do not define __eq__ but func_code objects apparently do. + # (If we're wrapping some other callable, they will be responsible for + # defining equality on their end.) + if self.body == other.body: + return True + else: + try: + return self.body.__code__ == other.body.__code__ + except AttributeError: + return False + + def __hash__(self) -> int: + # Presumes name and body will never be changed. Hrm. + # Potentially cleaner to just not use Tasks as hash keys, but let's do + # this for now. + return hash(self.name) + hash(self.body) + + def __call__(self, *args: Any, **kwargs: Any) -> T: + # Guard against calling tasks with no context. + if not isinstance(args[0], Context): + err = "Task expected a Context as its first arg, got {} instead!" + # TODO: raise a custom subclass _of_ TypeError instead + raise TypeError(err.format(type(args[0]))) + result = self.body(*args, **kwargs) + self.times_called += 1 + return result + + @property + def called(self) -> bool: + return self.times_called > 0 + + def argspec(self, body: Callable) -> "Signature": + """ + Returns a modified `inspect.Signature` based on that of ``body``. + + :returns: + an `inspect.Signature` matching that of ``body``, but with the + initial context argument removed. + :raises TypeError: + if the task lacks an initial positional `.Context` argument. + + .. versionadded:: 1.0 + .. versionchanged:: 2.0 + Changed from returning a two-tuple of ``(arg_names, spec_dict)`` to + returning an `inspect.Signature`. + """ + # Handle callable-but-not-function objects + func = ( + body + if isinstance(body, types.FunctionType) + else body.__call__ # type: ignore + ) + # Rebuild signature with first arg dropped, or die usefully(ish trying + sig = inspect.signature(func) + params = list(sig.parameters.values()) + # TODO: this ought to also check if an extant 1st param _was_ a Context + # arg, and yell similarly if not. + if not len(params): + # TODO: see TODO under __call__, this should be same type + raise TypeError("Tasks must have an initial Context argument!") + return sig.replace(parameters=params[1:]) + + def fill_implicit_positionals( + self, positional: Optional[Iterable[str]] + ) -> Iterable[str]: + # If positionals is None, everything lacking a default + # value will be automatically considered positional. + if positional is None: + positional = [ + x.name + for x in self.argspec(self.body).parameters.values() + if x.default is inspect.Signature.empty + ] + return positional + + def arg_opts( + self, name: str, default: str, taken_names: Set[str] + ) -> Dict[str, Any]: + opts: Dict[str, Any] = {} + # Whether it's positional or not + opts["positional"] = name in self.positional + # Whether it is a value-optional flag + opts["optional"] = name in self.optional + # Whether it should be of an iterable (list) kind + if name in self.iterable: + opts["kind"] = list + # If user gave a non-None default, hopefully they know better + # than us what they want here (and hopefully it offers the list + # protocol...) - otherwise supply useful default + opts["default"] = default if default is not None else [] + # Whether it should increment its value or not + if name in self.incrementable: + opts["incrementable"] = True + # Argument name(s) (replace w/ dashed version if underscores present, + # and move the underscored version to be the attr_name instead.) + original_name = name # For reference in eg help= + if "_" in name: + opts["attr_name"] = name + name = translate_underscores(name) + names = [name] + if self.auto_shortflags: + # Must know what short names are available + for char in name: + if not (char == name or char in taken_names): + names.append(char) + break + opts["names"] = names + # Handle default value & kind if possible + if default not in (None, inspect.Signature.empty): + # TODO: allow setting 'kind' explicitly. + # NOTE: skip setting 'kind' if optional is True + type(default) is + # bool; that results in a nonsensical Argument which gives the + # parser grief in a few ways. + kind = type(default) + if not (opts["optional"] and kind is bool): + opts["kind"] = kind + opts["default"] = default + # Help + for possibility in name, original_name: + if possibility in self.help: + opts["help"] = self.help.pop(possibility) + break + return opts + + def get_arguments( + self, ignore_unknown_help: Optional[bool] = None + ) -> List[Argument]: + """ + Return a list of Argument objects representing this task's signature. + + :param bool ignore_unknown_help: + Controls whether unknown help flags cause errors. See the config + option by the same name for details. + + .. versionadded:: 1.0 + .. versionchanged:: 1.7 + Added the ``ignore_unknown_help`` kwarg. + """ + # Core argspec + sig = self.argspec(self.body) + # Prime the list of all already-taken names (mostly for help in + # choosing auto shortflags) + taken_names = set(sig.parameters.keys()) + # Build arg list (arg_opts will take care of setting up shortnames, + # etc) + args = [] + for param in sig.parameters.values(): + new_arg = Argument( + **self.arg_opts(param.name, param.default, taken_names) + ) + args.append(new_arg) + # Update taken_names list with new argument's full name list + # (which may include new shortflags) so subsequent Argument + # creation knows what's taken. + taken_names.update(set(new_arg.names)) + # If any values were leftover after consuming a 'help' dict, it implies + # the user messed up & had a typo or similar. Let's explode. + if self.help and not ignore_unknown_help: + raise ValueError( + "Help field was set for param(s) that don't exist: {}".format( + list(self.help.keys()) + ) + ) + # Now we need to ensure positionals end up in the front of the list, in + # order given in self.positionals, so that when Context consumes them, + # this order is preserved. + for posarg in reversed(list(self.positional)): + for i, arg in enumerate(args): + if arg.name == posarg: + args.insert(0, args.pop(i)) + break + return args + + +def task(*args: Any, **kwargs: Any) -> Callable: + """ + Marks wrapped callable object as a valid Invoke task. + + May be called without any parentheses if no extra options need to be + specified. Otherwise, the following keyword arguments are allowed in the + parenthese'd form: + + * ``name``: Default name to use when binding to a `.Collection`. Useful for + avoiding Python namespace issues (i.e. when the desired CLI level name + can't or shouldn't be used as the Python level name.) + * ``aliases``: Specify one or more aliases for this task, allowing it to be + invoked as multiple different names. For example, a task named ``mytask`` + with a simple ``@task`` wrapper may only be invoked as ``"mytask"``. + Changing the decorator to be ``@task(aliases=['myothertask'])`` allows + invocation as ``"mytask"`` *or* ``"myothertask"``. + * ``positional``: Iterable overriding the parser's automatic "args with no + default value are considered positional" behavior. If a list of arg + names, no args besides those named in this iterable will be considered + positional. (This means that an empty list will force all arguments to be + given as explicit flags.) + * ``optional``: Iterable of argument names, declaring those args to + have :ref:`optional values `. Such arguments may be + given as value-taking options (e.g. ``--my-arg=myvalue``, wherein the + task is given ``"myvalue"``) or as Boolean flags (``--my-arg``, resulting + in ``True``). + * ``iterable``: Iterable of argument names, declaring them to :ref:`build + iterable values `. + * ``incrementable``: Iterable of argument names, declaring them to + :ref:`increment their values `. + * ``default``: Boolean option specifying whether this task should be its + collection's default task (i.e. called if the collection's own name is + given.) + * ``auto_shortflags``: Whether or not to automatically create short + flags from task options; defaults to True. + * ``help``: Dict mapping argument names to their help strings. Will be + displayed in ``--help`` output. For arguments containing underscores + (which are transformed into dashes on the CLI by default), either the + dashed or underscored version may be supplied here. + * ``pre``, ``post``: Lists of task objects to execute prior to, or after, + the wrapped task whenever it is executed. + * ``autoprint``: Boolean determining whether to automatically print this + task's return value to standard output when invoked directly via the CLI. + Defaults to False. + * ``klass``: Class to instantiate/return. Defaults to `.Task`. + + If any non-keyword arguments are given, they are taken as the value of the + ``pre`` kwarg for convenience's sake. (It is an error to give both + ``*args`` and ``pre`` at the same time.) + + .. versionadded:: 1.0 + .. versionchanged:: 1.1 + Added the ``klass`` keyword argument. + """ + klass: Type[Task] = kwargs.pop("klass", Task) + # @task -- no options were (probably) given. + if len(args) == 1 and callable(args[0]) and not isinstance(args[0], Task): + return klass(args[0], **kwargs) + # @task(pre, tasks, here) + if args: + if "pre" in kwargs: + raise TypeError( + "May not give *args and 'pre' kwarg simultaneously!" + ) + kwargs["pre"] = args + + def inner(body: Callable) -> Task[T]: + _task = klass(body, **kwargs) + return _task + + # update_wrapper(inner, klass) + return inner + + +class Call: + """ + Represents a call/execution of a `.Task` with given (kw)args. + + Similar to `~functools.partial` with some added functionality (such as the + delegation to the inner task, and optional tracking of the name it's being + called by.) + + .. versionadded:: 1.0 + """ + + def __init__( + self, + task: "Task", + called_as: Optional[str] = None, + args: Optional[Tuple[str, ...]] = None, + kwargs: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Create a new `.Call` object. + + :param task: The `.Task` object to be executed. + + :param str called_as: + The name the task is being called as, e.g. if it was called by an + alias or other rebinding. Defaults to ``None``, aka, the task was + referred to by its default name. + + :param tuple args: + Positional arguments to call with, if any. Default: ``None``. + + :param dict kwargs: + Keyword arguments to call with, if any. Default: ``None``. + """ + self.task = task + self.called_as = called_as + self.args = args or tuple() + self.kwargs = kwargs or dict() + + # TODO: just how useful is this? feels like maybe overkill magic + def __getattr__(self, name: str) -> Any: + return getattr(self.task, name) + + def __deepcopy__(self, memo: object) -> "Call": + return self.clone() + + def __repr__(self) -> str: + aka = "" + if self.called_as is not None and self.called_as != self.task.name: + aka = " (called as: {!r})".format(self.called_as) + return "<{} {!r}{}, args: {!r}, kwargs: {!r}>".format( + self.__class__.__name__, + self.task.name, + aka, + self.args, + self.kwargs, + ) + + def __eq__(self, other: object) -> bool: + # NOTE: Not comparing 'called_as'; a named call of a given Task with + # same args/kwargs should be considered same as an unnamed call of the + # same Task with the same args/kwargs (e.g. pre/post task specified w/o + # name). Ditto tasks with multiple aliases. + for attr in "task args kwargs".split(): + if getattr(self, attr) != getattr(other, attr): + return False + return True + + def make_context(self, config: "Config") -> Context: + """ + Generate a `.Context` appropriate for this call, with given config. + + .. versionadded:: 1.0 + """ + return Context(config=config) + + def clone_data(self) -> Dict[str, Any]: + """ + Return keyword args suitable for cloning this call into another. + + .. versionadded:: 1.1 + """ + return dict( + task=self.task, + called_as=self.called_as, + args=deepcopy(self.args), + kwargs=deepcopy(self.kwargs), + ) + + def clone( + self, + into: Optional[Type["Call"]] = None, + with_: Optional[Dict[str, Any]] = None, + ) -> "Call": + """ + Return a standalone copy of this Call. + + Useful when parameterizing task executions. + + :param into: + A subclass to generate instead of the current class. Optional. + + :param dict with_: + A dict of additional keyword arguments to use when creating the new + clone; typically used when cloning ``into`` a subclass that has + extra args on top of the base class. Optional. + + .. note:: + This dict is used to ``.update()`` the original object's data + (the return value from its `clone_data`), so in the event of + a conflict, values in ``with_`` will win out. + + .. versionadded:: 1.0 + .. versionchanged:: 1.1 + Added the ``with_`` kwarg. + """ + klass = into if into is not None else self.__class__ + data = self.clone_data() + if with_ is not None: + data.update(with_) + return klass(**data) + + +def call(task: "Task", *args: Any, **kwargs: Any) -> "Call": + """ + Describes execution of a `.Task`, typically with pre-supplied arguments. + + Useful for setting up :ref:`pre/post task invocations + `. It's actually just a convenient wrapper + around the `.Call` class, which may be used directly instead if desired. + + For example, here's two build-like tasks that both refer to a ``setup`` + pre-task, one with no baked-in argument values (and thus no need to use + `.call`), and one that toggles a boolean flag:: + + @task + def setup(c, clean=False): + if clean: + c.run("rm -rf target") + # ... setup things here ... + c.run("tar czvf target.tgz target") + + @task(pre=[setup]) + def build(c): + c.run("build, accounting for leftover files...") + + @task(pre=[call(setup, clean=True)]) + def clean_build(c): + c.run("build, assuming clean slate...") + + Please see the constructor docs for `.Call` for details - this function's + ``args`` and ``kwargs`` map directly to the same arguments as in that + method. + + .. versionadded:: 1.0 + """ + return Call(task, args=args, kwargs=kwargs) diff --git a/lib/invoke/terminals.py b/lib/invoke/terminals.py new file mode 100644 index 0000000..4151ba5 --- /dev/null +++ b/lib/invoke/terminals.py @@ -0,0 +1,248 @@ +""" +Utility functions surrounding terminal devices & I/O. + +Much of this code performs platform-sensitive branching, e.g. Windows support. + +This is its own module to abstract away what would otherwise be distracting +logic-flow interruptions. +""" + +from contextlib import contextmanager +from typing import Generator, IO, Optional, Tuple +import os +import select +import sys + +# TODO: move in here? They're currently platform-agnostic... +from .util import has_fileno, isatty + + +WINDOWS = sys.platform == "win32" +""" +Whether or not the current platform appears to be Windows in nature. + +Note that Cygwin's Python is actually close enough to "real" UNIXes that it +doesn't need (or want!) to use PyWin32 -- so we only test for literal Win32 +setups (vanilla Python, ActiveState etc) here. + +.. versionadded:: 1.0 +""" + +if sys.platform == "win32": + import msvcrt + from ctypes import ( + Structure, + c_ushort, + windll, + POINTER, + byref, + ) + from ctypes.wintypes import HANDLE, _COORD, _SMALL_RECT +else: + import fcntl + import struct + import termios + import tty + + +if sys.platform == "win32": + + def _pty_size() -> Tuple[Optional[int], Optional[int]]: + class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", c_ushort), + ("srWindow", _SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + GetStdHandle = windll.kernel32.GetStdHandle + GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo + GetStdHandle.restype = HANDLE + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + + hstd = GetStdHandle(-11) # STD_OUTPUT_HANDLE = -11 + csbi = CONSOLE_SCREEN_BUFFER_INFO() + ret = GetConsoleScreenBufferInfo(hstd, byref(csbi)) + + if ret: + sizex = csbi.srWindow.Right - csbi.srWindow.Left + 1 + sizey = csbi.srWindow.Bottom - csbi.srWindow.Top + 1 + return sizex, sizey + else: + return (None, None) + +else: + + def _pty_size() -> Tuple[Optional[int], Optional[int]]: + """ + Suitable for most POSIX platforms. + + .. versionadded:: 1.0 + """ + # Sentinel values to be replaced w/ defaults by caller + size = (None, None) + # We want two short unsigned integers (rows, cols) + # Note: TIOCGWINSZ struct contains 4 unsigned shorts, 2 unused + fmt = "HHHH" + # Create an empty (zeroed) buffer for ioctl to map onto. Yay for C! + buf = struct.pack(fmt, 0, 0, 0, 0) + # Call TIOCGWINSZ to get window size of stdout, returns our filled + # buffer + try: + result = fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, buf) + # Unpack buffer back into Python data types + # NOTE: this unpack gives us rows x cols, but we return the + # inverse. + rows, cols, *_ = struct.unpack(fmt, result) + return (cols, rows) + # Fallback to emptyish return value in various failure cases: + # * sys.stdout being monkeypatched, such as in testing, and lacking + # * .fileno + # * sys.stdout having a .fileno but not actually being attached to a + # * TTY + # * termios not having a TIOCGWINSZ attribute (happens sometimes...) + # * other situations where ioctl doesn't explode but the result isn't + # something unpack can deal with + except (struct.error, TypeError, IOError, AttributeError): + pass + return size + + +def pty_size() -> Tuple[int, int]: + """ + Determine current local pseudoterminal dimensions. + + :returns: + A ``(num_cols, num_rows)`` two-tuple describing PTY size. Defaults to + ``(80, 24)`` if unable to get a sensible result dynamically. + + .. versionadded:: 1.0 + """ + cols, rows = _pty_size() + # TODO: make defaults configurable? + return (cols or 80, rows or 24) + + +def stdin_is_foregrounded_tty(stream: IO) -> bool: + """ + Detect if given stdin ``stream`` seems to be in the foreground of a TTY. + + Specifically, compares the current Python process group ID to that of the + stream's file descriptor to see if they match; if they do not match, it is + likely that the process has been placed in the background. + + This is used as a test to determine whether we should manipulate an active + stdin so it runs in a character-buffered mode; touching the terminal in + this way when the process is backgrounded, causes most shells to pause + execution. + + .. note:: + Processes that aren't attached to a terminal to begin with, will always + fail this test, as it starts with "do you have a real ``fileno``?". + + .. versionadded:: 1.0 + """ + if not has_fileno(stream): + return False + return os.getpgrp() == os.tcgetpgrp(stream.fileno()) + + +def cbreak_already_set(stream: IO) -> bool: + # Explicitly not docstringed to remain private, for now. Eh. + # Checks whether tty.setcbreak appears to have already been run against + # ``stream`` (or if it would otherwise just not do anything). + # Used to effect idempotency for character-buffering a stream, which also + # lets us avoid multiple capture-then-restore cycles. + attrs = termios.tcgetattr(stream) + lflags, cc = attrs[3], attrs[6] + echo = bool(lflags & termios.ECHO) + icanon = bool(lflags & termios.ICANON) + # setcbreak sets ECHO and ICANON to 0/off, CC[VMIN] to 1-ish, and CC[VTIME] + # to 0-ish. If any of that is not true we can reasonably assume it has not + # yet been executed against this stream. + sentinels = ( + not echo, + not icanon, + cc[termios.VMIN] in [1, b"\x01"], + cc[termios.VTIME] in [0, b"\x00"], + ) + return all(sentinels) + + +@contextmanager +def character_buffered( + stream: IO, +) -> Generator[None, None, None]: + """ + Force local terminal ``stream`` be character, not line, buffered. + + Only applies to Unix-based systems; on Windows this is a no-op. + + .. versionadded:: 1.0 + """ + if ( + WINDOWS + or not isatty(stream) + or not stdin_is_foregrounded_tty(stream) + or cbreak_already_set(stream) + ): + yield + else: + old_settings = termios.tcgetattr(stream) + tty.setcbreak(stream) + try: + yield + finally: + termios.tcsetattr(stream, termios.TCSADRAIN, old_settings) + + +def ready_for_reading(input_: IO) -> bool: + """ + Test ``input_`` to determine whether a read action will succeed. + + :param input_: Input stream object (file-like). + + :returns: ``True`` if a read should succeed, ``False`` otherwise. + + .. versionadded:: 1.0 + """ + # A "real" terminal stdin needs select/kbhit to tell us when it's ready for + # a nonblocking read(). + # Otherwise, assume a "safer" file-like object that can be read from in a + # nonblocking fashion (e.g. a StringIO or regular file). + if not has_fileno(input_): + return True + if sys.platform == "win32": + return msvcrt.kbhit() + else: + reads, _, _ = select.select([input_], [], [], 0.0) + return bool(reads and reads[0] is input_) + + +def bytes_to_read(input_: IO) -> int: + """ + Query stream ``input_`` to see how many bytes may be readable. + + .. note:: + If we are unable to tell (e.g. if ``input_`` isn't a true file + descriptor or isn't a valid TTY) we fall back to suggesting reading 1 + byte only. + + :param input: Input stream object (file-like). + + :returns: `int` number of bytes to read. + + .. versionadded:: 1.0 + """ + # NOTE: we have to check both possibilities here; situations exist where + # it's not a tty but has a fileno, or vice versa; neither is typically + # going to work re: ioctl(). + if not WINDOWS and isatty(input_) and has_fileno(input_): + fionread = fcntl.ioctl(input_, termios.FIONREAD, b" ") + return int(struct.unpack("h", fionread)[0]) + return 1 diff --git a/lib/invoke/util.py b/lib/invoke/util.py new file mode 100644 index 0000000..df29c84 --- /dev/null +++ b/lib/invoke/util.py @@ -0,0 +1,268 @@ +from collections import namedtuple +from contextlib import contextmanager +from types import TracebackType +from typing import Any, Generator, List, IO, Optional, Tuple, Type, Union +import io +import logging +import os +import threading +import sys + +# NOTE: This is the canonical location for commonly-used vendored modules, +# which is the only spot that performs this try/except to allow repackaged +# Invoke to function (e.g. distro packages which unvendor the vendored bits and +# thus must import our 'vendored' stuff from the overall environment.) +# All other uses of Lexicon, etc should do 'from .util import lexicon' etc. +# Saves us from having to update the same logic in a dozen places. +# TODO: would this make more sense to put _into_ invoke.vendor? That way, the +# import lines which now read 'from .util import ' would be +# more obvious. Requires packagers to leave invoke/vendor/__init__.py alone tho +try: + from .vendor.lexicon import Lexicon # noqa + from .vendor import yaml # noqa +except ImportError: + from lexicon import Lexicon # type: ignore[no-redef] # noqa + import yaml # type: ignore[no-redef] # noqa + + +LOG_FORMAT = "%(name)s.%(module)s.%(funcName)s: %(message)s" + + +def enable_logging() -> None: + logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) + + +# Allow from-the-start debugging (vs toggled during load of tasks module) via +# shell env var. +if os.environ.get("INVOKE_DEBUG"): + enable_logging() + +# Add top level logger functions to global namespace. Meh. +log = logging.getLogger("invoke") +debug = log.debug + + +def task_name_sort_key(name: str) -> Tuple[List[str], str]: + """ + Return key tuple for use sorting dotted task names, via e.g. `sorted`. + + .. versionadded:: 1.0 + """ + parts = name.split(".") + return ( + # First group/sort by non-leaf path components. This keeps everything + # grouped in its hierarchy, and incidentally puts top-level tasks + # (whose non-leaf path set is the empty list) first, where we want them + parts[:-1], + # Then we sort lexicographically by the actual task name + parts[-1], + ) + + +# TODO: Make part of public API sometime +@contextmanager +def cd(where: str) -> Generator[None, None, None]: + cwd = os.getcwd() + os.chdir(where) + try: + yield + finally: + os.chdir(cwd) + + +def has_fileno(stream: IO) -> bool: + """ + Cleanly determine whether ``stream`` has a useful ``.fileno()``. + + .. note:: + This function helps determine if a given file-like object can be used + with various terminal-oriented modules and functions such as `select`, + `termios`, and `tty`. For most of those, a fileno is all that is + required; they'll function even if ``stream.isatty()`` is ``False``. + + :param stream: A file-like object. + + :returns: + ``True`` if ``stream.fileno()`` returns an integer, ``False`` otherwise + (this includes when ``stream`` lacks a ``fileno`` method). + + .. versionadded:: 1.0 + """ + try: + return isinstance(stream.fileno(), int) + except (AttributeError, io.UnsupportedOperation): + return False + + +def isatty(stream: IO) -> Union[bool, Any]: + """ + Cleanly determine whether ``stream`` is a TTY. + + Specifically, first try calling ``stream.isatty()``, and if that fails + (e.g. due to lacking the method entirely) fallback to `os.isatty`. + + .. note:: + Most of the time, we don't actually care about true TTY-ness, but + merely whether the stream seems to have a fileno (per `has_fileno`). + However, in some cases (notably the use of `pty.fork` to present a + local pseudoterminal) we need to tell if a given stream has a valid + fileno but *isn't* tied to an actual terminal. Thus, this function. + + :param stream: A file-like object. + + :returns: + A boolean depending on the result of calling ``.isatty()`` and/or + `os.isatty`. + + .. versionadded:: 1.0 + """ + # If there *is* an .isatty, ask it. + if hasattr(stream, "isatty") and callable(stream.isatty): + return stream.isatty() + # If there wasn't, see if it has a fileno, and if so, ask os.isatty + elif has_fileno(stream): + return os.isatty(stream.fileno()) + # If we got here, none of the above worked, so it's reasonable to assume + # the darn thing isn't a real TTY. + return False + + +def helpline(obj: object) -> Optional[str]: + """ + Yield an object's first docstring line, or None if there was no docstring. + + .. versionadded:: 1.0 + """ + docstring = obj.__doc__ + if ( + not docstring + or not docstring.strip() + or docstring == type(obj).__doc__ + ): + return None + return docstring.lstrip().splitlines()[0] + + +class ExceptionHandlingThread(threading.Thread): + """ + Thread handler making it easier for parent to handle thread exceptions. + + Based in part on Fabric 1's ThreadHandler. See also Fabric GH issue #204. + + When used directly, can be used in place of a regular ``threading.Thread``. + If subclassed, the subclass must do one of: + + - supply ``target`` to ``__init__`` + - define ``_run()`` instead of ``run()`` + + This is because this thread's entire point is to wrap behavior around the + thread's execution; subclasses could not redefine ``run()`` without + breaking that functionality. + + .. versionadded:: 1.0 + """ + + def __init__(self, **kwargs: Any) -> None: + """ + Create a new exception-handling thread instance. + + Takes all regular `threading.Thread` keyword arguments, via + ``**kwargs`` for easier display of thread identity when raising + captured exceptions. + """ + super().__init__(**kwargs) + # No record of why, but Fabric used daemon threads ever since the + # switch from select.select, so let's keep doing that. + self.daemon = True + # Track exceptions raised in run() + self.kwargs = kwargs + # TODO: legacy cruft that needs to be removed + self.exc_info: Optional[ + Union[ + Tuple[Type[BaseException], BaseException, TracebackType], + Tuple[None, None, None], + ] + ] = None + + def run(self) -> None: + try: + # Allow subclasses implemented using the "override run()'s body" + # approach to work, by using _run() instead of run(). If that + # doesn't appear to be the case, then assume we're being used + # directly and just use super() ourselves. + # XXX https://github.com/python/mypy/issues/1424 + if hasattr(self, "_run") and callable(self._run): # type: ignore + # TODO: this could be: + # - io worker with no 'result' (always local) + # - tunnel worker, also with no 'result' (also always local) + # - threaded concurrent run(), sudo(), put(), etc, with a + # result (not necessarily local; might want to be a subproc or + # whatever eventually) + # TODO: so how best to conditionally add a "capture result + # value of some kind"? + # - update so all use cases use subclassing, add functionality + # alongside self.exception() that is for the result of _run() + # - split out class that does not care about result of _run() + # and let it continue acting like a normal thread (meh) + # - assume the run/sudo/etc case will use a queue inside its + # worker body, orthogonal to how exception handling works + self._run() # type: ignore + else: + super().run() + except BaseException: + # Store for actual reraising later + self.exc_info = sys.exc_info() + # And log now, in case we never get to later (e.g. if executing + # program is hung waiting for us to do something) + msg = "Encountered exception {!r} in thread for {!r}" + # Name is either target function's dunder-name, or just "_run" if + # we were run subclass-wise. + name = "_run" + if "target" in self.kwargs: + name = self.kwargs["target"].__name__ + debug(msg.format(self.exc_info[1], name)) # noqa + + def exception(self) -> Optional["ExceptionWrapper"]: + """ + If an exception occurred, return an `.ExceptionWrapper` around it. + + :returns: + An `.ExceptionWrapper` managing the result of `sys.exc_info`, if an + exception was raised during thread execution. If no exception + occurred, returns ``None`` instead. + + .. versionadded:: 1.0 + """ + if self.exc_info is None: + return None + return ExceptionWrapper(self.kwargs, *self.exc_info) + + @property + def is_dead(self) -> bool: + """ + Returns ``True`` if not alive and has a stored exception. + + Used to detect threads that have excepted & shut down. + + .. versionadded:: 1.0 + """ + # NOTE: it seems highly unlikely that a thread could still be + # is_alive() but also have encountered an exception. But hey. Why not + # be thorough? + return (not self.is_alive()) and self.exc_info is not None + + def __repr__(self) -> str: + # TODO: beef this up more + return str(self.kwargs["target"].__name__) + + +# NOTE: ExceptionWrapper defined here, not in exceptions.py, to avoid circular +# dependency issues (e.g. Failure subclasses need to use some bits from this +# module...) +#: A namedtuple wrapping a thread-borne exception & that thread's arguments. +#: Mostly used as an intermediate between `.ExceptionHandlingThread` (which +#: preserves initial exceptions) and `.ThreadException` (which holds 1..N such +#: exceptions, as typically multiple threads are involved.) +ExceptionWrapper = namedtuple( + "ExceptionWrapper", "kwargs type value traceback" +) diff --git a/lib/invoke/vendor/__init__.py b/lib/invoke/vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/invoke/vendor/__pycache__/__init__.cpython-314.pyc b/lib/invoke/vendor/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..c68c386 Binary files /dev/null and b/lib/invoke/vendor/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/vendor/fluidity/__init__.py b/lib/invoke/vendor/fluidity/__init__.py new file mode 100644 index 0000000..3339fef --- /dev/null +++ b/lib/invoke/vendor/fluidity/__init__.py @@ -0,0 +1,4 @@ +from .machine import (StateMachine, state, transition, + InvalidConfiguration, InvalidTransition, + GuardNotSatisfied, ForkedTransition) + diff --git a/lib/invoke/vendor/fluidity/__pycache__/__init__.cpython-314.pyc b/lib/invoke/vendor/fluidity/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..5f331a5 Binary files /dev/null and b/lib/invoke/vendor/fluidity/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/vendor/fluidity/__pycache__/backwardscompat.cpython-314.pyc b/lib/invoke/vendor/fluidity/__pycache__/backwardscompat.cpython-314.pyc new file mode 100644 index 0000000..c9cfa29 Binary files /dev/null and b/lib/invoke/vendor/fluidity/__pycache__/backwardscompat.cpython-314.pyc differ diff --git a/lib/invoke/vendor/fluidity/__pycache__/machine.cpython-314.pyc b/lib/invoke/vendor/fluidity/__pycache__/machine.cpython-314.pyc new file mode 100644 index 0000000..a37fec4 Binary files /dev/null and b/lib/invoke/vendor/fluidity/__pycache__/machine.cpython-314.pyc differ diff --git a/lib/invoke/vendor/fluidity/backwardscompat.py b/lib/invoke/vendor/fluidity/backwardscompat.py new file mode 100644 index 0000000..88eac4f --- /dev/null +++ b/lib/invoke/vendor/fluidity/backwardscompat.py @@ -0,0 +1,8 @@ +import sys + +if sys.version_info >= (3,): + def callable(obj): + return hasattr(obj, '__call__') +else: + callable = callable + diff --git a/lib/invoke/vendor/fluidity/machine.py b/lib/invoke/vendor/fluidity/machine.py new file mode 100644 index 0000000..da9fdda --- /dev/null +++ b/lib/invoke/vendor/fluidity/machine.py @@ -0,0 +1,270 @@ +import re +import inspect +from .backwardscompat import callable + +# metaclass implementation idea from +# http://blog.ianbicking.org/more-on-python-metaprogramming-comment-14.html +_transition_gatherer = [] + +def transition(event, from_, to, action=None, guard=None): + _transition_gatherer.append([event, from_, to, action, guard]) + +_state_gatherer = [] + +def state(name, enter=None, exit=None): + _state_gatherer.append([name, enter, exit]) + + +class MetaStateMachine(type): + + def __new__(cls, name, bases, dictionary): + global _transition_gatherer, _state_gatherer + Machine = super(MetaStateMachine, cls).__new__(cls, name, bases, dictionary) + Machine._class_transitions = [] + Machine._class_states = {} + for s in _state_gatherer: + Machine._add_class_state(*s) + for i in _transition_gatherer: + Machine._add_class_transition(*i) + _transition_gatherer = [] + _state_gatherer = [] + return Machine + + +StateMachineBase = MetaStateMachine('StateMachineBase', (object, ), {}) + + +class StateMachine(StateMachineBase): + + def __init__(self): + self._bring_definitions_to_object_level() + self._inject_into_parts() + self._validate_machine_definitions() + if callable(self.initial_state): + self.initial_state = self.initial_state() + self._current_state_object = self._state_by_name(self.initial_state) + self._current_state_object.run_enter(self) + self._create_state_getters() + + def __new__(cls, *args, **kwargs): + obj = super(StateMachine, cls).__new__(cls) + obj._states = {} + obj._transitions = [] + return obj + + def _bring_definitions_to_object_level(self): + self._states.update(self.__class__._class_states) + self._transitions.extend(self.__class__._class_transitions) + + def _inject_into_parts(self): + for collection in [self._states.values(), self._transitions]: + for component in collection: + component.machine = self + + def _validate_machine_definitions(self): + if len(self._states) < 2: + raise InvalidConfiguration('There must be at least two states') + if not getattr(self, 'initial_state', None): + raise InvalidConfiguration('There must exist an initial state') + + @classmethod + def _add_class_state(cls, name, enter, exit): + cls._class_states[name] = _State(name, enter, exit) + + def add_state(self, name, enter=None, exit=None): + state = _State(name, enter, exit) + setattr(self, state.getter_name(), state.getter_method().__get__(self, self.__class__)) + self._states[name] = state + + def _current_state_name(self): + return self._current_state_object.name + + current_state = property(_current_state_name) + + def changing_state(self, from_, to): + """ + This method is called whenever a state change is executed + """ + pass + + def _new_state(self, state): + self.changing_state(self._current_state_object.name, state.name) + self._current_state_object = state + + def _state_objects(self): + return list(self._states.values()) + + def states(self): + return [s.name for s in self._state_objects()] + + @classmethod + def _add_class_transition(cls, event, from_, to, action, guard): + transition = _Transition(event, [cls._class_states[s] for s in _listize(from_)], + cls._class_states[to], action, guard) + cls._class_transitions.append(transition) + setattr(cls, event, transition.event_method()) + + def add_transition(self, event, from_, to, action=None, guard=None): + transition = _Transition(event, [self._state_by_name(s) for s in _listize(from_)], + self._state_by_name(to), action, guard) + self._transitions.append(transition) + setattr(self, event, transition.event_method().__get__(self, self.__class__)) + + def _process_transitions(self, event_name, *args, **kwargs): + transitions = self._transitions_by_name(event_name) + transitions = self._ensure_from_validity(transitions) + this_transition = self._check_guards(transitions) + this_transition.run(self, *args, **kwargs) + + def _create_state_getters(self): + for state in self._state_objects(): + setattr(self, state.getter_name(), state.getter_method().__get__(self, self.__class__)) + + def _state_by_name(self, name): + for state in self._state_objects(): + if state.name == name: + return state + + def _transitions_by_name(self, name): + return list(filter(lambda transition: transition.event == name, self._transitions)) + + def _ensure_from_validity(self, transitions): + valid_transitions = list(filter( + lambda transition: transition.is_valid_from(self._current_state_object), + transitions)) + if len(valid_transitions) == 0: + raise InvalidTransition("Cannot %s from %s" % ( + transitions[0].event, self.current_state)) + return valid_transitions + + def _check_guards(self, transitions): + allowed_transitions = [] + for transition in transitions: + if transition.check_guard(self): + allowed_transitions.append(transition) + if len(allowed_transitions) == 0: + raise GuardNotSatisfied("Guard is not satisfied for this transition") + elif len(allowed_transitions) > 1: + raise ForkedTransition("More than one transition was allowed for this event") + return allowed_transitions[0] + + +class _Transition(object): + + def __init__(self, event, from_, to, action, guard): + self.event = event + self.from_ = from_ + self.to = to + self.action = action + self.guard = _Guard(guard) + + def event_method(self): + def generated_event(machine, *args, **kwargs): + these_transitions = machine._process_transitions(self.event, *args, **kwargs) + generated_event.__doc__ = 'event %s' % self.event + generated_event.__name__ = self.event + return generated_event + + def is_valid_from(self, from_): + return from_ in _listize(self.from_) + + def check_guard(self, machine): + return self.guard.check(machine) + + def run(self, machine, *args, **kwargs): + machine._current_state_object.run_exit(machine) + machine._new_state(self.to) + self.to.run_enter(machine) + _ActionRunner(machine).run(self.action, *args, **kwargs) + + +class _Guard(object): + + def __init__(self, action): + self.action = action + + def check(self, machine): + if self.action is None: + return True + items = _listize(self.action) + result = True + for item in items: + result = result and self._evaluate(machine, item) + return result + + def _evaluate(self, machine, item): + if callable(item): + return item(machine) + else: + guard = getattr(machine, item) + if callable(guard): + guard = guard() + return guard + + +class _State(object): + + def __init__(self, name, enter, exit): + self.name = name + self.enter = enter + self.exit = exit + + def getter_name(self): + return 'is_%s' % self.name + + def getter_method(self): + def state_getter(self_machine): + return self_machine.current_state == self.name + return state_getter + + def run_enter(self, machine): + _ActionRunner(machine).run(self.enter) + + def run_exit(self, machine): + _ActionRunner(machine).run(self.exit) + + +class _ActionRunner(object): + + def __init__(self, machine): + self.machine = machine + + def run(self, action_param, *args, **kwargs): + if not action_param: + return + action_items = _listize(action_param) + for action_item in action_items: + self._run_action(action_item, *args, **kwargs) + + def _run_action(self, action, *args, **kwargs): + if callable(action): + self._try_to_run_with_args(action, self.machine, *args, **kwargs) + else: + self._try_to_run_with_args(getattr(self.machine, action), *args, **kwargs) + + def _try_to_run_with_args(self, action, *args, **kwargs): + try: + action(*args, **kwargs) + except TypeError: + action() + + +class InvalidConfiguration(Exception): + pass + + +class InvalidTransition(Exception): + pass + + +class GuardNotSatisfied(Exception): + pass + + +class ForkedTransition(Exception): + pass + + +def _listize(value): + return type(value) in [list, tuple] and value or [value] + diff --git a/lib/invoke/vendor/lexicon/__init__.py b/lib/invoke/vendor/lexicon/__init__.py new file mode 100644 index 0000000..c7f65d3 --- /dev/null +++ b/lib/invoke/vendor/lexicon/__init__.py @@ -0,0 +1,24 @@ +from ._version import __version_info__, __version__ # noqa +from .attribute_dict import AttributeDict +from .alias_dict import AliasDict + + +class Lexicon(AttributeDict, AliasDict): + def __init__(self, *args, **kwargs): + # Need to avoid combining AliasDict's initial attribute write on + # self.aliases, with AttributeDict's __setattr__. Doing so results in + # an infinite loop. Instead, just skip straight to dict() for both + # explicitly (i.e. we override AliasDict.__init__ instead of extending + # it.) + # NOTE: could tickle AttributeDict.__init__ instead, in case it ever + # grows one. + dict.__init__(self, *args, **kwargs) + dict.__setattr__(self, "aliases", {}) + + def __getattr__(self, key): + # Intercept deepcopy/etc driven access to self.aliases when not + # actually set. (Only a problem for us, due to abovementioned combo of + # Alias and Attribute Dicts, so not solvable in a parent alone.) + if key == "aliases" and key not in self.__dict__: + self.__dict__[key] = {} + return super(Lexicon, self).__getattr__(key) diff --git a/lib/invoke/vendor/lexicon/__pycache__/__init__.cpython-314.pyc b/lib/invoke/vendor/lexicon/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..26eed86 Binary files /dev/null and b/lib/invoke/vendor/lexicon/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/vendor/lexicon/__pycache__/_version.cpython-314.pyc b/lib/invoke/vendor/lexicon/__pycache__/_version.cpython-314.pyc new file mode 100644 index 0000000..8b4a4dd Binary files /dev/null and b/lib/invoke/vendor/lexicon/__pycache__/_version.cpython-314.pyc differ diff --git a/lib/invoke/vendor/lexicon/__pycache__/alias_dict.cpython-314.pyc b/lib/invoke/vendor/lexicon/__pycache__/alias_dict.cpython-314.pyc new file mode 100644 index 0000000..5afa69b Binary files /dev/null and b/lib/invoke/vendor/lexicon/__pycache__/alias_dict.cpython-314.pyc differ diff --git a/lib/invoke/vendor/lexicon/__pycache__/attribute_dict.cpython-314.pyc b/lib/invoke/vendor/lexicon/__pycache__/attribute_dict.cpython-314.pyc new file mode 100644 index 0000000..50a4d2a Binary files /dev/null and b/lib/invoke/vendor/lexicon/__pycache__/attribute_dict.cpython-314.pyc differ diff --git a/lib/invoke/vendor/lexicon/_version.py b/lib/invoke/vendor/lexicon/_version.py new file mode 100644 index 0000000..f55a4f1 --- /dev/null +++ b/lib/invoke/vendor/lexicon/_version.py @@ -0,0 +1,2 @@ +__version_info__ = (2, 0, 1) +__version__ = ".".join(map(str, __version_info__)) diff --git a/lib/invoke/vendor/lexicon/alias_dict.py b/lib/invoke/vendor/lexicon/alias_dict.py new file mode 100644 index 0000000..f2191fb --- /dev/null +++ b/lib/invoke/vendor/lexicon/alias_dict.py @@ -0,0 +1,95 @@ +class AliasDict(dict): + def __init__(self, *args, **kwargs): + super(AliasDict, self).__init__(*args, **kwargs) + self.aliases = {} + + def alias(self, from_, to): + self.aliases[from_] = to + + def unalias(self, from_): + del self.aliases[from_] + + def aliases_of(self, name): + """ + Returns other names for given real key or alias ``name``. + + If given a real key, returns its aliases. + + If given an alias, returns the real key it points to, plus any other + aliases of that real key. (The given alias itself is not included in + the return value.) + """ + names = [] + key = name + # self.aliases keys are aliases, not realkeys. Easy test to see if we + # should flip around to the POV of a realkey when given an alias. + if name in self.aliases: + key = self.aliases[name] + # Ensure the real key shows up in output. + names.append(key) + # 'key' is now a realkey, whose aliases are all keys whose value is + # itself. Filter out the original name given. + names.extend( + [k for k, v in self.aliases.items() if v == key and k != name] + ) + return names + + def _handle(self, key, value, single, multi, unaliased): + # Attribute existence test required to not blow up when deepcopy'd + if key in getattr(self, "aliases", {}): + target = self.aliases[key] + # Single-string targets + if isinstance(target, str): + return single(self, target, value) + # Multi-string targets + else: + if multi: + return multi(self, target, value) + else: + for subkey in target: + single(self, subkey, value) + else: + return unaliased(self, key, value) + + def __setitem__(self, key, value): + def single(d, target, value): + d[target] = value + + def unaliased(d, key, value): + super(AliasDict, d).__setitem__(key, value) + + return self._handle(key, value, single, None, unaliased) + + def __getitem__(self, key): + def single(d, target, value): + return d[target] + + def unaliased(d, key, value): + return super(AliasDict, d).__getitem__(key) + + def multi(d, target, value): + msg = "Multi-target aliases have no well-defined value and can't be read." # noqa + raise ValueError(msg) + + return self._handle(key, None, single, multi, unaliased) + + def __contains__(self, key): + def single(d, target, value): + return target in d + + def multi(d, target, value): + return all(subkey in self for subkey in self.aliases[key]) + + def unaliased(d, key, value): + return super(AliasDict, d).__contains__(key) + + return self._handle(key, None, single, multi, unaliased) + + def __delitem__(self, key): + def single(d, target, value): + del d[target] + + def unaliased(d, key, value): + return super(AliasDict, d).__delitem__(key) + + return self._handle(key, None, single, None, unaliased) diff --git a/lib/invoke/vendor/lexicon/attribute_dict.py b/lib/invoke/vendor/lexicon/attribute_dict.py new file mode 100644 index 0000000..5d09f13 --- /dev/null +++ b/lib/invoke/vendor/lexicon/attribute_dict.py @@ -0,0 +1,16 @@ +class AttributeDict(dict): + def __getattr__(self, key): + try: + return self[key] + except KeyError: + # to conform with __getattr__ spec + raise AttributeError(key) + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + del self[key] + + def __dir__(self): + return dir(type(self)) + list(self.keys()) diff --git a/lib/invoke/vendor/yaml/__init__.py b/lib/invoke/vendor/yaml/__init__.py new file mode 100644 index 0000000..86d07b5 --- /dev/null +++ b/lib/invoke/vendor/yaml/__init__.py @@ -0,0 +1,427 @@ + +from .error import * + +from .tokens import * +from .events import * +from .nodes import * + +from .loader import * +from .dumper import * + +__version__ = '5.4.1' +try: + from .cyaml import * + __with_libyaml__ = True +except ImportError: + __with_libyaml__ = False + +import io + +#------------------------------------------------------------------------------ +# Warnings control +#------------------------------------------------------------------------------ + +# 'Global' warnings state: +_warnings_enabled = { + 'YAMLLoadWarning': True, +} + +# Get or set global warnings' state +def warnings(settings=None): + if settings is None: + return _warnings_enabled + + if type(settings) is dict: + for key in settings: + if key in _warnings_enabled: + _warnings_enabled[key] = settings[key] + +# Warn when load() is called without Loader=... +class YAMLLoadWarning(RuntimeWarning): + pass + +def load_warning(method): + if _warnings_enabled['YAMLLoadWarning'] is False: + return + + import warnings + + message = ( + "calling yaml.%s() without Loader=... is deprecated, as the " + "default Loader is unsafe. Please read " + "https://msg.pyyaml.org/load for full details." + ) % method + + warnings.warn(message, YAMLLoadWarning, stacklevel=3) + +#------------------------------------------------------------------------------ +def scan(stream, Loader=Loader): + """ + Scan a YAML stream and produce scanning tokens. + """ + loader = Loader(stream) + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() + +def parse(stream, Loader=Loader): + """ + Parse a YAML stream and produce parsing events. + """ + loader = Loader(stream) + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() + +def compose(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding representation tree. + """ + loader = Loader(stream) + try: + return loader.get_single_node() + finally: + loader.dispose() + +def compose_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponding representation trees. + """ + loader = Loader(stream) + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() + +def load(stream, Loader=None): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + if Loader is None: + load_warning('load') + Loader = FullLoader + + loader = Loader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + +def load_all(stream, Loader=None): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + """ + if Loader is None: + load_warning('load_all') + Loader = FullLoader + + loader = Loader(stream) + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() + +def full_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load(stream, FullLoader) + +def full_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load_all(stream, FullLoader) + +def safe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load(stream, SafeLoader) + +def safe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load_all(stream, SafeLoader) + +def unsafe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load(stream, UnsafeLoader) + +def unsafe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load_all(stream, UnsafeLoader) + +def emit(events, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + """ + Emit YAML parsing events into a stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + stream = io.StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize_all(nodes, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of representation trees into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + stream = io.StringIO() + else: + stream = io.BytesIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize(node, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a representation tree into a YAML stream. + If stream is None, return the produced string instead. + """ + return serialize_all([node], stream, Dumper=Dumper, **kwds) + +def dump_all(documents, stream=None, Dumper=Dumper, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + """ + Serialize a sequence of Python objects into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + stream = io.StringIO() + else: + stream = io.BytesIO() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def dump(data, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a Python object into a YAML stream. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=Dumper, **kwds) + +def safe_dump_all(documents, stream=None, **kwds): + """ + Serialize a sequence of Python objects into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + +def safe_dump(data, stream=None, **kwds): + """ + Serialize a Python object into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + +def add_implicit_resolver(tag, regexp, first=None, + Loader=None, Dumper=Dumper): + """ + Add an implicit scalar detector. + If an implicit scalar value matches the given regexp, + the corresponding tag is assigned to the scalar. + first is a sequence of possible initial characters or None. + """ + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): + """ + Add a path based resolver for the given tag. + A path is a list of keys that forms a path + to a node in the representation tree. + Keys can be string values, integers, or None. + """ + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) + Dumper.add_path_resolver(tag, path, kind) + +def add_constructor(tag, constructor, Loader=None): + """ + Add a constructor for the given tag. + Constructor is a function that accepts a Loader instance + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): + """ + Add a multi-constructor for the given tag prefix. + Multi-constructor is called for a node if its tag starts with tag_prefix. + Multi-constructor accepts a Loader instance, a tag suffix, + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) + +def add_representer(data_type, representer, Dumper=Dumper): + """ + Add a representer for the given type. + Representer is a function accepting a Dumper instance + and an instance of the given data type + and producing the corresponding representation node. + """ + Dumper.add_representer(data_type, representer) + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + """ + Add a representer for the given type. + Multi-representer is a function accepting a Dumper instance + and an instance of the given data type or subtype + and producing the corresponding representation node. + """ + Dumper.add_multi_representer(data_type, multi_representer) + +class YAMLObjectMetaclass(type): + """ + The metaclass for YAMLObject. + """ + def __init__(cls, name, bases, kwds): + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + +class YAMLObject(metaclass=YAMLObjectMetaclass): + """ + An object that can dump itself to a YAML stream + and load itself from a YAML stream. + """ + + __slots__ = () # no direct instantiation, so allow immutable subclasses + + yaml_loader = [Loader, FullLoader, UnsafeLoader] + yaml_dumper = Dumper + + yaml_tag = None + yaml_flow_style = None + + @classmethod + def from_yaml(cls, loader, node): + """ + Convert a representation node to a Python object. + """ + return loader.construct_yaml_object(node, cls) + + @classmethod + def to_yaml(cls, dumper, data): + """ + Convert a Python object to a representation node. + """ + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, + flow_style=cls.yaml_flow_style) + diff --git a/lib/invoke/vendor/yaml/__pycache__/__init__.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..809967c Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/composer.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/composer.cpython-314.pyc new file mode 100644 index 0000000..6393247 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/composer.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/constructor.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/constructor.cpython-314.pyc new file mode 100644 index 0000000..f88016c Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/constructor.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/cyaml.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/cyaml.cpython-314.pyc new file mode 100644 index 0000000..436d9b2 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/cyaml.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/dumper.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/dumper.cpython-314.pyc new file mode 100644 index 0000000..2118109 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/dumper.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/emitter.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/emitter.cpython-314.pyc new file mode 100644 index 0000000..fdca2e7 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/emitter.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/error.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/error.cpython-314.pyc new file mode 100644 index 0000000..2605a6d Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/error.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/events.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/events.cpython-314.pyc new file mode 100644 index 0000000..2a47ef4 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/events.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/loader.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/loader.cpython-314.pyc new file mode 100644 index 0000000..b7b34e5 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/loader.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/nodes.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/nodes.cpython-314.pyc new file mode 100644 index 0000000..75eeda0 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/nodes.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/parser.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/parser.cpython-314.pyc new file mode 100644 index 0000000..e26548c Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/parser.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/reader.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/reader.cpython-314.pyc new file mode 100644 index 0000000..18fb9ae Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/reader.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/representer.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/representer.cpython-314.pyc new file mode 100644 index 0000000..6423a01 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/representer.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/resolver.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/resolver.cpython-314.pyc new file mode 100644 index 0000000..7d66192 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/resolver.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/scanner.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/scanner.cpython-314.pyc new file mode 100644 index 0000000..d187078 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/scanner.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/serializer.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/serializer.cpython-314.pyc new file mode 100644 index 0000000..c50ffa0 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/serializer.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/__pycache__/tokens.cpython-314.pyc b/lib/invoke/vendor/yaml/__pycache__/tokens.cpython-314.pyc new file mode 100644 index 0000000..c8f5586 Binary files /dev/null and b/lib/invoke/vendor/yaml/__pycache__/tokens.cpython-314.pyc differ diff --git a/lib/invoke/vendor/yaml/composer.py b/lib/invoke/vendor/yaml/composer.py new file mode 100644 index 0000000..6d15cb4 --- /dev/null +++ b/lib/invoke/vendor/yaml/composer.py @@ -0,0 +1,139 @@ + +__all__ = ['Composer', 'ComposerError'] + +from .error import MarkedYAMLError +from .events import * +from .nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer: + + def __init__(self): + self.anchors = {} + + def check_node(self): + # Drop the STREAM-START event. + if self.check_event(StreamStartEvent): + self.get_event() + + # If there are more documents available? + return not self.check_event(StreamEndEvent) + + def get_node(self): + # Get the root node of the next document. + if not self.check_event(StreamEndEvent): + return self.compose_document() + + def get_single_node(self): + # Drop the STREAM-START event. + self.get_event() + + # Compose a document if the stream is not empty. + document = None + if not self.check_event(StreamEndEvent): + document = self.compose_document() + + # Ensure that the stream contains no more documents. + if not self.check_event(StreamEndEvent): + event = self.get_event() + raise ComposerError("expected a single document in the stream", + document.start_mark, "but found another document", + event.start_mark) + + # Drop the STREAM-END event. + self.get_event() + + return document + + def compose_document(self): + # Drop the DOCUMENT-START event. + self.get_event() + + # Compose the root node. + node = self.compose_node(None, None) + + # Drop the DOCUMENT-END event. + self.get_event() + + self.anchors = {} + return node + + def compose_node(self, parent, index): + if self.check_event(AliasEvent): + event = self.get_event() + anchor = event.anchor + if anchor not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor, event.start_mark) + return self.anchors[anchor] + event = self.peek_event() + anchor = event.anchor + if anchor is not None: + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurrence" + % anchor, self.anchors[anchor].start_mark, + "second occurrence", event.start_mark) + self.descend_resolver(parent, index) + if self.check_event(ScalarEvent): + node = self.compose_scalar_node(anchor) + elif self.check_event(SequenceStartEvent): + node = self.compose_sequence_node(anchor) + elif self.check_event(MappingStartEvent): + node = self.compose_mapping_node(anchor) + self.ascend_resolver() + return node + + def compose_scalar_node(self, anchor): + event = self.get_event() + tag = event.tag + if tag is None or tag == '!': + tag = self.resolve(ScalarNode, event.value, event.implicit) + node = ScalarNode(tag, event.value, + event.start_mark, event.end_mark, style=event.style) + if anchor is not None: + self.anchors[anchor] = node + return node + + def compose_sequence_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == '!': + tag = self.resolve(SequenceNode, None, start_event.implicit) + node = SequenceNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + index = 0 + while not self.check_event(SequenceEndEvent): + node.value.append(self.compose_node(node, index)) + index += 1 + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + + def compose_mapping_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == '!': + tag = self.resolve(MappingNode, None, start_event.implicit) + node = MappingNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + while not self.check_event(MappingEndEvent): + #key_event = self.peek_event() + item_key = self.compose_node(node, None) + #if item_key in node.value: + # raise ComposerError("while composing a mapping", start_event.start_mark, + # "found duplicate key", key_event.start_mark) + item_value = self.compose_node(node, item_key) + #node.value[item_key] = item_value + node.value.append((item_key, item_value)) + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + diff --git a/lib/invoke/vendor/yaml/constructor.py b/lib/invoke/vendor/yaml/constructor.py new file mode 100644 index 0000000..619acd3 --- /dev/null +++ b/lib/invoke/vendor/yaml/constructor.py @@ -0,0 +1,748 @@ + +__all__ = [ + 'BaseConstructor', + 'SafeConstructor', + 'FullConstructor', + 'UnsafeConstructor', + 'Constructor', + 'ConstructorError' +] + +from .error import * +from .nodes import * + +import collections.abc, datetime, base64, binascii, re, sys, types + +class ConstructorError(MarkedYAMLError): + pass + +class BaseConstructor: + + yaml_constructors = {} + yaml_multi_constructors = {} + + def __init__(self): + self.constructed_objects = {} + self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False + + def check_data(self): + # If there are more documents available? + return self.check_node() + + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + + def get_data(self): + # Construct and return the next document. + if self.check_node(): + return self.construct_document(self.get_node()) + + def get_single_data(self): + # Ensure that the stream contains a single document and construct it. + node = self.get_single_node() + if node is not None: + return self.construct_document(node) + return None + + def construct_document(self, node): + data = self.construct_object(node) + while self.state_generators: + state_generators = self.state_generators + self.state_generators = [] + for generator in state_generators: + for dummy in generator: + pass + self.constructed_objects = {} + self.recursive_objects = {} + self.deep_construct = False + return data + + def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if node in self.recursive_objects: + raise ConstructorError(None, None, + "found unconstructable recursive node", node.start_mark) + self.recursive_objects[node] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + for tag_prefix in self.yaml_multi_constructors: + if tag_prefix is not None and node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = self.yaml_multi_constructors[tag_prefix] + break + else: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = self.__class__.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.__class__.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, types.GeneratorType): + generator = data + data = next(generator) + if self.deep_construct: + for dummy in generator: + pass + else: + self.state_generators.append(generator) + self.constructed_objects[node] = data + del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + raise ConstructorError(None, None, + "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child, deep=deep) + for child in node.value] + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, collections.abc.Hashable): + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unhashable key", key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + value = self.construct_object(value_node, deep=deep) + pairs.append((key, value)) + return pairs + + @classmethod + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + + @classmethod + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + +class SafeConstructor(BaseConstructor): + + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == 'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return super().construct_scalar(node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == 'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == 'tag:yaml.org,2002:value': + key_node.tag = 'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return super().construct_mapping(node, deep=deep) + + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + 'yes': True, + 'no': False, + 'true': True, + 'false': False, + 'on': True, + 'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_int(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + def construct_yaml_float(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_yaml_binary(self, node): + try: + value = self.construct_scalar(node).encode('ascii') + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, + node.start_mark) + try: + if hasattr(base64, 'decodebytes'): + return base64.decodebytes(value) + else: + return base64.decodestring(value) + except binascii.Error as exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + timestamp_regexp = re.compile( + r'''^(?P[0-9][0-9][0-9][0-9]) + -(?P[0-9][0-9]?) + -(?P[0-9][0-9]?) + (?:(?:[Tt]|[ \t]+) + (?P[0-9][0-9]?) + :(?P[0-9][0-9]) + :(?P[0-9][0-9]) + (?:\.(?P[0-9]*))? + (?:[ \t]*(?PZ|(?P[-+])(?P[0-9][0-9]?) + (?::(?P[0-9][0-9]))?))?)?$''', re.X) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = self.timestamp_regexp.match(node.value) + values = match.groupdict() + year = int(values['year']) + month = int(values['month']) + day = int(values['day']) + if not values['hour']: + return datetime.date(year, month, day) + hour = int(values['hour']) + minute = int(values['minute']) + second = int(values['second']) + fraction = 0 + tzinfo = None + if values['fraction']: + fraction = values['fraction'][:6] + while len(fraction) < 6: + fraction += '0' + fraction = int(fraction) + if values['tz_sign']: + tz_hour = int(values['tz_hour']) + tz_minute = int(values['tz_minute'] or 0) + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) + if values['tz_sign'] == '-': + delta = -delta + tzinfo = datetime.timezone(delta) + elif values['tz']: + tzinfo = datetime.timezone.utc + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) + + def construct_yaml_omap(self, node): + # Note: we do not check for duplicate keys, because it's too + # CPU-expensive. + omap = [] + yield omap + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + omap.append((key, value)) + + def construct_yaml_pairs(self, node): + # Note: the same code as `construct_yaml_omap`. + pairs = [] + yield pairs + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + pairs.append((key, value)) + + def construct_yaml_set(self, node): + data = set() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_object(self, node, cls): + data = cls.__new__(cls) + yield data + if hasattr(data, '__setstate__'): + state = self.construct_mapping(node, deep=True) + data.__setstate__(state) + else: + state = self.construct_mapping(node) + data.__dict__.update(state) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag, + node.start_mark) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:null', + SafeConstructor.construct_yaml_null) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:bool', + SafeConstructor.construct_yaml_bool) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:int', + SafeConstructor.construct_yaml_int) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:float', + SafeConstructor.construct_yaml_float) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:binary', + SafeConstructor.construct_yaml_binary) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:timestamp', + SafeConstructor.construct_yaml_timestamp) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:omap', + SafeConstructor.construct_yaml_omap) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:pairs', + SafeConstructor.construct_yaml_pairs) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:set', + SafeConstructor.construct_yaml_set) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:str', + SafeConstructor.construct_yaml_str) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:seq', + SafeConstructor.construct_yaml_seq) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:map', + SafeConstructor.construct_yaml_map) + +SafeConstructor.add_constructor(None, + SafeConstructor.construct_undefined) + +class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp + + def construct_python_str(self, node): + return self.construct_scalar(node) + + def construct_python_unicode(self, node): + return self.construct_scalar(node) + + def construct_python_bytes(self, node): + try: + value = self.construct_scalar(node).encode('ascii') + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, + node.start_mark) + try: + if hasattr(base64, 'decodebytes'): + return base64.decodebytes(value) + else: + return base64.decodestring(value) + except binascii.Error as exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + def construct_python_long(self, node): + return self.construct_yaml_int(node) + + def construct_python_complex(self, node): + return complex(self.construct_scalar(node)) + + def construct_python_tuple(self, node): + return tuple(self.construct_sequence(node)) + + def find_python_module(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python module", mark, + "expected non-empty name appended to the tag", mark) + if unsafe: + try: + __import__(name) + except ImportError as exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name, exc), mark) + if name not in sys.modules: + raise ConstructorError("while constructing a Python module", mark, + "module %r is not imported" % name, mark) + return sys.modules[name] + + def find_python_name(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python object", mark, + "expected non-empty name appended to the tag", mark) + if '.' in name: + module_name, object_name = name.rsplit('.', 1) + else: + module_name = 'builtins' + object_name = name + if unsafe: + try: + __import__(module_name) + except ImportError as exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name, exc), mark) + if module_name not in sys.modules: + raise ConstructorError("while constructing a Python object", mark, + "module %r is not imported" % module_name, mark) + module = sys.modules[module_name] + if not hasattr(module, object_name): + raise ConstructorError("while constructing a Python object", mark, + "cannot find %r in the module %r" + % (object_name, module.__name__), mark) + return getattr(module, object_name) + + def construct_python_name(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python name", node.start_mark, + "expected the empty value, but found %r" % value, node.start_mark) + return self.find_python_name(suffix, node.start_mark) + + def construct_python_module(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python module", node.start_mark, + "expected the empty value, but found %r" % value, node.start_mark) + return self.find_python_module(suffix, node.start_mark) + + def make_python_instance(self, suffix, node, + args=None, kwds=None, newobj=False, unsafe=False): + if not args: + args = [] + if not kwds: + kwds = {} + cls = self.find_python_name(suffix, node.start_mark) + if not (unsafe or isinstance(cls, type)): + raise ConstructorError("while constructing a Python instance", node.start_mark, + "expected a class, but found %r" % type(cls), + node.start_mark) + if newobj and isinstance(cls, type): + return cls.__new__(cls, *args, **kwds) + else: + return cls(*args, **kwds) + + def set_python_instance_state(self, instance, state, unsafe=False): + if hasattr(instance, '__setstate__'): + instance.__setstate__(state) + else: + slotstate = {} + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) + instance.__dict__.update(state) + elif state: + slotstate.update(state) + for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) + setattr(instance, key, value) + + def construct_python_object(self, suffix, node): + # Format: + # !!python/object:module.name { ... state ... } + instance = self.make_python_instance(suffix, node, newobj=True) + yield instance + deep = hasattr(instance, '__setstate__') + state = self.construct_mapping(node, deep=deep) + self.set_python_instance_state(instance, state) + + def construct_python_object_apply(self, suffix, node, newobj=False): + # Format: + # !!python/object/apply # (or !!python/object/new) + # args: [ ... arguments ... ] + # kwds: { ... keywords ... } + # state: ... state ... + # listitems: [ ... listitems ... ] + # dictitems: { ... dictitems ... } + # or short format: + # !!python/object/apply [ ... arguments ... ] + # The difference between !!python/object/apply and !!python/object/new + # is how an object is created, check make_python_instance for details. + if isinstance(node, SequenceNode): + args = self.construct_sequence(node, deep=True) + kwds = {} + state = {} + listitems = [] + dictitems = {} + else: + value = self.construct_mapping(node, deep=True) + args = value.get('args', []) + kwds = value.get('kwds', {}) + state = value.get('state', {}) + listitems = value.get('listitems', []) + dictitems = value.get('dictitems', {}) + instance = self.make_python_instance(suffix, node, args, kwds, newobj) + if state: + self.set_python_instance_state(instance, state) + if listitems: + instance.extend(listitems) + if dictitems: + for key in dictitems: + instance[key] = dictitems[key] + return instance + + def construct_python_object_new(self, suffix, node): + return self.construct_python_object_apply(suffix, node, newobj=True) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/none', + FullConstructor.construct_yaml_null) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/bool', + FullConstructor.construct_yaml_bool) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/str', + FullConstructor.construct_python_str) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/unicode', + FullConstructor.construct_python_unicode) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/bytes', + FullConstructor.construct_python_bytes) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/int', + FullConstructor.construct_yaml_int) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/long', + FullConstructor.construct_python_long) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/float', + FullConstructor.construct_yaml_float) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/complex', + FullConstructor.construct_python_complex) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/list', + FullConstructor.construct_yaml_seq) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/tuple', + FullConstructor.construct_python_tuple) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/dict', + FullConstructor.construct_yaml_map) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/name:', + FullConstructor.construct_python_name) + +class UnsafeConstructor(FullConstructor): + + def find_python_module(self, name, mark): + return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True) + + def find_python_name(self, name, mark): + return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True) + + def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False): + return super(UnsafeConstructor, self).make_python_instance( + suffix, node, args, kwds, newobj, unsafe=True) + + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/module:', + UnsafeConstructor.construct_python_module) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object:', + UnsafeConstructor.construct_python_object) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/new:', + UnsafeConstructor.construct_python_object_new) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + +# Constructor is same as UnsafeConstructor. Need to leave this in place in case +# people have extended it directly. +class Constructor(UnsafeConstructor): + pass diff --git a/lib/invoke/vendor/yaml/cyaml.py b/lib/invoke/vendor/yaml/cyaml.py new file mode 100644 index 0000000..0c21345 --- /dev/null +++ b/lib/invoke/vendor/yaml/cyaml.py @@ -0,0 +1,101 @@ + +__all__ = [ + 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper' +] + +from yaml._yaml import CParser, CEmitter + +from .constructor import * + +from .serializer import * +from .representer import * + +from .resolver import * + +class CBaseLoader(CParser, BaseConstructor, BaseResolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class CSafeLoader(CParser, SafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class CFullLoader(CParser, FullConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class CUnsafeLoader(CParser, UnsafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + UnsafeConstructor.__init__(self) + Resolver.__init__(self) + +class CLoader(CParser, Constructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + Constructor.__init__(self) + Resolver.__init__(self) + +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CSafeDumper(CEmitter, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CDumper(CEmitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/lib/invoke/vendor/yaml/dumper.py b/lib/invoke/vendor/yaml/dumper.py new file mode 100644 index 0000000..6aadba5 --- /dev/null +++ b/lib/invoke/vendor/yaml/dumper.py @@ -0,0 +1,62 @@ + +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] + +from .emitter import * +from .serializer import * +from .representer import * +from .resolver import * + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class Dumper(Emitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/lib/invoke/vendor/yaml/emitter.py b/lib/invoke/vendor/yaml/emitter.py new file mode 100644 index 0000000..a664d01 --- /dev/null +++ b/lib/invoke/vendor/yaml/emitter.py @@ -0,0 +1,1137 @@ + +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +__all__ = ['Emitter', 'EmitterError'] + +from .error import YAMLError +from .events import * + +class EmitterError(YAMLError): + pass + +class ScalarAnalysis: + def __init__(self, scalar, empty, multiline, + allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, + allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + +class Emitter: + + DEFAULT_TAG_PREFIXES = { + '!' : '!', + 'tag:yaml.org,2002:' : '!!', + } + + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overridden by STREAM-START. + self.encoding = None + + # Emitter is a state machine with a stack of states to handle nested + # structures. + self.states = [] + self.state = self.expect_stream_start + + # Current event and the event queue. + self.events = [] + self.event = None + + # The current indentation level and the stack of previous indents. + self.indents = [] + self.indent = None + + # Flow level. + self.flow_level = 0 + + # Contexts. + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + + # Whether the document requires an explicit document indicator + self.open_ended = False + + # Formatting details. + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width + self.best_line_break = '\n' + if line_break in ['\r', '\n', '\r\n']: + self.best_line_break = line_break + + # Tag prefixes. + self.tag_prefixes = None + + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def emit(self, event): + self.events.append(event) + while not self.need_more_events(): + self.event = self.events.pop(0) + self.state() + self.event = None + + # In some cases, we wait for a few next events before emitting. + + def need_more_events(self): + if not self.events: + return True + event = self.events[0] + if isinstance(event, DocumentStartEvent): + return self.need_events(1) + elif isinstance(event, SequenceStartEvent): + return self.need_events(2) + elif isinstance(event, MappingStartEvent): + return self.need_events(3) + else: + return False + + def need_events(self, count): + level = 0 + for event in self.events[1:]: + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)): + level += 1 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)): + level -= 1 + elif isinstance(event, StreamEndEvent): + level = -1 + if level < 0: + return False + return (len(self.events) < count+1) + + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + # States. + + # Stream handlers. + + def expect_stream_start(self): + if isinstance(self.event, StreamStartEvent): + if self.event.encoding and not hasattr(self.stream, 'encoding'): + self.encoding = self.event.encoding + self.write_stream_start() + self.state = self.expect_first_document_start + else: + raise EmitterError("expected StreamStartEvent, but got %s" + % self.event) + + def expect_nothing(self): + raise EmitterError("expected nothing, but got %s" % self.event) + + # Document handlers. + + def expect_first_document_start(self): + return self.expect_document_start(first=True) + + def expect_document_start(self, first=False): + if isinstance(self.event, DocumentStartEvent): + if (self.event.version or self.event.tags) and self.open_ended: + self.write_indicator('...', True) + self.write_indent() + if self.event.version: + version_text = self.prepare_version(self.event.version) + self.write_version_directive(version_text) + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() + if self.event.tags: + handles = sorted(self.event.tags.keys()) + for handle in handles: + prefix = self.event.tags[handle] + self.tag_prefixes[prefix] = handle + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) + self.write_tag_directive(handle_text, prefix_text) + implicit = (first and not self.event.explicit and not self.canonical + and not self.event.version and not self.event.tags + and not self.check_empty_document()) + if not implicit: + self.write_indent() + self.write_indicator('---', True) + if self.canonical: + self.write_indent() + self.state = self.expect_document_root + elif isinstance(self.event, StreamEndEvent): + if self.open_ended: + self.write_indicator('...', True) + self.write_indent() + self.write_stream_end() + self.state = self.expect_nothing + else: + raise EmitterError("expected DocumentStartEvent, but got %s" + % self.event) + + def expect_document_end(self): + if isinstance(self.event, DocumentEndEvent): + self.write_indent() + if self.event.explicit: + self.write_indicator('...', True) + self.write_indent() + self.flush_stream() + self.state = self.expect_document_start + else: + raise EmitterError("expected DocumentEndEvent, but got %s" + % self.event) + + def expect_document_root(self): + self.states.append(self.expect_document_end) + self.expect_node(root=True) + + # Node handlers. + + def expect_node(self, root=False, sequence=False, mapping=False, + simple_key=False): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + if isinstance(self.event, AliasEvent): + self.expect_alias() + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)): + self.process_anchor('&') + self.process_tag() + if isinstance(self.event, ScalarEvent): + self.expect_scalar() + elif isinstance(self.event, SequenceStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_sequence(): + self.expect_flow_sequence() + else: + self.expect_block_sequence() + elif isinstance(self.event, MappingStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_mapping(): + self.expect_flow_mapping() + else: + self.expect_block_mapping() + else: + raise EmitterError("expected NodeEvent, but got %s" % self.event) + + def expect_alias(self): + if self.event.anchor is None: + raise EmitterError("anchor is not specified for alias") + self.process_anchor('*') + self.state = self.states.pop() + + def expect_scalar(self): + self.increase_indent(flow=True) + self.process_scalar() + self.indent = self.indents.pop() + self.state = self.states.pop() + + # Flow sequence handlers. + + def expect_flow_sequence(self): + self.write_indicator('[', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_sequence_item + + def expect_first_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(']', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + def expect_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(',', False) + self.write_indent() + self.write_indicator(']', False) + self.state = self.states.pop() + else: + self.write_indicator(',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + # Flow mapping handlers. + + def expect_flow_mapping(self): + self.write_indicator('{', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_mapping_key + + def expect_first_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator('}', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(',', False) + self.write_indent() + self.write_indicator('}', False) + self.state = self.states.pop() + else: + self.write_indicator(',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_simple_value(self): + self.write_indicator(':', False) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + def expect_flow_mapping_value(self): + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(':', True) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + # Block sequence handlers. + + def expect_block_sequence(self): + indentless = (self.mapping_context and not self.indention) + self.increase_indent(flow=False, indentless=indentless) + self.state = self.expect_first_block_sequence_item + + def expect_first_block_sequence_item(self): + return self.expect_block_sequence_item(first=True) + + def expect_block_sequence_item(self, first=False): + if not first and isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + self.write_indicator('-', True, indention=True) + self.states.append(self.expect_block_sequence_item) + self.expect_node(sequence=True) + + # Block mapping handlers. + + def expect_block_mapping(self): + self.increase_indent(flow=False) + self.state = self.expect_first_block_mapping_key + + def expect_first_block_mapping_key(self): + return self.expect_block_mapping_key(first=True) + + def expect_block_mapping_key(self, first=False): + if not first and isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + if self.check_simple_key(): + self.states.append(self.expect_block_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True, indention=True) + self.states.append(self.expect_block_mapping_value) + self.expect_node(mapping=True) + + def expect_block_mapping_simple_value(self): + self.write_indicator(':', False) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + def expect_block_mapping_value(self): + self.write_indent() + self.write_indicator(':', True, indention=True) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + # Checkers. + + def check_empty_sequence(self): + return (isinstance(self.event, SequenceStartEvent) and self.events + and isinstance(self.events[0], SequenceEndEvent)) + + def check_empty_mapping(self): + return (isinstance(self.event, MappingStartEvent) and self.events + and isinstance(self.events[0], MappingEndEvent)) + + def check_empty_document(self): + if not isinstance(self.event, DocumentStartEvent) or not self.events: + return False + event = self.events[0] + return (isinstance(event, ScalarEvent) and event.anchor is None + and event.tag is None and event.implicit and event.value == '') + + def check_simple_key(self): + length = 0 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None: + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ + and self.event.tag is not None: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) + if isinstance(self.event, ScalarEvent): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) + return (length < 128 and (isinstance(self.event, AliasEvent) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) + or self.check_empty_sequence() or self.check_empty_mapping())) + + # Anchor, Tag, and Scalar processors. + + def process_anchor(self, indicator): + if self.event.anchor is None: + self.prepared_anchor = None + return + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None + + def process_tag(self): + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if ((not self.canonical or tag is None) and + ((self.style == '' and self.event.implicit[0]) + or (self.style != '' and self.event.implicit[1]))): + self.prepared_tag = None + return + if self.event.implicit[0] and tag is None: + tag = '!' + self.prepared_tag = None + else: + if (not self.canonical or tag is None) and self.event.implicit: + self.prepared_tag = None + return + if tag is None: + raise EmitterError("tag is not specified") + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: + return '"' + if not self.event.style and self.event.implicit[0]: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if (not self.flow_level and not self.simple_key_context + and self.analysis.allow_block): + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' + + def process_scalar(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None + + # Analyzers. + + def prepare_version(self, version): + major, minor = version + if major != 1: + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor)) + return '%d.%d' % (major, minor) + + def prepare_tag_handle(self, handle): + if not handle: + raise EmitterError("tag handle must not be empty") + if handle[0] != '!' or handle[-1] != '!': + raise EmitterError("tag handle must start and end with '!': %r" % handle) + for ch in handle[1:-1]: + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_'): + raise EmitterError("invalid character %r in the tag handle: %r" + % (ch, handle)) + return handle + + def prepare_tag_prefix(self, prefix): + if not prefix: + raise EmitterError("tag prefix must not be empty") + chunks = [] + start = end = 0 + if prefix[0] == '!': + end = 1 + while end < len(prefix): + ch = prefix[end] + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?!:@&=+$,_.~*\'()[]': + end += 1 + else: + if start < end: + chunks.append(prefix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append('%%%02X' % ord(ch)) + if start < end: + chunks.append(prefix[start:end]) + return ''.join(chunks) + + def prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == '!': + return tag + handle = None + suffix = tag + prefixes = sorted(self.tag_prefixes.keys()) + for prefix in prefixes: + if tag.startswith(prefix) \ + and (prefix == '!' or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + start = end = 0 + while end < len(suffix): + ch = suffix[end] + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?:@&=+$,_.~*\'()[]' \ + or (ch == '!' and handle != '!'): + end += 1 + else: + if start < end: + chunks.append(suffix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append('%%%02X' % ch) + if start < end: + chunks.append(suffix[start:end]) + suffix_text = ''.join(chunks) + if handle: + return '%s%s' % (handle, suffix_text) + else: + return '!<%s>' % suffix_text + + def prepare_anchor(self, anchor): + if not anchor: + raise EmitterError("anchor must not be empty") + for ch in anchor: + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_'): + raise EmitterError("invalid character %r in the anchor: %r" + % (ch, anchor)) + return anchor + + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. + if not scalar: + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, + allow_flow_plain=False, allow_block_plain=True, + allow_single_quoted=True, allow_double_quoted=True, + allow_block=False) + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Important whitespace combinations. + leading_space = False + leading_break = False + trailing_space = False + trailing_break = False + break_space = False + space_break = False + + # Check document indicators. + if scalar.startswith('---') or scalar.startswith('...'): + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceded_by_whitespace = True + + # Last character or followed by a whitespace. + followed_by_whitespace = (len(scalar) == 1 or + scalar[1] in '\0 \t\r\n\x85\u2028\u2029') + + # The previous character is a space. + previous_space = False + + # The previous character is a break. + previous_break = False + + index = 0 + while index < len(scalar): + ch = scalar[index] + + # Check for indicators. + if index == 0: + # Leading indicators are special characters. + if ch in '#,[]{}&*!|>\'\"%@`': + flow_indicators = True + block_indicators = True + if ch in '?:': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == '-' and followed_by_whitespace: + flow_indicators = True + block_indicators = True + else: + # Some indicators cannot appear within a scalar as well. + if ch in ',?[]{}': + flow_indicators = True + if ch == ':': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == '#' and preceded_by_whitespace: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + if ch in '\n\x85\u2028\u2029': + line_breaks = True + if not (ch == '\n' or '\x20' <= ch <= '\x7E'): + if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF' + or '\uE000' <= ch <= '\uFFFD' + or '\U00010000' <= ch < '\U0010ffff') and ch != '\uFEFF': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + + # Detect important whitespace combinations. + if ch == ' ': + if index == 0: + leading_space = True + if index == len(scalar)-1: + trailing_space = True + if previous_break: + break_space = True + previous_space = True + previous_break = False + elif ch in '\n\x85\u2028\u2029': + if index == 0: + leading_break = True + if index == len(scalar)-1: + trailing_break = True + if previous_space: + space_break = True + previous_space = False + previous_break = True + else: + previous_space = False + previous_break = False + + # Prepare for the next character. + index += 1 + preceded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029') + followed_by_whitespace = (index+1 >= len(scalar) or + scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029') + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + + # Leading and trailing whitespaces are bad for plain scalars. + if (leading_space or leading_break + or trailing_space or trailing_break): + allow_flow_plain = allow_block_plain = False + + # We do not permit trailing spaces for block scalars. + if trailing_space: + allow_block = False + + # Spaces at the beginning of a new line are only acceptable for block + # scalars. + if break_space: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Spaces followed by breaks, as well as special character are only + # allowed for double quoted scalars. + if space_break or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # Although the plain scalar writer supports breaks, we never emit + # multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, + allow_block=allow_block) + + # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() + + def write_stream_start(self): + # Write BOM if needed. + if self.encoding and self.encoding.startswith('utf-16'): + self.stream.write('\uFEFF'.encode(self.encoding)) + + def write_stream_end(self): + self.flush_stream() + + def write_indicator(self, indicator, need_whitespace, + whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = ' '+indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + self.open_ended = False + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indent(self): + indent = self.indent or 0 + if not self.indention or self.column > indent \ + or (self.column == indent and not self.whitespace): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = ' '*(indent-self.column) + self.column = indent + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_version_directive(self, version_text): + data = '%%YAML %s' % version_text + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + def write_tag_directive(self, handle_text, prefix_text): + data = '%%TAG %s %s' % (handle_text, prefix_text) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + # Scalar streams. + + def write_single_quoted(self, text, split=True): + self.write_indicator('\'', True) + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch is None or ch != ' ': + if start+1 == end and self.column > self.best_width and split \ + and start != 0 and end != len(text): + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + if text[start] == '\n': + self.write_line_break() + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'': + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch == '\'': + data = '\'\'' + self.column += 2 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + 1 + if ch is not None: + spaces = (ch == ' ') + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 + self.write_indicator('\'', False) + + ESCAPE_REPLACEMENTS = { + '\0': '0', + '\x07': 'a', + '\x08': 'b', + '\x09': 't', + '\x0A': 'n', + '\x0B': 'v', + '\x0C': 'f', + '\x0D': 'r', + '\x1B': 'e', + '\"': '\"', + '\\': '\\', + '\x85': 'N', + '\xA0': '_', + '\u2028': 'L', + '\u2029': 'P', + } + + def write_double_quoted(self, text, split=True): + self.write_indicator('"', True) + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \ + or not ('\x20' <= ch <= '\x7E' + or (self.allow_unicode + and ('\xA0' <= ch <= '\uD7FF' + or '\uE000' <= ch <= '\uFFFD'))): + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + if ch in self.ESCAPE_REPLACEMENTS: + data = '\\'+self.ESCAPE_REPLACEMENTS[ch] + elif ch <= '\xFF': + data = '\\x%02X' % ord(ch) + elif ch <= '\uFFFF': + data = '\\u%04X' % ord(ch) + else: + data = '\\U%08X' % ord(ch) + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end+1 + if 0 < end < len(text)-1 and (ch == ' ' or start >= end) \ + and self.column+(end-start) > self.best_width and split: + data = text[start:end]+'\\' + if start < end: + start = end + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == ' ': + data = '\\' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + end += 1 + self.write_indicator('"', False) + + def determine_block_hints(self, text): + hints = '' + if text: + if text[0] in ' \n\x85\u2028\u2029': + hints += str(self.best_indent) + if text[-1] not in '\n\x85\u2028\u2029': + hints += '-' + elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029': + hints += '+' + return hints + + def write_folded(self, text): + hints = self.determine_block_hints(text) + self.write_indicator('>'+hints, True) + if hints[-1:] == '+': + self.open_ended = True + self.write_line_break() + leading_space = True + spaces = False + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + if not leading_space and ch is not None and ch != ' ' \ + and text[start] == '\n': + self.write_line_break() + leading_space = (ch == ' ') + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != ' ': + if start+1 == end and self.column > self.best_width: + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in '\n\x85\u2028\u2029') + spaces = (ch == ' ') + end += 1 + + def write_literal(self, text): + hints = self.determine_block_hints(text) + self.write_indicator('|'+hints, True) + if hints[-1:] == '+': + self.open_ended = True + self.write_line_break() + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in '\n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 + + def write_plain(self, text, split=True): + if self.root_context: + self.open_ended = True + if not text: + return + if not self.whitespace: + data = ' ' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.whitespace = False + self.indention = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch != ' ': + if start+1 == end and self.column > self.best_width and split: + self.write_indent() + self.whitespace = False + self.indention = False + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch not in '\n\x85\u2028\u2029': + if text[start] == '\n': + self.write_line_break() + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + spaces = (ch == ' ') + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 diff --git a/lib/invoke/vendor/yaml/error.py b/lib/invoke/vendor/yaml/error.py new file mode 100644 index 0000000..b796b4d --- /dev/null +++ b/lib/invoke/vendor/yaml/error.py @@ -0,0 +1,75 @@ + +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] + +class Mark: + + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + head = '' + start = self.pointer + while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029': + start -= 1 + if self.pointer-start > max_length/2-1: + head = ' ... ' + start += 5 + break + tail = '' + end = self.pointer + while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029': + end += 1 + if end-self.pointer > max_length/2-1: + tail = ' ... ' + end -= 5 + break + snippet = self.buffer[start:end] + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where + +class YAMLError(Exception): + pass + +class MarkedYAMLError(YAMLError): + + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None): + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return '\n'.join(lines) + diff --git a/lib/invoke/vendor/yaml/events.py b/lib/invoke/vendor/yaml/events.py new file mode 100644 index 0000000..f79ad38 --- /dev/null +++ b/lib/invoke/vendor/yaml/events.py @@ -0,0 +1,86 @@ + +# Abstract classes. + +class Event(object): + def __init__(self, start_mark=None, end_mark=None): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value'] + if hasattr(self, key)] + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +class NodeEvent(Event): + def __init__(self, anchor, start_mark=None, end_mark=None): + self.anchor = anchor + self.start_mark = start_mark + self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + +class AliasEvent(NodeEvent): + pass + +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, value, + start_mark=None, end_mark=None, style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class SequenceStartEvent(CollectionStartEvent): + pass + +class SequenceEndEvent(CollectionEndEvent): + pass + +class MappingStartEvent(CollectionStartEvent): + pass + +class MappingEndEvent(CollectionEndEvent): + pass + diff --git a/lib/invoke/vendor/yaml/loader.py b/lib/invoke/vendor/yaml/loader.py new file mode 100644 index 0000000..e90c112 --- /dev/null +++ b/lib/invoke/vendor/yaml/loader.py @@ -0,0 +1,63 @@ + +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] + +from .reader import * +from .scanner import * +from .parser import * +from .composer import * +from .constructor import * +from .resolver import * + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) + +# UnsafeLoader is the same as Loader (which is and was always unsafe on +# untrusted input). Use of either Loader or UnsafeLoader should be rare, since +# FullLoad should be able to load almost all YAML safely. Loader is left intact +# to ensure backwards compatibility. +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) diff --git a/lib/invoke/vendor/yaml/nodes.py b/lib/invoke/vendor/yaml/nodes.py new file mode 100644 index 0000000..c4f070c --- /dev/null +++ b/lib/invoke/vendor/yaml/nodes.py @@ -0,0 +1,49 @@ + +class Node(object): + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + value = self.value + #if isinstance(value, list): + # if len(value) == 0: + # value = '' + # elif len(value) == 1: + # value = '<1 item>' + # else: + # value = '<%d items>' % len(value) + #else: + # if len(value) > 75: + # value = repr(value[:70]+u' ... ') + # else: + # value = repr(value) + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + id = 'scalar' + def __init__(self, tag, value, + start_mark=None, end_mark=None, style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class CollectionNode(Node): + def __init__(self, tag, value, + start_mark=None, end_mark=None, flow_style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class SequenceNode(CollectionNode): + id = 'sequence' + +class MappingNode(CollectionNode): + id = 'mapping' + diff --git a/lib/invoke/vendor/yaml/parser.py b/lib/invoke/vendor/yaml/parser.py new file mode 100644 index 0000000..13a5995 --- /dev/null +++ b/lib/invoke/vendor/yaml/parser.py @@ -0,0 +1,589 @@ + +# The following YAML grammar is LL(1) and is parsed by a recursive descent +# parser. +# +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# implicit_document ::= block_node DOCUMENT-END* +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +# block_node_or_indentless_sequence ::= +# ALIAS +# | properties (block_content | indentless_block_sequence)? +# | block_content +# | indentless_block_sequence +# block_node ::= ALIAS +# | properties block_content? +# | block_content +# flow_node ::= ALIAS +# | properties flow_content? +# | flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# flow_collection ::= flow_sequence | flow_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +# block_mapping ::= BLOCK-MAPPING_START +# ((KEY block_node_or_indentless_sequence?)? +# (VALUE block_node_or_indentless_sequence?)?)* +# BLOCK-END +# flow_sequence ::= FLOW-SEQUENCE-START +# (flow_sequence_entry FLOW-ENTRY)* +# flow_sequence_entry? +# FLOW-SEQUENCE-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping ::= FLOW-MAPPING-START +# (flow_mapping_entry FLOW-ENTRY)* +# flow_mapping_entry? +# FLOW-MAPPING-END +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# +# FIRST sets: +# +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +__all__ = ['Parser', 'ParserError'] + +from .error import MarkedYAMLError +from .tokens import * +from .events import * +from .scanner import * + +class ParserError(MarkedYAMLError): + pass + +class Parser: + # Since writing a recursive-descendant parser is a straightforward task, we + # do not give many comments here. + + DEFAULT_TAGS = { + '!': '!', + '!!': 'tag:yaml.org,2002:', + } + + def __init__(self): + self.current_event = None + self.yaml_version = None + self.tag_handles = {} + self.states = [] + self.marks = [] + self.state = self.parse_stream_start + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def check_event(self, *choices): + # Check the type of the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + if self.current_event is not None: + if not choices: + return True + for choice in choices: + if isinstance(self.current_event, choice): + return True + return False + + def peek_event(self): + # Get the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + return self.current_event + + def get_event(self): + # Get the next event and proceed further. + if self.current_event is None: + if self.state: + self.current_event = self.state() + value = self.current_event + self.current_event = None + return value + + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END + # implicit_document ::= block_node DOCUMENT-END* + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + + def parse_stream_start(self): + + # Parse the stream start. + token = self.get_token() + event = StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) + + # Prepare the next state. + self.state = self.parse_implicit_document_start + + return event + + def parse_implicit_document_start(self): + + # Parse an implicit document. + if not self.check_token(DirectiveToken, DocumentStartToken, + StreamEndToken): + self.tag_handles = self.DEFAULT_TAGS + token = self.peek_token() + start_mark = end_mark = token.start_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=False) + + # Prepare the next state. + self.states.append(self.parse_document_end) + self.state = self.parse_block_node + + return event + + else: + return self.parse_document_start() + + def parse_document_start(self): + + # Parse any extra document end indicators. + while self.check_token(DocumentEndToken): + self.get_token() + + # Parse an explicit document. + if not self.check_token(StreamEndToken): + token = self.peek_token() + start_mark = token.start_mark + version, tags = self.process_directives() + if not self.check_token(DocumentStartToken): + raise ParserError(None, None, + "expected '', but found %r" + % self.peek_token().id, + self.peek_token().start_mark) + token = self.get_token() + end_mark = token.end_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=True, version=version, tags=tags) + self.states.append(self.parse_document_end) + self.state = self.parse_document_content + else: + # Parse the end of the stream. + token = self.get_token() + event = StreamEndEvent(token.start_mark, token.end_mark) + assert not self.states + assert not self.marks + self.state = None + return event + + def parse_document_end(self): + + # Parse the document end. + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + if self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit = True + event = DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Prepare the next state. + self.state = self.parse_document_start + + return event + + def parse_document_content(self): + if self.check_token(DirectiveToken, + DocumentStartToken, DocumentEndToken, StreamEndToken): + event = self.process_empty_scalar(self.peek_token().start_mark) + self.state = self.states.pop() + return event + else: + return self.parse_block_node() + + def process_directives(self): + self.yaml_version = None + self.tag_handles = {} + while self.check_token(DirectiveToken): + token = self.get_token() + if token.name == 'YAML': + if self.yaml_version is not None: + raise ParserError(None, None, + "found duplicate YAML directive", token.start_mark) + major, minor = token.value + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value + elif token.name == 'TAG': + handle, prefix = token.value + if handle in self.tag_handles: + raise ParserError(None, None, + "duplicate tag handle %r" % handle, + token.start_mark) + self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None + for key in self.DEFAULT_TAGS: + if key not in self.tag_handles: + self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value + + # block_node_or_indentless_sequence ::= ALIAS + # | properties (block_content | indentless_block_sequence)? + # | block_content + # | indentless_block_sequence + # block_node ::= ALIAS + # | properties block_content? + # | block_content + # flow_node ::= ALIAS + # | properties flow_content? + # | flow_content + # properties ::= TAG ANCHOR? | ANCHOR TAG? + # block_content ::= block_collection | flow_collection | SCALAR + # flow_content ::= flow_collection | SCALAR + # block_collection ::= block_sequence | block_mapping + # flow_collection ::= flow_sequence | flow_mapping + + def parse_block_node(self): + return self.parse_node(block=True) + + def parse_flow_node(self): + return self.parse_node() + + def parse_block_node_or_indentless_sequence(self): + return self.parse_node(block=True, indentless_sequence=True) + + def parse_node(self, block=False, indentless_sequence=False): + if self.check_token(AliasToken): + token = self.get_token() + event = AliasEvent(token.value, token.start_mark, token.end_mark) + self.state = self.states.pop() + else: + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self.check_token(AnchorToken): + token = self.get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self.check_token(TagToken): + token = self.get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self.check_token(TagToken): + token = self.get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self.check_token(AnchorToken): + token = self.get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None: + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle, + tag_mark) + tag = self.tag_handles[handle]+suffix + else: + tag = suffix + #if tag == '!': + # raise ParserError("while parsing a node", start_mark, + # "found non-specific tag '!'", tag_mark, + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.") + if start_mark is None: + start_mark = end_mark = self.peek_token().start_mark + event = None + implicit = (tag is None or tag == '!') + if indentless_sequence and self.check_token(BlockEntryToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark) + self.state = self.parse_indentless_sequence_entry + else: + if self.check_token(ScalarToken): + token = self.get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == '!': + implicit = (True, False) + elif tag is None: + implicit = (False, True) + else: + implicit = (False, False) + event = ScalarEvent(anchor, tag, implicit, token.value, + start_mark, end_mark, style=token.style) + self.state = self.states.pop() + elif self.check_token(FlowSequenceStartToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_sequence_first_entry + elif self.check_token(FlowMappingStartToken): + end_mark = self.peek_token().end_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_mapping_first_key + elif block and self.check_token(BlockSequenceStartToken): + end_mark = self.peek_token().start_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_sequence_first_entry + elif block and self.check_token(BlockMappingStartToken): + end_mark = self.peek_token().start_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_mapping_first_key + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, (implicit, False), '', + start_mark, end_mark) + self.state = self.states.pop() + else: + if block: + node = 'block' + else: + node = 'flow' + token = self.peek_token() + raise ParserError("while parsing a %s node" % node, start_mark, + "expected the node content, but found %r" % token.id, + token.start_mark) + return event + + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + + def parse_block_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_sequence_entry() + + def parse_block_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, BlockEndToken): + self.states.append(self.parse_block_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_block_sequence_entry + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block collection", self.marks[-1], + "expected , but found %r" % token.id, token.start_mark) + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+ + + def parse_indentless_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, + KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_indentless_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_indentless_sequence_entry + return self.process_empty_scalar(token.end_mark) + token = self.peek_token() + event = SequenceEndEvent(token.start_mark, token.start_mark) + self.state = self.states.pop() + return event + + # block_mapping ::= BLOCK-MAPPING_START + # ((KEY block_node_or_indentless_sequence?)? + # (VALUE block_node_or_indentless_sequence?)?)* + # BLOCK-END + + def parse_block_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_mapping_key() + + def parse_block_mapping_key(self): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_value) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_value + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block mapping", self.marks[-1], + "expected , but found %r" % token.id, token.start_mark) + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_block_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_key) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_block_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + # flow_sequence ::= FLOW-SEQUENCE-START + # (flow_sequence_entry FLOW-ENTRY)* + # flow_sequence_entry? + # FLOW-SEQUENCE-END + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + # + # Note that while production rules for both flow_sequence_entry and + # flow_mapping_entry are equal, their interpretations are different. + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?` + # generate an inline mapping (set syntax). + + def parse_flow_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_sequence_entry(first=True) + + def parse_flow_sequence_entry(self, first=False): + if not self.check_token(FlowSequenceEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow sequence", self.marks[-1], + "expected ',' or ']', but got %r" % token.id, token.start_mark) + + if self.check_token(KeyToken): + token = self.peek_token() + event = MappingStartEvent(None, None, True, + token.start_mark, token.end_mark, + flow_style=True) + self.state = self.parse_flow_sequence_entry_mapping_key + return event + elif not self.check_token(FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry) + return self.parse_flow_node() + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_sequence_entry_mapping_key(self): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_value + return self.process_empty_scalar(token.end_mark) + + def parse_flow_sequence_entry_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_end) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_end + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_sequence_entry_mapping_end + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_sequence_entry_mapping_end(self): + self.state = self.parse_flow_sequence_entry + token = self.peek_token() + return MappingEndEvent(token.start_mark, token.start_mark) + + # flow_mapping ::= FLOW-MAPPING-START + # (flow_mapping_entry FLOW-ENTRY)* + # flow_mapping_entry? + # FLOW-MAPPING-END + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + + def parse_flow_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_mapping_key(first=True) + + def parse_flow_mapping_key(self, first=False): + if not self.check_token(FlowMappingEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow mapping", self.marks[-1], + "expected ',' or '}', but got %r" % token.id, token.start_mark) + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_value + return self.process_empty_scalar(token.end_mark) + elif not self.check_token(FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_empty_value) + return self.parse_flow_node() + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_key) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_mapping_empty_value(self): + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(self.peek_token().start_mark) + + def process_empty_scalar(self, mark): + return ScalarEvent(None, None, (True, False), '', mark, mark) + diff --git a/lib/invoke/vendor/yaml/reader.py b/lib/invoke/vendor/yaml/reader.py new file mode 100644 index 0000000..774b021 --- /dev/null +++ b/lib/invoke/vendor/yaml/reader.py @@ -0,0 +1,185 @@ +# This module contains abstractions for the input stream. You don't have to +# looks further, there are no pretty code. +# +# We define two classes here. +# +# Mark(source, line, column) +# It's just a record and its only use is producing nice error messages. +# Parser does not use it for any other purposes. +# +# Reader(source, data) +# Reader determines the encoding of `data` and converts it to unicode. +# Reader provides the following methods and attributes: +# reader.peek(length=1) - return the next `length` characters +# reader.forward(length=1) - move the current position to `length` characters. +# reader.index - the number of the current character. +# reader.line, stream.column - the line and the column of the current character. + +__all__ = ['Reader', 'ReaderError'] + +from .error import YAMLError, Mark + +import codecs, re + +class ReaderError(YAMLError): + + def __init__(self, name, position, character, encoding, reason): + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, bytes): + return "'%s' codec can't decode byte #x%02x: %s\n" \ + " in \"%s\", position %d" \ + % (self.encoding, ord(self.character), self.reason, + self.name, self.position) + else: + return "unacceptable character #x%04x: %s\n" \ + " in \"%s\", position %d" \ + % (self.character, self.reason, + self.name, self.position) + +class Reader(object): + # Reader: + # - determines the data encoding and converts it to a unicode string, + # - checks if characters are in allowed range, + # - adds '\0' to the end. + + # Reader accepts + # - a `bytes` object, + # - a `str` object, + # - a file-like object with its `read` method returning `str`, + # - a file-like object with its `read` method returning `unicode`. + + # Yeah, it's ugly and slow. + + def __init__(self, stream): + self.name = None + self.stream = None + self.stream_pointer = 0 + self.eof = True + self.buffer = '' + self.pointer = 0 + self.raw_buffer = None + self.raw_decode = None + self.encoding = None + self.index = 0 + self.line = 0 + self.column = 0 + if isinstance(stream, str): + self.name = "" + self.check_printable(stream) + self.buffer = stream+'\0' + elif isinstance(stream, bytes): + self.name = "" + self.raw_buffer = stream + self.determine_encoding() + else: + self.stream = stream + self.name = getattr(stream, 'name', "") + self.eof = False + self.raw_buffer = None + self.determine_encoding() + + def peek(self, index=0): + try: + return self.buffer[self.pointer+index] + except IndexError: + self.update(index+1) + return self.buffer[self.pointer+index] + + def prefix(self, length=1): + if self.pointer+length >= len(self.buffer): + self.update(length) + return self.buffer[self.pointer:self.pointer+length] + + def forward(self, length=1): + if self.pointer+length+1 >= len(self.buffer): + self.update(length+1) + while length: + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in '\n\x85\u2028\u2029' \ + or (ch == '\r' and self.buffer[self.pointer] != '\n'): + self.line += 1 + self.column = 0 + elif ch != '\uFEFF': + self.column += 1 + length -= 1 + + def get_mark(self): + if self.stream is None: + return Mark(self.name, self.index, self.line, self.column, + self.buffer, self.pointer) + else: + return Mark(self.name, self.index, self.line, self.column, + None, None) + + def determine_encoding(self): + while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2): + self.update_raw() + if isinstance(self.raw_buffer, bytes): + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): + self.raw_decode = codecs.utf_16_le_decode + self.encoding = 'utf-16-le' + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): + self.raw_decode = codecs.utf_16_be_decode + self.encoding = 'utf-16-be' + else: + self.raw_decode = codecs.utf_8_decode + self.encoding = 'utf-8' + self.update(1) + + NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') + def check_printable(self, data): + match = self.NON_PRINTABLE.search(data) + if match: + character = match.group() + position = self.index+(len(self.buffer)-self.pointer)+match.start() + raise ReaderError(self.name, position, ord(character), + 'unicode', "special characters are not allowed") + + def update(self, length): + if self.raw_buffer is None: + return + self.buffer = self.buffer[self.pointer:] + self.pointer = 0 + while len(self.buffer) < length: + if not self.eof: + self.update_raw() + if self.raw_decode is not None: + try: + data, converted = self.raw_decode(self.raw_buffer, + 'strict', self.eof) + except UnicodeDecodeError as exc: + character = self.raw_buffer[exc.start] + if self.stream is not None: + position = self.stream_pointer-len(self.raw_buffer)+exc.start + else: + position = exc.start + raise ReaderError(self.name, position, character, + exc.encoding, exc.reason) + else: + data = self.raw_buffer + converted = len(data) + self.check_printable(data) + self.buffer += data + self.raw_buffer = self.raw_buffer[converted:] + if self.eof: + self.buffer += '\0' + self.raw_buffer = None + break + + def update_raw(self, size=4096): + data = self.stream.read(size) + if self.raw_buffer is None: + self.raw_buffer = data + else: + self.raw_buffer += data + self.stream_pointer += len(data) + if not data: + self.eof = True diff --git a/lib/invoke/vendor/yaml/representer.py b/lib/invoke/vendor/yaml/representer.py new file mode 100644 index 0000000..3b0b192 --- /dev/null +++ b/lib/invoke/vendor/yaml/representer.py @@ -0,0 +1,389 @@ + +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError'] + +from .error import * +from .nodes import * + +import datetime, copyreg, types, base64, collections + +class RepresenterError(YAMLError): + pass + +class BaseRepresenter: + + yaml_representers = {} + yaml_multi_representers = {} + + def __init__(self, default_style=None, default_flow_style=False, sort_keys=True): + self.default_style = default_style + self.sort_keys = sort_keys + self.default_flow_style = default_flow_style + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent_data(self, data): + if self.ignore_aliases(data): + self.alias_key = None + else: + self.alias_key = id(data) + if self.alias_key is not None: + if self.alias_key in self.represented_objects: + node = self.represented_objects[self.alias_key] + #if node is None: + # raise RepresenterError("recursive objects are not allowed: %r" % data) + return node + #self.represented_objects[alias_key] = None + self.object_keeper.append(data) + data_types = type(data).__mro__ + if data_types[0] in self.yaml_representers: + node = self.yaml_representers[data_types[0]](self, data) + else: + for data_type in data_types: + if data_type in self.yaml_multi_representers: + node = self.yaml_multi_representers[data_type](self, data) + break + else: + if None in self.yaml_multi_representers: + node = self.yaml_multi_representers[None](self, data) + elif None in self.yaml_representers: + node = self.yaml_representers[None](self, data) + else: + node = ScalarNode(None, str(data)) + #if alias_key is not None: + # self.represented_objects[alias_key] = node + return node + + @classmethod + def add_representer(cls, data_type, representer): + if not 'yaml_representers' in cls.__dict__: + cls.yaml_representers = cls.yaml_representers.copy() + cls.yaml_representers[data_type] = representer + + @classmethod + def add_multi_representer(cls, data_type, representer): + if not 'yaml_multi_representers' in cls.__dict__: + cls.yaml_multi_representers = cls.yaml_multi_representers.copy() + cls.yaml_multi_representers[data_type] = representer + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + node = ScalarNode(tag, value, style=style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + def represent_sequence(self, tag, sequence, flow_style=None): + value = [] + node = SequenceNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(node_item) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + mapping = list(mapping.items()) + if self.sort_keys: + try: + mapping = sorted(mapping) + except TypeError: + pass + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def ignore_aliases(self, data): + return False + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True + + def represent_none(self, data): + return self.represent_scalar('tag:yaml.org,2002:null', 'null') + + def represent_str(self, data): + return self.represent_scalar('tag:yaml.org,2002:str', data) + + def represent_binary(self, data): + if hasattr(base64, 'encodebytes'): + data = base64.encodebytes(data).decode('ascii') + else: + data = base64.encodestring(data).decode('ascii') + return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') + + def represent_bool(self, data): + if data: + value = 'true' + else: + value = 'false' + return self.represent_scalar('tag:yaml.org,2002:bool', value) + + def represent_int(self, data): + return self.represent_scalar('tag:yaml.org,2002:int', str(data)) + + inf_value = 1e300 + while repr(inf_value) != repr(inf_value*inf_value): + inf_value *= inf_value + + def represent_float(self, data): + if data != data or (data == 0.0 and data == 1.0): + value = '.nan' + elif data == self.inf_value: + value = '.inf' + elif data == -self.inf_value: + value = '-.inf' + else: + value = repr(data).lower() + # Note that in some cases `repr(data)` represents a float number + # without the decimal parts. For instance: + # >>> repr(1e17) + # '1e17' + # Unfortunately, this is not a valid float representation according + # to the definition of the `!!float` tag. We fix this by adding + # '.0' before the 'e' symbol. + if '.' not in value and 'e' in value: + value = value.replace('e', '.0e', 1) + return self.represent_scalar('tag:yaml.org,2002:float', value) + + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence('tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping('tag:yaml.org,2002:map', data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping('tag:yaml.org,2002:set', value) + + def represent_date(self, data): + value = data.isoformat() + return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + + def represent_datetime(self, data): + value = data.isoformat(' ') + return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + +SafeRepresenter.add_representer(type(None), + SafeRepresenter.represent_none) + +SafeRepresenter.add_representer(str, + SafeRepresenter.represent_str) + +SafeRepresenter.add_representer(bytes, + SafeRepresenter.represent_binary) + +SafeRepresenter.add_representer(bool, + SafeRepresenter.represent_bool) + +SafeRepresenter.add_representer(int, + SafeRepresenter.represent_int) + +SafeRepresenter.add_representer(float, + SafeRepresenter.represent_float) + +SafeRepresenter.add_representer(list, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(tuple, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(dict, + SafeRepresenter.represent_dict) + +SafeRepresenter.add_representer(set, + SafeRepresenter.represent_set) + +SafeRepresenter.add_representer(datetime.date, + SafeRepresenter.represent_date) + +SafeRepresenter.add_representer(datetime.datetime, + SafeRepresenter.represent_datetime) + +SafeRepresenter.add_representer(None, + SafeRepresenter.represent_undefined) + +class Representer(SafeRepresenter): + + def represent_complex(self, data): + if data.imag == 0.0: + data = '%r' % data.real + elif data.real == 0.0: + data = '%rj' % data.imag + elif data.imag > 0: + data = '%r+%rj' % (data.real, data.imag) + else: + data = '%r%rj' % (data.real, data.imag) + return self.represent_scalar('tag:yaml.org,2002:python/complex', data) + + def represent_tuple(self, data): + return self.represent_sequence('tag:yaml.org,2002:python/tuple', data) + + def represent_name(self, data): + name = '%s.%s' % (data.__module__, data.__name__) + return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '') + + def represent_module(self, data): + return self.represent_scalar( + 'tag:yaml.org,2002:python/module:'+data.__name__, '') + + def represent_object(self, data): + # We use __reduce__ API to save the data. data.__reduce__ returns + # a tuple of length 2-5: + # (function, args, state, listitems, dictitems) + + # For reconstructing, we calls function(*args), then set its state, + # listitems, and dictitems if they are not None. + + # A special case is when function.__name__ == '__newobj__'. In this + # case we create the object with args[0].__new__(*args). + + # Another special case is when __reduce__ returns a string - we don't + # support it. + + # We produce a !!python/object, !!python/object/new or + # !!python/object/apply node. + + cls = type(data) + if cls in copyreg.dispatch_table: + reduce = copyreg.dispatch_table[cls](data) + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent an object", data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = 'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = 'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = '%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + return self.represent_mapping( + 'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return self.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return self.represent_mapping(tag+function_name, value) + + def represent_ordered_dict(self, data): + # Provide uniform representation across different Python versions. + data_type = type(data) + tag = 'tag:yaml.org,2002:python/object/apply:%s.%s' \ + % (data_type.__module__, data_type.__name__) + items = [[key, value] for key, value in data.items()] + return self.represent_sequence(tag, [items]) + +Representer.add_representer(complex, + Representer.represent_complex) + +Representer.add_representer(tuple, + Representer.represent_tuple) + +Representer.add_representer(type, + Representer.represent_name) + +Representer.add_representer(collections.OrderedDict, + Representer.represent_ordered_dict) + +Representer.add_representer(types.FunctionType, + Representer.represent_name) + +Representer.add_representer(types.BuiltinFunctionType, + Representer.represent_name) + +Representer.add_representer(types.ModuleType, + Representer.represent_module) + +Representer.add_multi_representer(object, + Representer.represent_object) + diff --git a/lib/invoke/vendor/yaml/resolver.py b/lib/invoke/vendor/yaml/resolver.py new file mode 100644 index 0000000..013896d --- /dev/null +++ b/lib/invoke/vendor/yaml/resolver.py @@ -0,0 +1,227 @@ + +__all__ = ['BaseResolver', 'Resolver'] + +from .error import * +from .nodes import * + +import re + +class ResolverError(YAMLError): + pass + +class BaseResolver: + + DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' + + yaml_implicit_resolvers = {} + yaml_path_resolvers = {} + + def __init__(self): + self.resolver_exact_paths = [] + self.resolver_prefix_paths = [] + + @classmethod + def add_implicit_resolver(cls, tag, regexp, first): + if not 'yaml_implicit_resolvers' in cls.__dict__: + implicit_resolvers = {} + for key in cls.yaml_implicit_resolvers: + implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:] + cls.yaml_implicit_resolvers = implicit_resolvers + if first is None: + first = [None] + for ch in first: + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp)) + + @classmethod + def add_path_resolver(cls, tag, path, kind=None): + # Note: `add_path_resolver` is experimental. The API could be changed. + # `new_path` is a pattern that is matched against the path from the + # root to the node that is being considered. `node_path` elements are + # tuples `(node_check, index_check)`. `node_check` is a node class: + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None` + # matches any kind of a node. `index_check` could be `None`, a boolean + # value, a string value, or a number. `None` and `False` match against + # any _value_ of sequence and mapping nodes. `True` matches against + # any _key_ of a mapping node. A string `index_check` matches against + # a mapping value that corresponds to a scalar key which content is + # equal to the `index_check` value. An integer `index_check` matches + # against a sequence value with the index equal to `index_check`. + if not 'yaml_path_resolvers' in cls.__dict__: + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy() + new_path = [] + for element in path: + if isinstance(element, (list, tuple)): + if len(element) == 2: + node_check, index_check = element + elif len(element) == 1: + node_check = element[0] + index_check = True + else: + raise ResolverError("Invalid path element: %s" % element) + else: + node_check = None + index_check = element + if node_check is str: + node_check = ScalarNode + elif node_check is list: + node_check = SequenceNode + elif node_check is dict: + node_check = MappingNode + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \ + and not isinstance(node_check, str) \ + and node_check is not None: + raise ResolverError("Invalid node checker: %s" % node_check) + if not isinstance(index_check, (str, int)) \ + and index_check is not None: + raise ResolverError("Invalid index checker: %s" % index_check) + new_path.append((node_check, index_check)) + if kind is str: + kind = ScalarNode + elif kind is list: + kind = SequenceNode + elif kind is dict: + kind = MappingNode + elif kind not in [ScalarNode, SequenceNode, MappingNode] \ + and kind is not None: + raise ResolverError("Invalid node kind: %s" % kind) + cls.yaml_path_resolvers[tuple(new_path), kind] = tag + + def descend_resolver(self, current_node, current_index): + if not self.yaml_path_resolvers: + return + exact_paths = {} + prefix_paths = [] + if current_node: + depth = len(self.resolver_prefix_paths) + for path, kind in self.resolver_prefix_paths[-1]: + if self.check_resolver_prefix(depth, path, kind, + current_node, current_index): + if len(path) > depth: + prefix_paths.append((path, kind)) + else: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + for path, kind in self.yaml_path_resolvers: + if not path: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + prefix_paths.append((path, kind)) + self.resolver_exact_paths.append(exact_paths) + self.resolver_prefix_paths.append(prefix_paths) + + def ascend_resolver(self): + if not self.yaml_path_resolvers: + return + self.resolver_exact_paths.pop() + self.resolver_prefix_paths.pop() + + def check_resolver_prefix(self, depth, path, kind, + current_node, current_index): + node_check, index_check = path[depth-1] + if isinstance(node_check, str): + if current_node.tag != node_check: + return + elif node_check is not None: + if not isinstance(current_node, node_check): + return + if index_check is True and current_index is not None: + return + if (index_check is False or index_check is None) \ + and current_index is None: + return + if isinstance(index_check, str): + if not (isinstance(current_index, ScalarNode) + and index_check == current_index.value): + return + elif isinstance(index_check, int) and not isinstance(index_check, bool): + if index_check != current_index: + return + return True + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == '': + resolvers = self.yaml_implicit_resolvers.get('', []) + else: + resolvers = self.yaml_implicit_resolvers.get(value[0], []) + wildcard_resolvers = self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers + wildcard_resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + if self.yaml_path_resolvers: + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + +class Resolver(BaseResolver): + pass + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:bool', + re.compile(r'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list('yYnNtTfFoO')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:float', + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:int', + re.compile(r'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list('-+0123456789')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:merge', + re.compile(r'^(?:<<)$'), + ['<']) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:null', + re.compile(r'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + ['~', 'n', 'N', '']) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:timestamp', + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list('0123456789')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:value', + re.compile(r'^(?:=)$'), + ['=']) + +# The following resolver is only for documentation purposes. It cannot work +# because plain scalars cannot start with '!', '&', or '*'. +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:yaml', + re.compile(r'^(?:!|&|\*)$'), + list('!&*')) + diff --git a/lib/invoke/vendor/yaml/scanner.py b/lib/invoke/vendor/yaml/scanner.py new file mode 100644 index 0000000..7437ede --- /dev/null +++ b/lib/invoke/vendor/yaml/scanner.py @@ -0,0 +1,1435 @@ + +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain, style) +# +# Read comments in the Scanner code for more details. +# + +__all__ = ['Scanner', 'ScannerError'] + +from .error import MarkedYAMLError +from .tokens import * + +class ScannerError(MarkedYAMLError): + pass + +class SimpleKey: + # See below simple keys treatment. + + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + +class Scanner: + + def __init__(self): + """Initialize the scanner.""" + # It is assumed that Scanner and Reader will have a common descendant. + # Reader do the dirty work of checking for BOM and converting the + # input data to Unicode. It also adds NUL to the end. + # + # Reader supports the following methods + # self.peek(i=0) # peek the next i-th character + # self.prefix(l=1) # peek the next l characters + # self.forward(l=1) # read the next l characters and move the pointer. + + # Had we reached the end of the stream? + self.done = False + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + self.flow_level = 0 + + # List of processed tokens that are not yet emitted. + self.tokens = [] + + # Add the STREAM-START token. + self.fetch_stream_start() + + # Number of tokens that were emitted through the `get_token` method. + self.tokens_taken = 0 + + # The current indentation level. + self.indent = -1 + + # Past indentation levels. + self.indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + self.allow_simple_key = True + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + self.possible_simple_keys = {} + + # Public methods. + + def check_token(self, *choices): + # Check if the next token is one of the given types. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + if not choices: + return True + for choice in choices: + if isinstance(self.tokens[0], choice): + return True + return False + + def peek_token(self): + # Return the next token, but do not delete if from the queue. + # Return None if no more tokens. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + return self.tokens[0] + else: + return None + + def get_token(self): + # Return the next token. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + + # Private methods. + + def need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + # The current token may be a potential simple key, so we + # need to look further. + self.stale_possible_simple_keys() + if self.next_possible_simple_key() == self.tokens_taken: + return True + + def fetch_more_tokens(self): + + # Eat whitespaces and comments until we reach the next token. + self.scan_to_next_token() + + # Remove obsolete possible simple keys. + self.stale_possible_simple_keys() + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + self.unwind_indent(self.column) + + # Peek the next character. + ch = self.peek() + + # Is it the end of stream? + if ch == '\0': + return self.fetch_stream_end() + + # Is it a directive? + if ch == '%' and self.check_directive(): + return self.fetch_directive() + + # Is it the document start? + if ch == '-' and self.check_document_start(): + return self.fetch_document_start() + + # Is it the document end? + if ch == '.' and self.check_document_end(): + return self.fetch_document_end() + + # TODO: support for BOM within a stream. + #if ch == '\uFEFF': + # return self.fetch_bom() <-- issue BOMToken + + # Note: the order of the following checks is NOT significant. + + # Is it the flow sequence start indicator? + if ch == '[': + return self.fetch_flow_sequence_start() + + # Is it the flow mapping start indicator? + if ch == '{': + return self.fetch_flow_mapping_start() + + # Is it the flow sequence end indicator? + if ch == ']': + return self.fetch_flow_sequence_end() + + # Is it the flow mapping end indicator? + if ch == '}': + return self.fetch_flow_mapping_end() + + # Is it the flow entry indicator? + if ch == ',': + return self.fetch_flow_entry() + + # Is it the block entry indicator? + if ch == '-' and self.check_block_entry(): + return self.fetch_block_entry() + + # Is it the key indicator? + if ch == '?' and self.check_key(): + return self.fetch_key() + + # Is it the value indicator? + if ch == ':' and self.check_value(): + return self.fetch_value() + + # Is it an alias? + if ch == '*': + return self.fetch_alias() + + # Is it an anchor? + if ch == '&': + return self.fetch_anchor() + + # Is it a tag? + if ch == '!': + return self.fetch_tag() + + # Is it a literal scalar? + if ch == '|' and not self.flow_level: + return self.fetch_literal() + + # Is it a folded scalar? + if ch == '>' and not self.flow_level: + return self.fetch_folded() + + # Is it a single quoted scalar? + if ch == '\'': + return self.fetch_single() + + # Is it a double quoted scalar? + if ch == '\"': + return self.fetch_double() + + # It must be a plain scalar then. + if self.check_plain(): + return self.fetch_plain() + + # No? It's an error. Let's produce a nice error message. + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" % ch, + self.get_mark()) + + # Simple keys treatment. + + def next_possible_simple_key(self): + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. We may replace it + # with the following code: + # if not self.possible_simple_keys: + # return None + # return self.possible_simple_keys[ + # min(self.possible_simple_keys.keys())].token_number + min_token_number = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if min_token_number is None or key.token_number < min_token_number: + min_token_number = key.token_number + return min_token_number + + def stale_possible_simple_keys(self): + # Remove entries that are no longer possible simple keys. According to + # the YAML specification, simple keys + # - should be limited to a single line, + # - should be no longer than 1024 characters. + # Disabling this procedure will allow simple keys of any length and + # height (may cause problems if indentation is broken though). + for level in list(self.possible_simple_keys): + key = self.possible_simple_keys[level] + if key.line != self.line \ + or self.index-key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + del self.possible_simple_keys[level] + + def save_possible_simple_key(self): + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + + # Check if a simple key is required at the current position. + required = not self.flow_level and self.indent == self.column + + # The next token might be a simple key. Let's save it's number and + # position. + if self.allow_simple_key: + self.remove_possible_simple_key() + token_number = self.tokens_taken+len(self.tokens) + key = SimpleKey(token_number, required, + self.index, self.line, self.column, self.get_mark()) + self.possible_simple_keys[self.flow_level] = key + + def remove_possible_simple_key(self): + # Remove the saved possible key position at the current flow level. + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + + del self.possible_simple_keys[self.flow_level] + + # Indentation functions. + + def unwind_indent(self, column): + + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `self.indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if self.flow_level and self.indent > column: + # raise ScannerError(None, None, + # "invalid indentation or unclosed '[' or '{'", + # self.get_mark()) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + if self.flow_level: + return + + # In block context, we may need to issue the BLOCK-END tokens. + while self.indent > column: + mark = self.get_mark() + self.indent = self.indents.pop() + self.tokens.append(BlockEndToken(mark, mark)) + + def add_indent(self, column): + # Check if we need to increase indentation. + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # Fetchers. + + def fetch_stream_start(self): + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + + # Read the token. + mark = self.get_mark() + + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.encoding)) + + + def fetch_stream_end(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + self.possible_simple_keys = {} + + # Read the token. + mark = self.get_mark() + + # Add STREAM-END. + self.tokens.append(StreamEndToken(mark, mark)) + + # The steam is finished. + self.done = True + + def fetch_directive(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Scan and add DIRECTIVE. + self.tokens.append(self.scan_directive()) + + def fetch_document_start(self): + self.fetch_document_indicator(DocumentStartToken) + + def fetch_document_end(self): + self.fetch_document_indicator(DocumentEndToken) + + def fetch_document_indicator(self, TokenClass): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. Note that there could not be a block collection + # after '---'. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = self.get_mark() + self.forward(3) + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_start(self): + self.fetch_flow_collection_start(FlowSequenceStartToken) + + def fetch_flow_mapping_start(self): + self.fetch_flow_collection_start(FlowMappingStartToken) + + def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() + + # Increase the flow level. + self.flow_level += 1 + + # Simple keys are allowed after '[' and '{'. + self.allow_simple_key = True + + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_end(self): + self.fetch_flow_collection_end(FlowSequenceEndToken) + + def fetch_flow_mapping_end(self): + self.fetch_flow_collection_end(FlowMappingEndToken) + + def fetch_flow_collection_end(self, TokenClass): + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Decrease the flow level. + self.flow_level -= 1 + + # No simple keys after ']' or '}'. + self.allow_simple_key = False + + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_entry(self): + + # Simple keys are allowed after ','. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add FLOW-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(FlowEntryToken(start_mark, end_mark)) + + def fetch_block_entry(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a new entry? + if not self.allow_simple_key: + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-SEQUENCE-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockSequenceStartToken(mark, mark)) + + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + else: + pass + + # Simple keys are allowed after '-'. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add BLOCK-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(BlockEntryToken(start_mark, end_mark)) + + def fetch_key(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a key (not necessary a simple)? + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-MAPPING-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after '?' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add KEY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(KeyToken(start_mark, end_mark)) + + def fetch_value(self): + + # Do we determine a simple key? + if self.flow_level in self.possible_simple_keys: + + # Add KEY. + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number-self.tokens_taken, + KeyToken(key.mark, key.mark)) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + if not self.flow_level: + if self.add_indent(key.column): + self.tokens.insert(key.token_number-self.tokens_taken, + BlockMappingStartToken(key.mark, key.mark)) + + # There cannot be two simple keys one after another. + self.allow_simple_key = False + + # It must be a part of a complex key. + else: + + # Block context needs additional checks. + # (Do we really need them? They will be caught by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.get_mark()) + + # If this value starts a new block mapping, we need to add + # BLOCK-MAPPING-START. It will be detected as an error later by + # the parser. + if not self.flow_level: + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after ':' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add VALUE. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(ValueToken(start_mark, end_mark)) + + def fetch_alias(self): + + # ALIAS could be a simple key. + self.save_possible_simple_key() + + # No simple keys after ALIAS. + self.allow_simple_key = False + + # Scan and add ALIAS. + self.tokens.append(self.scan_anchor(AliasToken)) + + def fetch_anchor(self): + + # ANCHOR could start a simple key. + self.save_possible_simple_key() + + # No simple keys after ANCHOR. + self.allow_simple_key = False + + # Scan and add ANCHOR. + self.tokens.append(self.scan_anchor(AnchorToken)) + + def fetch_tag(self): + + # TAG could start a simple key. + self.save_possible_simple_key() + + # No simple keys after TAG. + self.allow_simple_key = False + + # Scan and add TAG. + self.tokens.append(self.scan_tag()) + + def fetch_literal(self): + self.fetch_block_scalar(style='|') + + def fetch_folded(self): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): + + # A simple key may follow a block scalar. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Scan and add SCALAR. + self.tokens.append(self.scan_block_scalar(style)) + + def fetch_single(self): + self.fetch_flow_scalar(style='\'') + + def fetch_double(self): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): + + # A flow scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after flow scalars. + self.allow_simple_key = False + + # Scan and add SCALAR. + self.tokens.append(self.scan_flow_scalar(style)) + + def fetch_plain(self): + + # A plain scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + self.allow_simple_key = False + + # Scan and add SCALAR. May change `allow_simple_key`. + self.tokens.append(self.scan_plain()) + + # Checkers. + + def check_directive(self): + + # DIRECTIVE: ^ '%' ... + # The '%' indicator is already checked. + if self.column == 0: + return True + + def check_document_start(self): + + # DOCUMENT-START: ^ '---' (' '|'\n') + if self.column == 0: + if self.prefix(3) == '---' \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return True + + def check_document_end(self): + + # DOCUMENT-END: ^ '...' (' '|'\n') + if self.column == 0: + if self.prefix(3) == '...' \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return True + + def check_block_entry(self): + + # BLOCK-ENTRY: '-' (' '|'\n') + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_key(self): + + # KEY(flow context): '?' + if self.flow_level: + return True + + # KEY(block context): '?' (' '|'\n') + else: + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_value(self): + + # VALUE(flow context): ':' + if self.flow_level: + return True + + # VALUE(block context): ':' (' '|'\n') + else: + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_plain(self): + + # A plain scalar may start with any non-space character except: + # '-', '?', ':', ',', '[', ']', '{', '}', + # '#', '&', '*', '!', '|', '>', '\'', '\"', + # '%', '@', '`'. + # + # It may also start with + # '-', '?', ':' + # if it is followed by a non-space character. + # + # Note that we limit the last rule to the block context (except the + # '-' character) because we want the flow context to be space + # independent. + ch = self.peek() + return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ + or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029' + and (ch == '-' or (not self.flow_level and ch in '?:'))) + + # Scanners. + + def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # The byte order mark is stripped if it's the first character in the + # stream. We do not yet support BOM inside the stream as the + # specification requires. Any such mark will be considered as a part + # of the document. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if : + # self.allow_simple_keys = False + # We also need to add the check for `allow_simple_keys == True` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + + if self.index == 0 and self.peek() == '\uFEFF': + self.forward() + found = False + while not found: + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + if self.scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def scan_directive(self): + # See the specification for details. + start_mark = self.get_mark() + self.forward() + name = self.scan_directive_name(start_mark) + value = None + if name == 'YAML': + value = self.scan_yaml_directive_value(start_mark) + end_mark = self.get_mark() + elif name == 'TAG': + value = self.scan_tag_directive_value(start_mark) + end_mark = self.get_mark() + else: + end_mark = self.get_mark() + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + self.scan_directive_ignored_line(start_mark) + return DirectiveToken(name, value, start_mark, end_mark) + + def scan_directive_name(self, start_mark): + # See the specification for details. + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + return value + + def scan_yaml_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + major = self.scan_yaml_directive_number(start_mark) + if self.peek() != '.': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or '.', but found %r" % self.peek(), + self.get_mark()) + self.forward() + minor = self.scan_yaml_directive_number(start_mark) + if self.peek() not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or ' ', but found %r" % self.peek(), + self.get_mark()) + return (major, minor) + + def scan_yaml_directive_number(self, start_mark): + # See the specification for details. + ch = self.peek() + if not ('0' <= ch <= '9'): + raise ScannerError("while scanning a directive", start_mark, + "expected a digit, but found %r" % ch, self.get_mark()) + length = 0 + while '0' <= self.peek(length) <= '9': + length += 1 + value = int(self.prefix(length)) + self.forward(length) + return value + + def scan_tag_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + handle = self.scan_tag_directive_handle(start_mark) + while self.peek() == ' ': + self.forward() + prefix = self.scan_tag_directive_prefix(start_mark) + return (handle, prefix) + + def scan_tag_directive_handle(self, start_mark): + # See the specification for details. + value = self.scan_tag_handle('directive', start_mark) + ch = self.peek() + if ch != ' ': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + return value + + def scan_tag_directive_prefix(self, start_mark): + # See the specification for details. + value = self.scan_tag_uri('directive', start_mark) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + return value + + def scan_directive_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in '\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a comment or a line break, but found %r" + % ch, self.get_mark()) + self.scan_line_break() + + def scan_anchor(self, TokenClass): + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpreted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = self.get_mark() + indicator = self.peek() + if indicator == '*': + name = 'alias' + else: + name = 'anchor' + self.forward() + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`': + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + end_mark = self.get_mark() + return TokenClass(value, start_mark, end_mark) + + def scan_tag(self): + # See the specification for details. + start_mark = self.get_mark() + ch = self.peek(1) + if ch == '<': + handle = None + self.forward(2) + suffix = self.scan_tag_uri('tag', start_mark) + if self.peek() != '>': + raise ScannerError("while parsing a tag", start_mark, + "expected '>', but found %r" % self.peek(), + self.get_mark()) + self.forward() + elif ch in '\0 \t\r\n\x85\u2028\u2029': + handle = None + suffix = '!' + self.forward() + else: + length = 1 + use_handle = False + while ch not in '\0 \r\n\x85\u2028\u2029': + if ch == '!': + use_handle = True + break + length += 1 + ch = self.peek(length) + handle = '!' + if use_handle: + handle = self.scan_tag_handle('tag', start_mark) + else: + handle = '!' + self.forward() + suffix = self.scan_tag_uri('tag', start_mark) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a tag", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + value = (handle, suffix) + end_mark = self.get_mark() + return TagToken(value, start_mark, end_mark) + + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False + + chunks = [] + start_mark = self.get_mark() + + # Scan the header. + self.forward() + chomping, increment = self.scan_block_scalar_indicators(start_mark) + self.scan_block_scalar_ignored_line(start_mark) + + # Determine the indentation level and go to the first non-empty line. + min_indent = self.indent+1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end_mark = self.scan_block_scalar_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent+increment-1 + breaks, end_mark = self.scan_block_scalar_breaks(indent) + line_break = '' + + # Scan the inner part of the block scalar. + while self.column == indent and self.peek() != '\0': + chunks.extend(breaks) + leading_non_space = self.peek() not in ' \t' + length = 0 + while self.peek(length) not in '\0\r\n\x85\u2028\u2029': + length += 1 + chunks.append(self.prefix(length)) + self.forward(length) + line_break = self.scan_line_break() + breaks, end_mark = self.scan_block_scalar_breaks(indent) + if self.column == indent and self.peek() != '\0': + + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + + if folded and line_break == '\n' \ + and leading_non_space and self.peek() not in ' \t': + if not breaks: + chunks.append(' ') + else: + chunks.append(line_break) + + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == '\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else: + break + + # Chomp the tail. + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + + # We are done. + return ScalarToken(''.join(chunks), False, start_mark, end_mark, + style) + + def scan_block_scalar_indicators(self, start_mark): + # See the specification for details. + chomping = None + increment = None + ch = self.peek() + if ch in '+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch in '0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + elif ch in '0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + ch = self.peek() + if ch in '+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected chomping or indentation indicators, but found %r" + % ch, self.get_mark()) + return chomping, increment + + def scan_block_scalar_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in '\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected a comment or a line break, but found %r" % ch, + self.get_mark()) + self.scan_line_break() + + def scan_block_scalar_indentation(self): + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = self.get_mark() + while self.peek() in ' \r\n\x85\u2028\u2029': + if self.peek() != ' ': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + else: + self.forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end_mark + + def scan_block_scalar_breaks(self, indent): + # See the specification for details. + chunks = [] + end_mark = self.get_mark() + while self.column < indent and self.peek() == ' ': + self.forward() + while self.peek() in '\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + while self.column < indent and self.peek() == ' ': + self.forward() + return chunks, end_mark + + def scan_flow_scalar(self, style): + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False + chunks = [] + start_mark = self.get_mark() + quote = self.peek() + self.forward() + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + while self.peek() != quote: + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark)) + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + self.forward() + end_mark = self.get_mark() + return ScalarToken(''.join(chunks), False, start_mark, end_mark, + style) + + ESCAPE_REPLACEMENTS = { + '0': '\0', + 'a': '\x07', + 'b': '\x08', + 't': '\x09', + '\t': '\x09', + 'n': '\x0A', + 'v': '\x0B', + 'f': '\x0C', + 'r': '\x0D', + 'e': '\x1B', + ' ': '\x20', + '\"': '\"', + '\\': '\\', + '/': '/', + 'N': '\x85', + '_': '\xA0', + 'L': '\u2028', + 'P': '\u2029', + } + + ESCAPE_CODES = { + 'x': 2, + 'u': 4, + 'U': 8, + } + + def scan_flow_scalar_non_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + length = 0 + while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029': + length += 1 + if length: + chunks.append(self.prefix(length)) + self.forward(length) + ch = self.peek() + if not double and ch == '\'' and self.peek(1) == '\'': + chunks.append('\'') + self.forward(2) + elif (double and ch == '\'') or (not double and ch in '\"\\'): + chunks.append(ch) + self.forward() + elif double and ch == '\\': + self.forward() + ch = self.peek() + if ch in self.ESCAPE_REPLACEMENTS: + chunks.append(self.ESCAPE_REPLACEMENTS[ch]) + self.forward() + elif ch in self.ESCAPE_CODES: + length = self.ESCAPE_CODES[ch] + self.forward() + for k in range(length): + if self.peek(k) not in '0123456789ABCDEFabcdef': + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of %d hexdecimal numbers, but found %r" % + (length, self.peek(k)), self.get_mark()) + code = int(self.prefix(length), 16) + chunks.append(chr(code)) + self.forward(length) + elif ch in '\r\n\x85\u2028\u2029': + self.scan_line_break() + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) + else: + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "found unknown escape character %r" % ch, self.get_mark()) + else: + return chunks + + def scan_flow_scalar_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + length = 0 + while self.peek(length) in ' \t': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch == '\0': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected end of stream", self.get_mark()) + elif ch in '\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + breaks = self.scan_flow_scalar_breaks(double, start_mark) + if line_break != '\n': + chunks.append(line_break) + elif not breaks: + chunks.append(' ') + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def scan_flow_scalar_breaks(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + # Instead of checking indentation, we check for document + # separators. + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected document separator", self.get_mark()) + while self.peek() in ' \t': + self.forward() + if self.peek() in '\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + else: + return chunks + + def scan_plain(self): + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',' or '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + start_mark = self.get_mark() + end_mark = start_mark + indent = self.indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0: + # indent = 1 + spaces = [] + while True: + length = 0 + if self.peek() == '#': + break + while True: + ch = self.peek(length) + if ch in '\0 \t\r\n\x85\u2028\u2029' \ + or (ch == ':' and + self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029' + + (u',[]{}' if self.flow_level else u''))\ + or (self.flow_level and ch in ',?[]{}'): + break + length += 1 + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self.prefix(length)) + self.forward(length) + end_mark = self.get_mark() + spaces = self.scan_plain_spaces(indent, start_mark) + if not spaces or self.peek() == '#' \ + or (not self.flow_level and self.column < indent): + break + return ScalarToken(''.join(chunks), True, start_mark, end_mark) + + def scan_plain_spaces(self, indent, start_mark): + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + while self.peek(length) in ' ': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch in '\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + self.allow_simple_key = True + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return + breaks = [] + while self.peek() in ' \r\n\x85\u2028\u2029': + if self.peek() == ' ': + self.forward() + else: + breaks.append(self.scan_line_break()) + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return + if line_break != '\n': + chunks.append(line_break) + elif not breaks: + chunks.append(' ') + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + def scan_tag_handle(self, name, start_mark): + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = self.peek() + if ch != '!': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch, self.get_mark()) + length = 1 + ch = self.peek(length) + if ch != ' ': + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if ch != '!': + self.forward(length) + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch, self.get_mark()) + length += 1 + value = self.prefix(length) + self.forward(length) + return value + + def scan_tag_uri(self, name, start_mark): + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?:@&=+$,_.!~*\'()[]%': + if ch == '%': + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + chunks.append(self.scan_uri_escapes(name, start_mark)) + else: + length += 1 + ch = self.peek(length) + if length: + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + if not chunks: + raise ScannerError("while parsing a %s" % name, start_mark, + "expected URI, but found %r" % ch, self.get_mark()) + return ''.join(chunks) + + def scan_uri_escapes(self, name, start_mark): + # See the specification for details. + codes = [] + mark = self.get_mark() + while self.peek() == '%': + self.forward() + for k in range(2): + if self.peek(k) not in '0123456789ABCDEFabcdef': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" + % self.peek(k), self.get_mark()) + codes.append(int(self.prefix(2), 16)) + self.forward(2) + try: + value = bytes(codes).decode('utf-8') + except UnicodeDecodeError as exc: + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark) + return value + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.peek() + if ch in '\r\n\x85': + if self.prefix(2) == '\r\n': + self.forward(2) + else: + self.forward() + return '\n' + elif ch in '\u2028\u2029': + self.forward() + return ch + return '' diff --git a/lib/invoke/vendor/yaml/serializer.py b/lib/invoke/vendor/yaml/serializer.py new file mode 100644 index 0000000..fe911e6 --- /dev/null +++ b/lib/invoke/vendor/yaml/serializer.py @@ -0,0 +1,111 @@ + +__all__ = ['Serializer', 'SerializerError'] + +from .error import YAMLError +from .events import * +from .nodes import * + +class SerializerError(YAMLError): + pass + +class Serializer: + + ANCHOR_TEMPLATE = 'id%03d' + + def __init__(self, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None): + self.use_encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.use_tags = tags + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + self.closed = None + + def open(self): + if self.closed is None: + self.emit(StreamStartEvent(encoding=self.use_encoding)) + self.closed = False + elif self.closed: + raise SerializerError("serializer is closed") + else: + raise SerializerError("serializer is already opened") + + def close(self): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif not self.closed: + self.emit(StreamEndEvent()) + self.closed = True + + #def __del__(self): + # self.close() + + def serialize(self, node): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif self.closed: + raise SerializerError("serializer is closed") + self.emit(DocumentStartEvent(explicit=self.use_explicit_start, + version=self.use_version, tags=self.use_tags)) + self.anchor_node(node) + self.serialize_node(node, None, None) + self.emit(DocumentEndEvent(explicit=self.use_explicit_end)) + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + + def anchor_node(self, node): + if node in self.anchors: + if self.anchors[node] is None: + self.anchors[node] = self.generate_anchor(node) + else: + self.anchors[node] = None + if isinstance(node, SequenceNode): + for item in node.value: + self.anchor_node(item) + elif isinstance(node, MappingNode): + for key, value in node.value: + self.anchor_node(key) + self.anchor_node(value) + + def generate_anchor(self, node): + self.last_anchor_id += 1 + return self.ANCHOR_TEMPLATE % self.last_anchor_id + + def serialize_node(self, node, parent, index): + alias = self.anchors[node] + if node in self.serialized_nodes: + self.emit(AliasEvent(alias)) + else: + self.serialized_nodes[node] = True + self.descend_resolver(parent, index) + if isinstance(node, ScalarNode): + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + self.emit(ScalarEvent(alias, node.tag, implicit, node.value, + style=node.style)) + elif isinstance(node, SequenceNode): + implicit = (node.tag + == self.resolve(SequenceNode, node.value, True)) + self.emit(SequenceStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + index = 0 + for item in node.value: + self.serialize_node(item, node, index) + index += 1 + self.emit(SequenceEndEvent()) + elif isinstance(node, MappingNode): + implicit = (node.tag + == self.resolve(MappingNode, node.value, True)) + self.emit(MappingStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + for key, value in node.value: + self.serialize_node(key, node, None) + self.serialize_node(value, node, key) + self.emit(MappingEndEvent()) + self.ascend_resolver() + diff --git a/lib/invoke/vendor/yaml/tokens.py b/lib/invoke/vendor/yaml/tokens.py new file mode 100644 index 0000000..4d0b48a --- /dev/null +++ b/lib/invoke/vendor/yaml/tokens.py @@ -0,0 +1,104 @@ + +class Token(object): + def __init__(self, start_mark, end_mark): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in self.__dict__ + if not key.endswith('_mark')] + attributes.sort() + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +#class BOMToken(Token): +# id = '' + +class DirectiveToken(Token): + id = '' + def __init__(self, name, value, start_mark, end_mark): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class DocumentStartToken(Token): + id = '' + +class DocumentEndToken(Token): + id = '' + +class StreamStartToken(Token): + id = '' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndToken(Token): + id = '' + +class BlockSequenceStartToken(Token): + id = '' + +class BlockMappingStartToken(Token): + id = '' + +class BlockEndToken(Token): + id = '' + +class FlowSequenceStartToken(Token): + id = '[' + +class FlowMappingStartToken(Token): + id = '{' + +class FlowSequenceEndToken(Token): + id = ']' + +class FlowMappingEndToken(Token): + id = '}' + +class KeyToken(Token): + id = '?' + +class ValueToken(Token): + id = ':' + +class BlockEntryToken(Token): + id = '-' + +class FlowEntryToken(Token): + id = ',' + +class AliasToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class AnchorToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class TagToken(Token): + id = '' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class ScalarToken(Token): + id = '' + def __init__(self, value, plain, start_mark, end_mark, style=None): + self.value = value + self.plain = plain + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + diff --git a/lib/invoke/watchers.py b/lib/invoke/watchers.py new file mode 100644 index 0000000..eb813df --- /dev/null +++ b/lib/invoke/watchers.py @@ -0,0 +1,145 @@ +import re +import threading +from typing import Generator, Iterable + +from .exceptions import ResponseNotAccepted + + +class StreamWatcher(threading.local): + """ + A class whose subclasses may act on seen stream data from subprocesses. + + Subclasses must exhibit the following API; see `Responder` for a concrete + example. + + * ``__init__`` is completely up to each subclass, though as usual, + subclasses *of* subclasses should be careful to make use of `super` where + appropriate. + * `submit` must accept the entire current contents of the stream being + watched, as a string, and may optionally return an iterable of strings + (or act as a generator iterator, i.e. multiple calls to ``yield + ``), which will each be written to the subprocess' standard + input. + + .. note:: + `StreamWatcher` subclasses exist in part to enable state tracking, such + as detecting when a submitted password didn't work & erroring (or + prompting a user, or etc). Such bookkeeping isn't easily achievable + with simple callback functions. + + .. note:: + `StreamWatcher` subclasses `threading.local` so that its instances can + be used to 'watch' both subprocess stdout and stderr in separate + threads. + + .. versionadded:: 1.0 + """ + + def submit(self, stream: str) -> Iterable[str]: + """ + Act on ``stream`` data, potentially returning responses. + + :param str stream: + All data read on this stream since the beginning of the session. + + :returns: + An iterable of ``str`` (which may be empty). + + .. versionadded:: 1.0 + """ + raise NotImplementedError + + +class Responder(StreamWatcher): + """ + A parameterizable object that submits responses to specific patterns. + + Commonly used to implement password auto-responds for things like ``sudo``. + + .. versionadded:: 1.0 + """ + + def __init__(self, pattern: str, response: str) -> None: + r""" + Imprint this `Responder` with necessary parameters. + + :param pattern: + A raw string (e.g. ``r"\[sudo\] password for .*:"``) which will be + turned into a regular expression. + + :param response: + The string to submit to the subprocess' stdin when ``pattern`` is + detected. + """ + # TODO: precompile the keys into regex objects + self.pattern = pattern + self.response = response + self.index = 0 + + def pattern_matches( + self, stream: str, pattern: str, index_attr: str + ) -> Iterable[str]: + """ + Generic "search for pattern in stream, using index" behavior. + + Used here and in some subclasses that want to track multiple patterns + concurrently. + + :param str stream: The same data passed to ``submit``. + :param str pattern: The pattern to search for. + :param str index_attr: The name of the index attribute to use. + :returns: An iterable of string matches. + + .. versionadded:: 1.0 + """ + # NOTE: generifies scanning so it can be used to scan for >1 pattern at + # once, e.g. in FailingResponder. + # Only look at stream contents we haven't seen yet, to avoid dupes. + index = getattr(self, index_attr) + new = stream[index:] + # Search, across lines if necessary + matches = re.findall(pattern, new, re.S) + # Update seek index if we've matched + if matches: + setattr(self, index_attr, index + len(new)) + return matches + + def submit(self, stream: str) -> Generator[str, None, None]: + # Iterate over findall() response in case >1 match occurred. + for _ in self.pattern_matches(stream, self.pattern, "index"): + yield self.response + + +class FailingResponder(Responder): + """ + Variant of `Responder` which is capable of detecting incorrect responses. + + This class adds a ``sentinel`` parameter to ``__init__``, and its + ``submit`` will raise `.ResponseNotAccepted` if it detects that sentinel + value in the stream. + + .. versionadded:: 1.0 + """ + + def __init__(self, pattern: str, response: str, sentinel: str) -> None: + super().__init__(pattern, response) + self.sentinel = sentinel + self.failure_index = 0 + self.tried = False + + def submit(self, stream: str) -> Generator[str, None, None]: + # Behave like regular Responder initially + response = super().submit(stream) + # Also check stream for our failure sentinel + failed = self.pattern_matches(stream, self.sentinel, "failure_index") + # Error out if we seem to have failed after a previous response. + if self.tried and failed: + err = 'Auto-response to r"{}" failed with {!r}!'.format( + self.pattern, self.sentinel + ) + raise ResponseNotAccepted(err) + # Once we see that we had a response, take note + if response: + self.tried = True + # Again, behave regularly by default. + return response diff --git a/lib/markdown_it/__init__.py b/lib/markdown_it/__init__.py new file mode 100644 index 0000000..9fac279 --- /dev/null +++ b/lib/markdown_it/__init__.py @@ -0,0 +1,6 @@ +"""A Python port of Markdown-It""" + +__all__ = ("MarkdownIt",) +__version__ = "4.0.0" + +from .main import MarkdownIt diff --git a/lib/markdown_it/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..b21ba87 Binary files /dev/null and b/lib/markdown_it/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/_compat.cpython-314.pyc b/lib/markdown_it/__pycache__/_compat.cpython-314.pyc new file mode 100644 index 0000000..3d07901 Binary files /dev/null and b/lib/markdown_it/__pycache__/_compat.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc b/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc new file mode 100644 index 0000000..2a74854 Binary files /dev/null and b/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/main.cpython-314.pyc b/lib/markdown_it/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000..fe403f5 Binary files /dev/null and b/lib/markdown_it/__pycache__/main.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc b/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc new file mode 100644 index 0000000..4ac9241 Binary files /dev/null and b/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc b/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc new file mode 100644 index 0000000..701ce57 Binary files /dev/null and b/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc b/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc new file mode 100644 index 0000000..7094f2d Binary files /dev/null and b/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/renderer.cpython-314.pyc b/lib/markdown_it/__pycache__/renderer.cpython-314.pyc new file mode 100644 index 0000000..4c8d010 Binary files /dev/null and b/lib/markdown_it/__pycache__/renderer.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/ruler.cpython-314.pyc b/lib/markdown_it/__pycache__/ruler.cpython-314.pyc new file mode 100644 index 0000000..d630a2d Binary files /dev/null and b/lib/markdown_it/__pycache__/ruler.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/token.cpython-314.pyc b/lib/markdown_it/__pycache__/token.cpython-314.pyc new file mode 100644 index 0000000..a979735 Binary files /dev/null and b/lib/markdown_it/__pycache__/token.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/tree.cpython-314.pyc b/lib/markdown_it/__pycache__/tree.cpython-314.pyc new file mode 100644 index 0000000..3729881 Binary files /dev/null and b/lib/markdown_it/__pycache__/tree.cpython-314.pyc differ diff --git a/lib/markdown_it/__pycache__/utils.cpython-314.pyc b/lib/markdown_it/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..8b15e23 Binary files /dev/null and b/lib/markdown_it/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/markdown_it/_compat.py b/lib/markdown_it/_compat.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/lib/markdown_it/_compat.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/lib/markdown_it/_punycode.py b/lib/markdown_it/_punycode.py new file mode 100644 index 0000000..312048b --- /dev/null +++ b/lib/markdown_it/_punycode.py @@ -0,0 +1,67 @@ +# Copyright 2014 Mathias Bynens +# Copyright 2021 Taneli Hukkinen +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import codecs +from collections.abc import Callable +import re + +REGEX_SEPARATORS = re.compile(r"[\x2E\u3002\uFF0E\uFF61]") +REGEX_NON_ASCII = re.compile(r"[^\0-\x7E]") + + +def encode(uni: str) -> str: + return codecs.encode(uni, encoding="punycode").decode() + + +def decode(ascii: str) -> str: + return codecs.decode(ascii, encoding="punycode") # type: ignore + + +def map_domain(string: str, fn: Callable[[str], str]) -> str: + parts = string.split("@") + result = "" + if len(parts) > 1: + # In email addresses, only the domain name should be punycoded. Leave + # the local part (i.e. everything up to `@`) intact. + result = parts[0] + "@" + string = parts[1] + labels = REGEX_SEPARATORS.split(string) + encoded = ".".join(fn(label) for label in labels) + return result + encoded + + +def to_unicode(obj: str) -> str: + def mapping(obj: str) -> str: + if obj.startswith("xn--"): + return decode(obj[4:].lower()) + return obj + + return map_domain(obj, mapping) + + +def to_ascii(obj: str) -> str: + def mapping(obj: str) -> str: + if REGEX_NON_ASCII.search(obj): + return "xn--" + encode(obj) + return obj + + return map_domain(obj, mapping) diff --git a/lib/markdown_it/cli/__init__.py b/lib/markdown_it/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..b024cce Binary files /dev/null and b/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc b/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc new file mode 100644 index 0000000..becd84d Binary files /dev/null and b/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc differ diff --git a/lib/markdown_it/cli/parse.py b/lib/markdown_it/cli/parse.py new file mode 100644 index 0000000..fe346b2 --- /dev/null +++ b/lib/markdown_it/cli/parse.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +""" +CLI interface to markdown-it-py + +Parse one or more markdown files, convert each to HTML, and print to stdout. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterable, Sequence +import sys + +from markdown_it import __version__ +from markdown_it.main import MarkdownIt + +version_str = f"markdown-it-py [version {__version__}]" + + +def main(args: Sequence[str] | None = None) -> int: + namespace = parse_args(args) + if namespace.filenames: + convert(namespace.filenames) + else: + interactive() + return 0 + + +def convert(filenames: Iterable[str]) -> None: + for filename in filenames: + convert_file(filename) + + +def convert_file(filename: str) -> None: + """ + Parse a Markdown file and dump the output to stdout. + """ + try: + with open(filename, encoding="utf8", errors="ignore") as fin: + rendered = MarkdownIt().render(fin.read()) + print(rendered, end="") + except OSError: + sys.stderr.write(f'Cannot open file "{filename}".\n') + sys.exit(1) + + +def interactive() -> None: + """ + Parse user input, dump to stdout, rinse and repeat. + Python REPL style. + """ + print_heading() + contents = [] + more = False + while True: + try: + prompt, more = ("... ", True) if more else (">>> ", True) + contents.append(input(prompt) + "\n") + except EOFError: + print("\n" + MarkdownIt().render("\n".join(contents)), end="") + more = False + contents = [] + except KeyboardInterrupt: + print("\nExiting.") + break + + +def parse_args(args: Sequence[str] | None) -> argparse.Namespace: + """Parse input CLI arguments.""" + parser = argparse.ArgumentParser( + description="Parse one or more markdown files, " + "convert each to HTML, and print to stdout", + # NOTE: Remember to update README.md w/ the output of `markdown-it -h` + epilog=( + f""" +Interactive: + + $ markdown-it + markdown-it-py [version {__version__}] (interactive) + Type Ctrl-D to complete input, or Ctrl-C to exit. + >>> # Example + ... > markdown *input* + ... +

Example

+
+

markdown input

+
+ +Batch: + + $ markdown-it README.md README.footer.md > index.html +""" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("-v", "--version", action="version", version=version_str) + parser.add_argument( + "filenames", nargs="*", help="specify an optional list of files to convert" + ) + return parser.parse_args(args) + + +def print_heading() -> None: + print(f"{version_str} (interactive)") + print("Type Ctrl-D to complete input, or Ctrl-C to exit.") + + +if __name__ == "__main__": + exit_code = main(sys.argv[1:]) + sys.exit(exit_code) diff --git a/lib/markdown_it/common/__init__.py b/lib/markdown_it/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..7a8d2a2 Binary files /dev/null and b/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc b/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc new file mode 100644 index 0000000..2875bc3 Binary files /dev/null and b/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc differ diff --git a/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc b/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc new file mode 100644 index 0000000..67f678c Binary files /dev/null and b/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc differ diff --git a/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc b/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc new file mode 100644 index 0000000..5df8f9f Binary files /dev/null and b/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc differ diff --git a/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc b/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc new file mode 100644 index 0000000..df8d56a Binary files /dev/null and b/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc differ diff --git a/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc b/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..4d978b0 Binary files /dev/null and b/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/markdown_it/common/entities.py b/lib/markdown_it/common/entities.py new file mode 100644 index 0000000..14d08ec --- /dev/null +++ b/lib/markdown_it/common/entities.py @@ -0,0 +1,5 @@ +"""HTML5 entities map: { name -> characters }.""" + +import html.entities + +entities = {name.rstrip(";"): chars for name, chars in html.entities.html5.items()} diff --git a/lib/markdown_it/common/html_blocks.py b/lib/markdown_it/common/html_blocks.py new file mode 100644 index 0000000..8a3b0b7 --- /dev/null +++ b/lib/markdown_it/common/html_blocks.py @@ -0,0 +1,69 @@ +"""List of valid html blocks names, according to commonmark spec +http://jgm.github.io/CommonMark/spec.html#html-blocks +""" + +# see https://spec.commonmark.org/0.31.2/#html-blocks +block_names = [ + "address", + "article", + "aside", + "base", + "basefont", + "blockquote", + "body", + "caption", + "center", + "col", + "colgroup", + "dd", + "details", + "dialog", + "dir", + "div", + "dl", + "dt", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "frame", + "frameset", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hr", + "html", + "iframe", + "legend", + "li", + "link", + "main", + "menu", + "menuitem", + "nav", + "noframes", + "ol", + "optgroup", + "option", + "p", + "param", + "search", + "section", + "summary", + "table", + "tbody", + "td", + "tfoot", + "th", + "thead", + "title", + "tr", + "track", + "ul", +] diff --git a/lib/markdown_it/common/html_re.py b/lib/markdown_it/common/html_re.py new file mode 100644 index 0000000..ab822c5 --- /dev/null +++ b/lib/markdown_it/common/html_re.py @@ -0,0 +1,39 @@ +"""Regexps to match html elements""" + +import re + +attr_name = "[a-zA-Z_:][a-zA-Z0-9:._-]*" + +unquoted = "[^\"'=<>`\\x00-\\x20]+" +single_quoted = "'[^']*'" +double_quoted = '"[^"]*"' + +attr_value = "(?:" + unquoted + "|" + single_quoted + "|" + double_quoted + ")" + +attribute = "(?:\\s+" + attr_name + "(?:\\s*=\\s*" + attr_value + ")?)" + +open_tag = "<[A-Za-z][A-Za-z0-9\\-]*" + attribute + "*\\s*\\/?>" + +close_tag = "<\\/[A-Za-z][A-Za-z0-9\\-]*\\s*>" +comment = "" +processing = "<[?][\\s\\S]*?[?]>" +declaration = "]*>" +cdata = "" + +HTML_TAG_RE = re.compile( + "^(?:" + + open_tag + + "|" + + close_tag + + "|" + + comment + + "|" + + processing + + "|" + + declaration + + "|" + + cdata + + ")" +) +HTML_OPEN_CLOSE_TAG_STR = "^(?:" + open_tag + "|" + close_tag + ")" +HTML_OPEN_CLOSE_TAG_RE = re.compile(HTML_OPEN_CLOSE_TAG_STR) diff --git a/lib/markdown_it/common/normalize_url.py b/lib/markdown_it/common/normalize_url.py new file mode 100644 index 0000000..92720b3 --- /dev/null +++ b/lib/markdown_it/common/normalize_url.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from collections.abc import Callable +from contextlib import suppress +import re +from urllib.parse import quote, unquote, urlparse, urlunparse # noqa: F401 + +import mdurl + +from .. import _punycode + +RECODE_HOSTNAME_FOR = ("http:", "https:", "mailto:") + + +def normalizeLink(url: str) -> str: + """Normalize destination URLs in links + + :: + + [label]: destination 'title' + ^^^^^^^^^^^ + """ + parsed = mdurl.parse(url, slashes_denote_host=True) + + # Encode hostnames in urls like: + # `http://host/`, `https://host/`, `mailto:user@host`, `//host/` + # + # We don't encode unknown schemas, because it's likely that we encode + # something we shouldn't (e.g. `skype:name` treated as `skype:host`) + # + if parsed.hostname and ( + not parsed.protocol or parsed.protocol in RECODE_HOSTNAME_FOR + ): + with suppress(Exception): + parsed = parsed._replace(hostname=_punycode.to_ascii(parsed.hostname)) + + return mdurl.encode(mdurl.format(parsed)) + + +def normalizeLinkText(url: str) -> str: + """Normalize autolink content + + :: + + + ~~~~~~~~~~~ + """ + parsed = mdurl.parse(url, slashes_denote_host=True) + + # Encode hostnames in urls like: + # `http://host/`, `https://host/`, `mailto:user@host`, `//host/` + # + # We don't encode unknown schemas, because it's likely that we encode + # something we shouldn't (e.g. `skype:name` treated as `skype:host`) + # + if parsed.hostname and ( + not parsed.protocol or parsed.protocol in RECODE_HOSTNAME_FOR + ): + with suppress(Exception): + parsed = parsed._replace(hostname=_punycode.to_unicode(parsed.hostname)) + + # add '%' to exclude list because of https://github.com/markdown-it/markdown-it/issues/720 + return mdurl.decode(mdurl.format(parsed), mdurl.DECODE_DEFAULT_CHARS + "%") + + +BAD_PROTO_RE = re.compile(r"^(vbscript|javascript|file|data):") +GOOD_DATA_RE = re.compile(r"^data:image\/(gif|png|jpeg|webp);") + + +def validateLink(url: str, validator: Callable[[str], bool] | None = None) -> bool: + """Validate URL link is allowed in output. + + This validator can prohibit more than really needed to prevent XSS. + It's a tradeoff to keep code simple and to be secure by default. + + Note: url should be normalized at this point, and existing entities decoded. + """ + if validator is not None: + return validator(url) + url = url.strip().lower() + return bool(GOOD_DATA_RE.search(url)) if BAD_PROTO_RE.search(url) else True diff --git a/lib/markdown_it/common/utils.py b/lib/markdown_it/common/utils.py new file mode 100644 index 0000000..11bda64 --- /dev/null +++ b/lib/markdown_it/common/utils.py @@ -0,0 +1,313 @@ +"""Utilities for parsing source text""" + +from __future__ import annotations + +import re +from re import Match +from typing import TypeVar +import unicodedata + +from .entities import entities + + +def charCodeAt(src: str, pos: int) -> int | None: + """ + Returns the Unicode value of the character at the specified location. + + @param - index The zero-based index of the desired character. + If there is no character at the specified index, NaN is returned. + + This was added for compatibility with python + """ + try: + return ord(src[pos]) + except IndexError: + return None + + +def charStrAt(src: str, pos: int) -> str | None: + """ + Returns the Unicode value of the character at the specified location. + + @param - index The zero-based index of the desired character. + If there is no character at the specified index, NaN is returned. + + This was added for compatibility with python + """ + try: + return src[pos] + except IndexError: + return None + + +_ItemTV = TypeVar("_ItemTV") + + +def arrayReplaceAt( + src: list[_ItemTV], pos: int, newElements: list[_ItemTV] +) -> list[_ItemTV]: + """ + Remove element from array and put another array at those position. + Useful for some operations with tokens + """ + return src[:pos] + newElements + src[pos + 1 :] + + +def isValidEntityCode(c: int) -> bool: + # broken sequence + if c >= 0xD800 and c <= 0xDFFF: + return False + # never used + if c >= 0xFDD0 and c <= 0xFDEF: + return False + if ((c & 0xFFFF) == 0xFFFF) or ((c & 0xFFFF) == 0xFFFE): + return False + # control codes + if c >= 0x00 and c <= 0x08: + return False + if c == 0x0B: + return False + if c >= 0x0E and c <= 0x1F: + return False + if c >= 0x7F and c <= 0x9F: + return False + # out of range + return not (c > 0x10FFFF) + + +def fromCodePoint(c: int) -> str: + """Convert ordinal to unicode. + + Note, in the original Javascript two string characters were required, + for codepoints larger than `0xFFFF`. + But Python 3 can represent any unicode codepoint in one character. + """ + return chr(c) + + +# UNESCAPE_MD_RE = re.compile(r'\\([!"#$%&\'()*+,\-.\/:;<=>?@[\\\]^_`{|}~])') +# ENTITY_RE_g = re.compile(r'&([a-z#][a-z0-9]{1,31})', re.IGNORECASE) +UNESCAPE_ALL_RE = re.compile( + r'\\([!"#$%&\'()*+,\-.\/:;<=>?@[\\\]^_`{|}~])' + "|" + r"&([a-z#][a-z0-9]{1,31});", + re.IGNORECASE, +) +DIGITAL_ENTITY_BASE10_RE = re.compile(r"#([0-9]{1,8})") +DIGITAL_ENTITY_BASE16_RE = re.compile(r"#x([a-f0-9]{1,8})", re.IGNORECASE) + + +def replaceEntityPattern(match: str, name: str) -> str: + """Convert HTML entity patterns, + see https://spec.commonmark.org/0.30/#entity-references + """ + if name in entities: + return entities[name] + + code: None | int = None + if pat := DIGITAL_ENTITY_BASE10_RE.fullmatch(name): + code = int(pat.group(1), 10) + elif pat := DIGITAL_ENTITY_BASE16_RE.fullmatch(name): + code = int(pat.group(1), 16) + + if code is not None and isValidEntityCode(code): + return fromCodePoint(code) + + return match + + +def unescapeAll(string: str) -> str: + def replacer_func(match: Match[str]) -> str: + escaped = match.group(1) + if escaped: + return escaped + entity = match.group(2) + return replaceEntityPattern(match.group(), entity) + + if "\\" not in string and "&" not in string: + return string + return UNESCAPE_ALL_RE.sub(replacer_func, string) + + +ESCAPABLE = r"""\\!"#$%&'()*+,./:;<=>?@\[\]^`{}|_~-""" +ESCAPE_CHAR = re.compile(r"\\([" + ESCAPABLE + r"])") + + +def stripEscape(string: str) -> str: + """Strip escape \\ characters""" + return ESCAPE_CHAR.sub(r"\1", string) + + +def escapeHtml(raw: str) -> str: + """Replace special characters "&", "<", ">" and '"' to HTML-safe sequences.""" + # like html.escape, but without escaping single quotes + raw = raw.replace("&", "&") # Must be done first! + raw = raw.replace("<", "<") + raw = raw.replace(">", ">") + raw = raw.replace('"', """) + return raw + + +# ////////////////////////////////////////////////////////////////////////////// + +REGEXP_ESCAPE_RE = re.compile(r"[.?*+^$[\]\\(){}|-]") + + +def escapeRE(string: str) -> str: + string = REGEXP_ESCAPE_RE.sub("\\$&", string) + return string + + +# ////////////////////////////////////////////////////////////////////////////// + + +def isSpace(code: int | None) -> bool: + """Check if character code is a whitespace.""" + return code in (0x09, 0x20) + + +def isStrSpace(ch: str | None) -> bool: + """Check if character is a whitespace.""" + return ch in ("\t", " ") + + +MD_WHITESPACE = { + 0x09, # \t + 0x0A, # \n + 0x0B, # \v + 0x0C, # \f + 0x0D, # \r + 0x20, # space + 0xA0, + 0x1680, + 0x202F, + 0x205F, + 0x3000, +} + + +def isWhiteSpace(code: int) -> bool: + r"""Zs (unicode class) || [\t\f\v\r\n]""" + if code >= 0x2000 and code <= 0x200A: + return True + return code in MD_WHITESPACE + + +# ////////////////////////////////////////////////////////////////////////////// + + +def isPunctChar(ch: str) -> bool: + """Check if character is a punctuation character.""" + return unicodedata.category(ch).startswith(("P", "S")) + + +MD_ASCII_PUNCT = { + 0x21, # /* ! */ + 0x22, # /* " */ + 0x23, # /* # */ + 0x24, # /* $ */ + 0x25, # /* % */ + 0x26, # /* & */ + 0x27, # /* ' */ + 0x28, # /* ( */ + 0x29, # /* ) */ + 0x2A, # /* * */ + 0x2B, # /* + */ + 0x2C, # /* , */ + 0x2D, # /* - */ + 0x2E, # /* . */ + 0x2F, # /* / */ + 0x3A, # /* : */ + 0x3B, # /* ; */ + 0x3C, # /* < */ + 0x3D, # /* = */ + 0x3E, # /* > */ + 0x3F, # /* ? */ + 0x40, # /* @ */ + 0x5B, # /* [ */ + 0x5C, # /* \ */ + 0x5D, # /* ] */ + 0x5E, # /* ^ */ + 0x5F, # /* _ */ + 0x60, # /* ` */ + 0x7B, # /* { */ + 0x7C, # /* | */ + 0x7D, # /* } */ + 0x7E, # /* ~ */ +} + + +def isMdAsciiPunct(ch: int) -> bool: + """Markdown ASCII punctuation characters. + + :: + + !, ", #, $, %, &, ', (, ), *, +, ,, -, ., /, :, ;, <, =, >, ?, @, [, \\, ], ^, _, `, {, |, }, or ~ + + See http://spec.commonmark.org/0.15/#ascii-punctuation-character + + Don't confuse with unicode punctuation !!! It lacks some chars in ascii range. + + """ + return ch in MD_ASCII_PUNCT + + +def normalizeReference(string: str) -> str: + """Helper to unify [reference labels].""" + # Trim and collapse whitespace + # + string = re.sub(r"\s+", " ", string.strip()) + + # In node v10 'ẞ'.toLowerCase() === 'Ṿ', which is presumed to be a bug + # fixed in v12 (couldn't find any details). + # + # So treat this one as a special case + # (remove this when node v10 is no longer supported). + # + # if ('ẞ'.toLowerCase() === 'Ṿ') { + # str = str.replace(/ẞ/g, 'ß') + # } + + # .toLowerCase().toUpperCase() should get rid of all differences + # between letter variants. + # + # Simple .toLowerCase() doesn't normalize 125 code points correctly, + # and .toUpperCase doesn't normalize 6 of them (list of exceptions: + # İ, ϴ, ẞ, Ω, K, Å - those are already uppercased, but have differently + # uppercased versions). + # + # Here's an example showing how it happens. Lets take greek letter omega: + # uppercase U+0398 (Θ), U+03f4 (ϴ) and lowercase U+03b8 (θ), U+03d1 (ϑ) + # + # Unicode entries: + # 0398;GREEK CAPITAL LETTER THETA;Lu;0;L;;;;;N;;;;03B8 + # 03B8;GREEK SMALL LETTER THETA;Ll;0;L;;;;;N;;;0398;;0398 + # 03D1;GREEK THETA SYMBOL;Ll;0;L; 03B8;;;;N;GREEK SMALL LETTER SCRIPT THETA;;0398;;0398 + # 03F4;GREEK CAPITAL THETA SYMBOL;Lu;0;L; 0398;;;;N;;;;03B8 + # + # Case-insensitive comparison should treat all of them as equivalent. + # + # But .toLowerCase() doesn't change ϑ (it's already lowercase), + # and .toUpperCase() doesn't change ϴ (already uppercase). + # + # Applying first lower then upper case normalizes any character: + # '\u0398\u03f4\u03b8\u03d1'.toLowerCase().toUpperCase() === '\u0398\u0398\u0398\u0398' + # + # Note: this is equivalent to unicode case folding; unicode normalization + # is a different step that is not required here. + # + # Final result should be uppercased, because it's later stored in an object + # (this avoid a conflict with Object.prototype members, + # most notably, `__proto__`) + # + return string.lower().upper() + + +LINK_OPEN_RE = re.compile(r"^\s]", flags=re.IGNORECASE) +LINK_CLOSE_RE = re.compile(r"^", flags=re.IGNORECASE) + + +def isLinkOpen(string: str) -> bool: + return bool(LINK_OPEN_RE.search(string)) + + +def isLinkClose(string: str) -> bool: + return bool(LINK_CLOSE_RE.search(string)) diff --git a/lib/markdown_it/helpers/__init__.py b/lib/markdown_it/helpers/__init__.py new file mode 100644 index 0000000..f4e2cd2 --- /dev/null +++ b/lib/markdown_it/helpers/__init__.py @@ -0,0 +1,6 @@ +"""Functions for parsing Links""" + +__all__ = ("parseLinkDestination", "parseLinkLabel", "parseLinkTitle") +from .parse_link_destination import parseLinkDestination +from .parse_link_label import parseLinkLabel +from .parse_link_title import parseLinkTitle diff --git a/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..2c0595b Binary files /dev/null and b/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc b/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc new file mode 100644 index 0000000..2fac45f Binary files /dev/null and b/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc differ diff --git a/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc b/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc new file mode 100644 index 0000000..e66d999 Binary files /dev/null and b/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc differ diff --git a/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc b/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc new file mode 100644 index 0000000..a30014a Binary files /dev/null and b/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc differ diff --git a/lib/markdown_it/helpers/parse_link_destination.py b/lib/markdown_it/helpers/parse_link_destination.py new file mode 100644 index 0000000..c98323c --- /dev/null +++ b/lib/markdown_it/helpers/parse_link_destination.py @@ -0,0 +1,83 @@ +""" +Parse link destination +""" + +from ..common.utils import charCodeAt, unescapeAll + + +class _Result: + __slots__ = ("ok", "pos", "str") + + def __init__(self) -> None: + self.ok = False + self.pos = 0 + self.str = "" + + +def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result: + start = pos + result = _Result() + + if charCodeAt(string, pos) == 0x3C: # /* < */ + pos += 1 + while pos < maximum: + code = charCodeAt(string, pos) + if code == 0x0A: # /* \n */) + return result + if code == 0x3C: # / * < * / + return result + if code == 0x3E: # /* > */) { + result.pos = pos + 1 + result.str = unescapeAll(string[start + 1 : pos]) + result.ok = True + return result + + if code == 0x5C and pos + 1 < maximum: # \ + pos += 2 + continue + + pos += 1 + + # no closing '>' + return result + + # this should be ... } else { ... branch + + level = 0 + while pos < maximum: + code = charCodeAt(string, pos) + + if code is None or code == 0x20: + break + + # ascii control characters + if code < 0x20 or code == 0x7F: + break + + if code == 0x5C and pos + 1 < maximum: + if charCodeAt(string, pos + 1) == 0x20: + break + pos += 2 + continue + + if code == 0x28: # /* ( */) + level += 1 + if level > 32: + return result + + if code == 0x29: # /* ) */) + if level == 0: + break + level -= 1 + + pos += 1 + + if start == pos: + return result + if level != 0: + return result + + result.str = unescapeAll(string[start:pos]) + result.pos = pos + result.ok = True + return result diff --git a/lib/markdown_it/helpers/parse_link_label.py b/lib/markdown_it/helpers/parse_link_label.py new file mode 100644 index 0000000..c80da5a --- /dev/null +++ b/lib/markdown_it/helpers/parse_link_label.py @@ -0,0 +1,44 @@ +""" +Parse link label + +this function assumes that first character ("[") already matches +returns the end of the label + +""" + +from markdown_it.rules_inline import StateInline + + +def parseLinkLabel(state: StateInline, start: int, disableNested: bool = False) -> int: + labelEnd = -1 + oldPos = state.pos + found = False + + state.pos = start + 1 + level = 1 + + while state.pos < state.posMax: + marker = state.src[state.pos] + if marker == "]": + level -= 1 + if level == 0: + found = True + break + + prevPos = state.pos + state.md.inline.skipToken(state) + if marker == "[": + if prevPos == state.pos - 1: + # increase level if we find text `[`, + # which is not a part of any token + level += 1 + elif disableNested: + state.pos = oldPos + return -1 + if found: + labelEnd = state.pos + + # restore old state + state.pos = oldPos + + return labelEnd diff --git a/lib/markdown_it/helpers/parse_link_title.py b/lib/markdown_it/helpers/parse_link_title.py new file mode 100644 index 0000000..a38ff0d --- /dev/null +++ b/lib/markdown_it/helpers/parse_link_title.py @@ -0,0 +1,75 @@ +"""Parse link title""" + +from ..common.utils import charCodeAt, unescapeAll + + +class _State: + __slots__ = ("can_continue", "marker", "ok", "pos", "str") + + def __init__(self) -> None: + self.ok = False + """if `true`, this is a valid link title""" + self.can_continue = False + """if `true`, this link can be continued on the next line""" + self.pos = 0 + """if `ok`, it's the position of the first character after the closing marker""" + self.str = "" + """if `ok`, it's the unescaped title""" + self.marker = 0 + """expected closing marker character code""" + + def __str__(self) -> str: + return self.str + + +def parseLinkTitle( + string: str, start: int, maximum: int, prev_state: _State | None = None +) -> _State: + """Parse link title within `str` in [start, max] range, + or continue previous parsing if `prev_state` is defined (equal to result of last execution). + """ + pos = start + state = _State() + + if prev_state is not None: + # this is a continuation of a previous parseLinkTitle call on the next line, + # used in reference links only + state.str = prev_state.str + state.marker = prev_state.marker + else: + if pos >= maximum: + return state + + marker = charCodeAt(string, pos) + + # /* " */ /* ' */ /* ( */ + if marker != 0x22 and marker != 0x27 and marker != 0x28: + return state + + start += 1 + pos += 1 + + # if opening marker is "(", switch it to closing marker ")" + if marker == 0x28: + marker = 0x29 + + state.marker = marker + + while pos < maximum: + code = charCodeAt(string, pos) + if code == state.marker: + state.pos = pos + 1 + state.str += unescapeAll(string[start:pos]) + state.ok = True + return state + elif code == 0x28 and state.marker == 0x29: # /* ( */ /* ) */ + return state + elif code == 0x5C and pos + 1 < maximum: # /* \ */ + pos += 1 + + pos += 1 + + # no closing marker found, but this link title may continue on the next line (for references) + state.can_continue = True + state.str += unescapeAll(string[start:pos]) + return state diff --git a/lib/markdown_it/main.py b/lib/markdown_it/main.py new file mode 100644 index 0000000..bf9fd18 --- /dev/null +++ b/lib/markdown_it/main.py @@ -0,0 +1,350 @@ +from __future__ import annotations + +from collections.abc import Callable, Generator, Iterable, Mapping, MutableMapping +from contextlib import contextmanager +from typing import Any, Literal, overload + +from . import helpers, presets +from .common import normalize_url, utils +from .parser_block import ParserBlock +from .parser_core import ParserCore +from .parser_inline import ParserInline +from .renderer import RendererHTML, RendererProtocol +from .rules_core.state_core import StateCore +from .token import Token +from .utils import EnvType, OptionsDict, OptionsType, PresetType + +try: + import linkify_it +except ModuleNotFoundError: + linkify_it = None + + +_PRESETS: dict[str, PresetType] = { + "default": presets.default.make(), + "js-default": presets.js_default.make(), + "zero": presets.zero.make(), + "commonmark": presets.commonmark.make(), + "gfm-like": presets.gfm_like.make(), +} + + +class MarkdownIt: + def __init__( + self, + config: str | PresetType = "commonmark", + options_update: Mapping[str, Any] | None = None, + *, + renderer_cls: Callable[[MarkdownIt], RendererProtocol] = RendererHTML, + ): + """Main parser class + + :param config: name of configuration to load or a pre-defined dictionary + :param options_update: dictionary that will be merged into ``config["options"]`` + :param renderer_cls: the class to load as the renderer: + ``self.renderer = renderer_cls(self) + """ + # add modules + self.utils = utils + self.helpers = helpers + + # initialise classes + self.inline = ParserInline() + self.block = ParserBlock() + self.core = ParserCore() + self.renderer = renderer_cls(self) + self.linkify = linkify_it.LinkifyIt() if linkify_it else None + + # set the configuration + if options_update and not isinstance(options_update, Mapping): + # catch signature change where renderer_cls was not used as a key-word + raise TypeError( + f"options_update should be a mapping: {options_update}" + "\n(Perhaps you intended this to be the renderer_cls?)" + ) + self.configure(config, options_update=options_update) + + def __repr__(self) -> str: + return f"{self.__class__.__module__}.{self.__class__.__name__}()" + + @overload + def __getitem__(self, name: Literal["inline"]) -> ParserInline: ... + + @overload + def __getitem__(self, name: Literal["block"]) -> ParserBlock: ... + + @overload + def __getitem__(self, name: Literal["core"]) -> ParserCore: ... + + @overload + def __getitem__(self, name: Literal["renderer"]) -> RendererProtocol: ... + + @overload + def __getitem__(self, name: str) -> Any: ... + + def __getitem__(self, name: str) -> Any: + return { + "inline": self.inline, + "block": self.block, + "core": self.core, + "renderer": self.renderer, + }[name] + + def set(self, options: OptionsType) -> None: + """Set parser options (in the same format as in constructor). + Probably, you will never need it, but you can change options after constructor call. + + __Note:__ To achieve the best possible performance, don't modify a + `markdown-it` instance options on the fly. If you need multiple configurations + it's best to create multiple instances and initialize each with separate config. + """ + self.options = OptionsDict(options) + + def configure( + self, presets: str | PresetType, options_update: Mapping[str, Any] | None = None + ) -> MarkdownIt: + """Batch load of all options and component settings. + This is an internal method, and you probably will not need it. + But if you will - see available presets and data structure + [here](https://github.com/markdown-it/markdown-it/tree/master/lib/presets) + + We strongly recommend to use presets instead of direct config loads. + That will give better compatibility with next versions. + """ + if isinstance(presets, str): + if presets not in _PRESETS: + raise KeyError(f"Wrong `markdown-it` preset '{presets}', check name") + config = _PRESETS[presets] + else: + config = presets + + if not config: + raise ValueError("Wrong `markdown-it` config, can't be empty") + + options = config.get("options", {}) or {} + if options_update: + options = {**options, **options_update} # type: ignore + + self.set(options) # type: ignore + + if "components" in config: + for name, component in config["components"].items(): + rules = component.get("rules", None) + if rules: + self[name].ruler.enableOnly(rules) + rules2 = component.get("rules2", None) + if rules2: + self[name].ruler2.enableOnly(rules2) + + return self + + def get_all_rules(self) -> dict[str, list[str]]: + """Return the names of all active rules.""" + rules = { + chain: self[chain].ruler.get_all_rules() + for chain in ["core", "block", "inline"] + } + rules["inline2"] = self.inline.ruler2.get_all_rules() + return rules + + def get_active_rules(self) -> dict[str, list[str]]: + """Return the names of all active rules.""" + rules = { + chain: self[chain].ruler.get_active_rules() + for chain in ["core", "block", "inline"] + } + rules["inline2"] = self.inline.ruler2.get_active_rules() + return rules + + def enable( + self, names: str | Iterable[str], ignoreInvalid: bool = False + ) -> MarkdownIt: + """Enable list or rules. (chainable) + + :param names: rule name or list of rule names to enable. + :param ignoreInvalid: set `true` to ignore errors when rule not found. + + It will automatically find appropriate components, + containing rules with given names. If rule not found, and `ignoreInvalid` + not set - throws exception. + + Example:: + + md = MarkdownIt().enable(['sub', 'sup']).disable('smartquotes') + + """ + result = [] + + if isinstance(names, str): + names = [names] + + for chain in ["core", "block", "inline"]: + result.extend(self[chain].ruler.enable(names, True)) + result.extend(self.inline.ruler2.enable(names, True)) + + missed = [name for name in names if name not in result] + if missed and not ignoreInvalid: + raise ValueError(f"MarkdownIt. Failed to enable unknown rule(s): {missed}") + + return self + + def disable( + self, names: str | Iterable[str], ignoreInvalid: bool = False + ) -> MarkdownIt: + """The same as [[MarkdownIt.enable]], but turn specified rules off. (chainable) + + :param names: rule name or list of rule names to disable. + :param ignoreInvalid: set `true` to ignore errors when rule not found. + + """ + result = [] + + if isinstance(names, str): + names = [names] + + for chain in ["core", "block", "inline"]: + result.extend(self[chain].ruler.disable(names, True)) + result.extend(self.inline.ruler2.disable(names, True)) + + missed = [name for name in names if name not in result] + if missed and not ignoreInvalid: + raise ValueError(f"MarkdownIt. Failed to disable unknown rule(s): {missed}") + return self + + @contextmanager + def reset_rules(self) -> Generator[None, None, None]: + """A context manager, that will reset the current enabled rules on exit.""" + chain_rules = self.get_active_rules() + yield + for chain, rules in chain_rules.items(): + if chain != "inline2": + self[chain].ruler.enableOnly(rules) + self.inline.ruler2.enableOnly(chain_rules["inline2"]) + + def add_render_rule( + self, name: str, function: Callable[..., Any], fmt: str = "html" + ) -> None: + """Add a rule for rendering a particular Token type. + + Only applied when ``renderer.__output__ == fmt`` + """ + if self.renderer.__output__ == fmt: + self.renderer.rules[name] = function.__get__(self.renderer) # type: ignore + + def use( + self, plugin: Callable[..., None], *params: Any, **options: Any + ) -> MarkdownIt: + """Load specified plugin with given params into current parser instance. (chainable) + + It's just a sugar to call `plugin(md, params)` with curring. + + Example:: + + def func(tokens, idx): + tokens[idx].content = tokens[idx].content.replace('foo', 'bar') + md = MarkdownIt().use(plugin, 'foo_replace', 'text', func) + + """ + plugin(self, *params, **options) + return self + + def parse(self, src: str, env: EnvType | None = None) -> list[Token]: + """Parse the source string to a token stream + + :param src: source string + :param env: environment sandbox + + Parse input string and return list of block tokens (special token type + "inline" will contain list of inline tokens). + + `env` is used to pass data between "distributed" rules and return additional + metadata like reference info, needed for the renderer. It also can be used to + inject data in specific cases. Usually, you will be ok to pass `{}`, + and then pass updated object to renderer. + """ + env = {} if env is None else env + if not isinstance(env, MutableMapping): + raise TypeError(f"Input data should be a MutableMapping, not {type(env)}") + if not isinstance(src, str): + raise TypeError(f"Input data should be a string, not {type(src)}") + state = StateCore(src, self, env) + self.core.process(state) + return state.tokens + + def render(self, src: str, env: EnvType | None = None) -> Any: + """Render markdown string into html. It does all magic for you :). + + :param src: source string + :param env: environment sandbox + :returns: The output of the loaded renderer + + `env` can be used to inject additional metadata (`{}` by default). + But you will not need it with high probability. See also comment + in [[MarkdownIt.parse]]. + """ + env = {} if env is None else env + return self.renderer.render(self.parse(src, env), self.options, env) + + def parseInline(self, src: str, env: EnvType | None = None) -> list[Token]: + """The same as [[MarkdownIt.parse]] but skip all block rules. + + :param src: source string + :param env: environment sandbox + + It returns the + block tokens list with the single `inline` element, containing parsed inline + tokens in `children` property. Also updates `env` object. + """ + env = {} if env is None else env + if not isinstance(env, MutableMapping): + raise TypeError(f"Input data should be an MutableMapping, not {type(env)}") + if not isinstance(src, str): + raise TypeError(f"Input data should be a string, not {type(src)}") + state = StateCore(src, self, env) + state.inlineMode = True + self.core.process(state) + return state.tokens + + def renderInline(self, src: str, env: EnvType | None = None) -> Any: + """Similar to [[MarkdownIt.render]] but for single paragraph content. + + :param src: source string + :param env: environment sandbox + + Similar to [[MarkdownIt.render]] but for single paragraph content. Result + will NOT be wrapped into `

` tags. + """ + env = {} if env is None else env + return self.renderer.render(self.parseInline(src, env), self.options, env) + + # link methods + + def validateLink(self, url: str) -> bool: + """Validate if the URL link is allowed in output. + + This validator can prohibit more than really needed to prevent XSS. + It's a tradeoff to keep code simple and to be secure by default. + + Note: the url should be normalized at this point, and existing entities decoded. + """ + return normalize_url.validateLink(url) + + def normalizeLink(self, url: str) -> str: + """Normalize destination URLs in links + + :: + + [label]: destination 'title' + ^^^^^^^^^^^ + """ + return normalize_url.normalizeLink(url) + + def normalizeLinkText(self, link: str) -> str: + """Normalize autolink content + + :: + + + ~~~~~~~~~~~ + """ + return normalize_url.normalizeLinkText(link) diff --git a/lib/markdown_it/parser_block.py b/lib/markdown_it/parser_block.py new file mode 100644 index 0000000..50a7184 --- /dev/null +++ b/lib/markdown_it/parser_block.py @@ -0,0 +1,113 @@ +"""Block-level tokenizer.""" + +from __future__ import annotations + +from collections.abc import Callable +import logging +from typing import TYPE_CHECKING + +from . import rules_block +from .ruler import Ruler +from .rules_block.state_block import StateBlock +from .token import Token +from .utils import EnvType + +if TYPE_CHECKING: + from markdown_it import MarkdownIt + +LOGGER = logging.getLogger(__name__) + + +RuleFuncBlockType = Callable[[StateBlock, int, int, bool], bool] +"""(state: StateBlock, startLine: int, endLine: int, silent: bool) -> matched: bool) + +`silent` disables token generation, useful for lookahead. +""" + +_rules: list[tuple[str, RuleFuncBlockType, list[str]]] = [ + # First 2 params - rule name & source. Secondary array - list of rules, + # which can be terminated by this one. + ("table", rules_block.table, ["paragraph", "reference"]), + ("code", rules_block.code, []), + ("fence", rules_block.fence, ["paragraph", "reference", "blockquote", "list"]), + ( + "blockquote", + rules_block.blockquote, + ["paragraph", "reference", "blockquote", "list"], + ), + ("hr", rules_block.hr, ["paragraph", "reference", "blockquote", "list"]), + ("list", rules_block.list_block, ["paragraph", "reference", "blockquote"]), + ("reference", rules_block.reference, []), + ("html_block", rules_block.html_block, ["paragraph", "reference", "blockquote"]), + ("heading", rules_block.heading, ["paragraph", "reference", "blockquote"]), + ("lheading", rules_block.lheading, []), + ("paragraph", rules_block.paragraph, []), +] + + +class ParserBlock: + """ + ParserBlock#ruler -> Ruler + + [[Ruler]] instance. Keep configuration of block rules. + """ + + def __init__(self) -> None: + self.ruler = Ruler[RuleFuncBlockType]() + for name, rule, alt in _rules: + self.ruler.push(name, rule, {"alt": alt}) + + def tokenize(self, state: StateBlock, startLine: int, endLine: int) -> None: + """Generate tokens for input range.""" + rules = self.ruler.getRules("") + line = startLine + maxNesting = state.md.options.maxNesting + hasEmptyLines = False + + while line < endLine: + state.line = line = state.skipEmptyLines(line) + if line >= endLine: + break + if state.sCount[line] < state.blkIndent: + # Termination condition for nested calls. + # Nested calls currently used for blockquotes & lists + break + if state.level >= maxNesting: + # If nesting level exceeded - skip tail to the end. + # That's not ordinary situation and we should not care about content. + state.line = endLine + break + + # Try all possible rules. + # On success, rule should: + # - update `state.line` + # - update `state.tokens` + # - return True + for rule in rules: + if rule(state, line, endLine, False): + break + + # set state.tight if we had an empty line before current tag + # i.e. latest empty line should not count + state.tight = not hasEmptyLines + + line = state.line + + # paragraph might "eat" one newline after it in nested lists + if (line - 1) < endLine and state.isEmpty(line - 1): + hasEmptyLines = True + + if line < endLine and state.isEmpty(line): + hasEmptyLines = True + line += 1 + state.line = line + + def parse( + self, src: str, md: MarkdownIt, env: EnvType, outTokens: list[Token] + ) -> list[Token] | None: + """Process input string and push block tokens into `outTokens`.""" + if not src: + return None + state = StateBlock(src, md, env, outTokens) + self.tokenize(state, state.line, state.lineMax) + return state.tokens diff --git a/lib/markdown_it/parser_core.py b/lib/markdown_it/parser_core.py new file mode 100644 index 0000000..8f5b921 --- /dev/null +++ b/lib/markdown_it/parser_core.py @@ -0,0 +1,46 @@ +""" +* class Core +* +* Top-level rules executor. Glues block/inline parsers and does intermediate +* transformations. +""" + +from __future__ import annotations + +from collections.abc import Callable + +from .ruler import Ruler +from .rules_core import ( + block, + inline, + linkify, + normalize, + replace, + smartquotes, + text_join, +) +from .rules_core.state_core import StateCore + +RuleFuncCoreType = Callable[[StateCore], None] + +_rules: list[tuple[str, RuleFuncCoreType]] = [ + ("normalize", normalize), + ("block", block), + ("inline", inline), + ("linkify", linkify), + ("replacements", replace), + ("smartquotes", smartquotes), + ("text_join", text_join), +] + + +class ParserCore: + def __init__(self) -> None: + self.ruler = Ruler[RuleFuncCoreType]() + for name, rule in _rules: + self.ruler.push(name, rule) + + def process(self, state: StateCore) -> None: + """Executes core chain rules.""" + for rule in self.ruler.getRules(""): + rule(state) diff --git a/lib/markdown_it/parser_inline.py b/lib/markdown_it/parser_inline.py new file mode 100644 index 0000000..26ec2e6 --- /dev/null +++ b/lib/markdown_it/parser_inline.py @@ -0,0 +1,148 @@ +"""Tokenizes paragraph content.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from . import rules_inline +from .ruler import Ruler +from .rules_inline.state_inline import StateInline +from .token import Token +from .utils import EnvType + +if TYPE_CHECKING: + from markdown_it import MarkdownIt + + +# Parser rules +RuleFuncInlineType = Callable[[StateInline, bool], bool] +"""(state: StateInline, silent: bool) -> matched: bool) + +`silent` disables token generation, useful for lookahead. +""" +_rules: list[tuple[str, RuleFuncInlineType]] = [ + ("text", rules_inline.text), + ("linkify", rules_inline.linkify), + ("newline", rules_inline.newline), + ("escape", rules_inline.escape), + ("backticks", rules_inline.backtick), + ("strikethrough", rules_inline.strikethrough.tokenize), + ("emphasis", rules_inline.emphasis.tokenize), + ("link", rules_inline.link), + ("image", rules_inline.image), + ("autolink", rules_inline.autolink), + ("html_inline", rules_inline.html_inline), + ("entity", rules_inline.entity), +] + +# Note `rule2` ruleset was created specifically for emphasis/strikethrough +# post-processing and may be changed in the future. +# +# Don't use this for anything except pairs (plugins working with `balance_pairs`). +# +RuleFuncInline2Type = Callable[[StateInline], None] +_rules2: list[tuple[str, RuleFuncInline2Type]] = [ + ("balance_pairs", rules_inline.link_pairs), + ("strikethrough", rules_inline.strikethrough.postProcess), + ("emphasis", rules_inline.emphasis.postProcess), + # rules for pairs separate '**' into its own text tokens, which may be left unused, + # rule below merges unused segments back with the rest of the text + ("fragments_join", rules_inline.fragments_join), +] + + +class ParserInline: + def __init__(self) -> None: + self.ruler = Ruler[RuleFuncInlineType]() + for name, rule in _rules: + self.ruler.push(name, rule) + # Second ruler used for post-processing (e.g. in emphasis-like rules) + self.ruler2 = Ruler[RuleFuncInline2Type]() + for name, rule2 in _rules2: + self.ruler2.push(name, rule2) + + def skipToken(self, state: StateInline) -> None: + """Skip single token by running all rules in validation mode; + returns `True` if any rule reported success + """ + ok = False + pos = state.pos + rules = self.ruler.getRules("") + maxNesting = state.md.options["maxNesting"] + cache = state.cache + + if pos in cache: + state.pos = cache[pos] + return + + if state.level < maxNesting: + for rule in rules: + # Increment state.level and decrement it later to limit recursion. + # It's harmless to do here, because no tokens are created. + # But ideally, we'd need a separate private state variable for this purpose. + state.level += 1 + ok = rule(state, True) + state.level -= 1 + if ok: + break + else: + # Too much nesting, just skip until the end of the paragraph. + # + # NOTE: this will cause links to behave incorrectly in the following case, + # when an amount of `[` is exactly equal to `maxNesting + 1`: + # + # [[[[[[[[[[[[[[[[[[[[[foo]() + # + # TODO: remove this workaround when CM standard will allow nested links + # (we can replace it by preventing links from being parsed in + # validation mode) + # + state.pos = state.posMax + + if not ok: + state.pos += 1 + cache[pos] = state.pos + + def tokenize(self, state: StateInline) -> None: + """Generate tokens for input range.""" + ok = False + rules = self.ruler.getRules("") + end = state.posMax + maxNesting = state.md.options["maxNesting"] + + while state.pos < end: + # Try all possible rules. + # On success, rule should: + # + # - update `state.pos` + # - update `state.tokens` + # - return true + + if state.level < maxNesting: + for rule in rules: + ok = rule(state, False) + if ok: + break + + if ok: + if state.pos >= end: + break + continue + + state.pending += state.src[state.pos] + state.pos += 1 + + if state.pending: + state.pushPending() + + def parse( + self, src: str, md: MarkdownIt, env: EnvType, tokens: list[Token] + ) -> list[Token]: + """Process input string and push inline tokens into `tokens`""" + state = StateInline(src, md, env, tokens) + self.tokenize(state) + rules2 = self.ruler2.getRules("") + for rule in rules2: + rule(state) + return state.tokens diff --git a/lib/markdown_it/port.yaml b/lib/markdown_it/port.yaml new file mode 100644 index 0000000..ce2dde9 --- /dev/null +++ b/lib/markdown_it/port.yaml @@ -0,0 +1,48 @@ +- package: markdown-it/markdown-it + version: 14.1.0 + commit: 0fe7ccb4b7f30236fb05f623be6924961d296d3d + date: Mar 19, 2024 + notes: + - Rename variables that use python built-in names, e.g. + - `max` -> `maximum` + - `len` -> `length` + - `str` -> `string` + - | + Convert JS `for` loops to `while` loops + this is generally the main difference between the codes, + because in python you can't do e.g. `for {i=1;i PresetType: + config = commonmark.make() + config["components"]["core"]["rules"].append("linkify") + config["components"]["block"]["rules"].append("table") + config["components"]["inline"]["rules"].extend(["strikethrough", "linkify"]) + config["components"]["inline"]["rules2"].append("strikethrough") + config["options"]["linkify"] = True + config["options"]["html"] = True + return config diff --git a/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..e46f728 Binary files /dev/null and b/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc b/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc new file mode 100644 index 0000000..80650fa Binary files /dev/null and b/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc differ diff --git a/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc b/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc new file mode 100644 index 0000000..0d4f15b Binary files /dev/null and b/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc differ diff --git a/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc b/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc new file mode 100644 index 0000000..e6b0aab Binary files /dev/null and b/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc differ diff --git a/lib/markdown_it/presets/commonmark.py b/lib/markdown_it/presets/commonmark.py new file mode 100644 index 0000000..ed0de0f --- /dev/null +++ b/lib/markdown_it/presets/commonmark.py @@ -0,0 +1,75 @@ +"""Commonmark default options. + +This differs to presets.default, +primarily in that it allows HTML and does not enable components: + +- block: table +- inline: strikethrough +""" + +from ..utils import PresetType + + +def make() -> PresetType: + return { + "options": { + "maxNesting": 20, # Internal protection, recursion limit + "html": True, # Enable HTML tags in source, + # this is just a shorthand for .enable(["html_inline", "html_block"]) + # used by the linkify rule: + "linkify": False, # autoconvert URL-like texts to links + # used by the replacements and smartquotes rules + # Enable some language-neutral replacements + quotes beautification + "typographer": False, + # used by the smartquotes rule: + # Double + single quotes replacement pairs, when typographer enabled, + # and smartquotes on. Could be either a String or an Array. + # + # For example, you can use '«»„“' for Russian, '„“‚‘' for German, + # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp). + "quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */ + # Renderer specific; these options are used directly in the HTML renderer + "xhtmlOut": True, # Use '/' to close single tags (
) + "breaks": False, # Convert '\n' in paragraphs into
+ "langPrefix": "language-", # CSS language prefix for fenced blocks + # Highlighter function. Should return escaped HTML, + # or '' if the source string is not changed and should be escaped externally. + # If result starts with PresetType: + return { + "options": { + "maxNesting": 100, # Internal protection, recursion limit + "html": False, # Enable HTML tags in source + # this is just a shorthand for .disable(["html_inline", "html_block"]) + # used by the linkify rule: + "linkify": False, # autoconvert URL-like texts to links + # used by the replacements and smartquotes rules: + # Enable some language-neutral replacements + quotes beautification + "typographer": False, + # used by the smartquotes rule: + # Double + single quotes replacement pairs, when typographer enabled, + # and smartquotes on. Could be either a String or an Array. + # For example, you can use '«»„“' for Russian, '„“‚‘' for German, + # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp). + "quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */ + # Renderer specific; these options are used directly in the HTML renderer + "xhtmlOut": False, # Use '/' to close single tags (
) + "breaks": False, # Convert '\n' in paragraphs into
+ "langPrefix": "language-", # CSS language prefix for fenced blocks + # Highlighter function. Should return escaped HTML, + # or '' if the source string is not changed and should be escaped externally. + # If result starts with PresetType: + return { + "options": { + "maxNesting": 20, # Internal protection, recursion limit + "html": False, # Enable HTML tags in source + # this is just a shorthand for .disable(["html_inline", "html_block"]) + # used by the linkify rule: + "linkify": False, # autoconvert URL-like texts to links + # used by the replacements and smartquotes rules: + # Enable some language-neutral replacements + quotes beautification + "typographer": False, + # used by the smartquotes rule: + # Double + single quotes replacement pairs, when typographer enabled, + # and smartquotes on. Could be either a String or an Array. + # For example, you can use '«»„“' for Russian, '„“‚‘' for German, + # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp). + "quotes": "\u201c\u201d\u2018\u2019", # /* “”‘’ */ + # Renderer specific; these options are used directly in the HTML renderer + "xhtmlOut": False, # Use '/' to close single tags (
) + "breaks": False, # Convert '\n' in paragraphs into
+ "langPrefix": "language-", # CSS language prefix for fenced blocks + # Highlighter function. Should return escaped HTML, + # or '' if the source string is not changed and should be escaped externally. + # If result starts with Any: ... + + +class RendererHTML(RendererProtocol): + """Contains render rules for tokens. Can be updated and extended. + + Example: + + Each rule is called as independent static function with fixed signature: + + :: + + class Renderer: + def token_type_name(self, tokens, idx, options, env) { + # ... + return renderedHTML + + :: + + class CustomRenderer(RendererHTML): + def strong_open(self, tokens, idx, options, env): + return '' + def strong_close(self, tokens, idx, options, env): + return '' + + md = MarkdownIt(renderer_cls=CustomRenderer) + + result = md.render(...) + + See https://github.com/markdown-it/markdown-it/blob/master/lib/renderer.js + for more details and examples. + """ + + __output__ = "html" + + def __init__(self, parser: Any = None): + self.rules = { + k: v + for k, v in inspect.getmembers(self, predicate=inspect.ismethod) + if not (k.startswith("render") or k.startswith("_")) + } + + def render( + self, tokens: Sequence[Token], options: OptionsDict, env: EnvType + ) -> str: + """Takes token stream and generates HTML. + + :param tokens: list on block tokens to render + :param options: params of parser instance + :param env: additional data from parsed input + + """ + result = "" + + for i, token in enumerate(tokens): + if token.type == "inline": + if token.children: + result += self.renderInline(token.children, options, env) + elif token.type in self.rules: + result += self.rules[token.type](tokens, i, options, env) + else: + result += self.renderToken(tokens, i, options, env) + + return result + + def renderInline( + self, tokens: Sequence[Token], options: OptionsDict, env: EnvType + ) -> str: + """The same as ``render``, but for single token of `inline` type. + + :param tokens: list on block tokens to render + :param options: params of parser instance + :param env: additional data from parsed input (references, for example) + """ + result = "" + + for i, token in enumerate(tokens): + if token.type in self.rules: + result += self.rules[token.type](tokens, i, options, env) + else: + result += self.renderToken(tokens, i, options, env) + + return result + + def renderToken( + self, + tokens: Sequence[Token], + idx: int, + options: OptionsDict, + env: EnvType, + ) -> str: + """Default token renderer. + + Can be overridden by custom function + + :param idx: token index to render + :param options: params of parser instance + """ + result = "" + needLf = False + token = tokens[idx] + + # Tight list paragraphs + if token.hidden: + return "" + + # Insert a newline between hidden paragraph and subsequent opening + # block-level tag. + # + # For example, here we should insert a newline before blockquote: + # - a + # > + # + if token.block and token.nesting != -1 and idx and tokens[idx - 1].hidden: + result += "\n" + + # Add token name, e.g. ``. + # + needLf = False + + result += ">\n" if needLf else ">" + + return result + + @staticmethod + def renderAttrs(token: Token) -> str: + """Render token attributes to string.""" + result = "" + + for key, value in token.attrItems(): + result += " " + escapeHtml(key) + '="' + escapeHtml(str(value)) + '"' + + return result + + def renderInlineAsText( + self, + tokens: Sequence[Token] | None, + options: OptionsDict, + env: EnvType, + ) -> str: + """Special kludge for image `alt` attributes to conform CommonMark spec. + + Don't try to use it! Spec requires to show `alt` content with stripped markup, + instead of simple escaping. + + :param tokens: list on block tokens to render + :param options: params of parser instance + :param env: additional data from parsed input + """ + result = "" + + for token in tokens or []: + if token.type == "text": + result += token.content + elif token.type == "image": + if token.children: + result += self.renderInlineAsText(token.children, options, env) + elif token.type == "softbreak": + result += "\n" + + return result + + ################################################### + + def code_inline( + self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType + ) -> str: + token = tokens[idx] + return ( + "" + + escapeHtml(tokens[idx].content) + + "" + ) + + def code_block( + self, + tokens: Sequence[Token], + idx: int, + options: OptionsDict, + env: EnvType, + ) -> str: + token = tokens[idx] + + return ( + "" + + escapeHtml(tokens[idx].content) + + "\n" + ) + + def fence( + self, + tokens: Sequence[Token], + idx: int, + options: OptionsDict, + env: EnvType, + ) -> str: + token = tokens[idx] + info = unescapeAll(token.info).strip() if token.info else "" + langName = "" + langAttrs = "" + + if info: + arr = info.split(maxsplit=1) + langName = arr[0] + if len(arr) == 2: + langAttrs = arr[1] + + if options.highlight: + highlighted = options.highlight( + token.content, langName, langAttrs + ) or escapeHtml(token.content) + else: + highlighted = escapeHtml(token.content) + + if highlighted.startswith("" + + highlighted + + "\n" + ) + + return ( + "

"
+            + highlighted
+            + "
\n" + ) + + def image( + self, + tokens: Sequence[Token], + idx: int, + options: OptionsDict, + env: EnvType, + ) -> str: + token = tokens[idx] + + # "alt" attr MUST be set, even if empty. Because it's mandatory and + # should be placed on proper position for tests. + if token.children: + token.attrSet("alt", self.renderInlineAsText(token.children, options, env)) + else: + token.attrSet("alt", "") + + return self.renderToken(tokens, idx, options, env) + + def hardbreak( + self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType + ) -> str: + return "
\n" if options.xhtmlOut else "
\n" + + def softbreak( + self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType + ) -> str: + return ( + ("
\n" if options.xhtmlOut else "
\n") if options.breaks else "\n" + ) + + def text( + self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType + ) -> str: + return escapeHtml(tokens[idx].content) + + def html_block( + self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType + ) -> str: + return tokens[idx].content + + def html_inline( + self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType + ) -> str: + return tokens[idx].content diff --git a/lib/markdown_it/ruler.py b/lib/markdown_it/ruler.py new file mode 100644 index 0000000..91ab580 --- /dev/null +++ b/lib/markdown_it/ruler.py @@ -0,0 +1,275 @@ +""" +class Ruler + +Helper class, used by [[MarkdownIt#core]], [[MarkdownIt#block]] and +[[MarkdownIt#inline]] to manage sequences of functions (rules): + +- keep rules in defined order +- assign the name to each rule +- enable/disable rules +- add/replace rules +- allow assign rules to additional named chains (in the same) +- caching lists of active rules + +You will not need use this class directly until write plugins. For simple +rules control use [[MarkdownIt.disable]], [[MarkdownIt.enable]] and +[[MarkdownIt.use]]. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Generic, TypedDict, TypeVar +import warnings + +from .utils import EnvType + +if TYPE_CHECKING: + from markdown_it import MarkdownIt + + +class StateBase: + def __init__(self, src: str, md: MarkdownIt, env: EnvType): + self.src = src + self.env = env + self.md = md + + @property + def src(self) -> str: + return self._src + + @src.setter + def src(self, value: str) -> None: + self._src = value + self._srcCharCode: tuple[int, ...] | None = None + + @property + def srcCharCode(self) -> tuple[int, ...]: + warnings.warn( + "StateBase.srcCharCode is deprecated. Use StateBase.src instead.", + DeprecationWarning, + stacklevel=2, + ) + if self._srcCharCode is None: + self._srcCharCode = tuple(ord(c) for c in self._src) + return self._srcCharCode + + +class RuleOptionsType(TypedDict, total=False): + alt: list[str] + + +RuleFuncTv = TypeVar("RuleFuncTv") +"""A rule function, whose signature is dependent on the state type.""" + + +@dataclass(slots=True) +class Rule(Generic[RuleFuncTv]): + name: str + enabled: bool + fn: RuleFuncTv = field(repr=False) + alt: list[str] + + +class Ruler(Generic[RuleFuncTv]): + def __init__(self) -> None: + # List of added rules. + self.__rules__: list[Rule[RuleFuncTv]] = [] + # Cached rule chains. + # First level - chain name, '' for default. + # Second level - diginal anchor for fast filtering by charcodes. + self.__cache__: dict[str, list[RuleFuncTv]] | None = None + + def __find__(self, name: str) -> int: + """Find rule index by name""" + for i, rule in enumerate(self.__rules__): + if rule.name == name: + return i + return -1 + + def __compile__(self) -> None: + """Build rules lookup cache""" + chains = {""} + # collect unique names + for rule in self.__rules__: + if not rule.enabled: + continue + for name in rule.alt: + chains.add(name) + self.__cache__ = {} + for chain in chains: + self.__cache__[chain] = [] + for rule in self.__rules__: + if not rule.enabled: + continue + if chain and (chain not in rule.alt): + continue + self.__cache__[chain].append(rule.fn) + + def at( + self, ruleName: str, fn: RuleFuncTv, options: RuleOptionsType | None = None + ) -> None: + """Replace rule by name with new function & options. + + :param ruleName: rule name to replace. + :param fn: new rule function. + :param options: new rule options (not mandatory). + :raises: KeyError if name not found + """ + index = self.__find__(ruleName) + options = options or {} + if index == -1: + raise KeyError(f"Parser rule not found: {ruleName}") + self.__rules__[index].fn = fn + self.__rules__[index].alt = options.get("alt", []) + self.__cache__ = None + + def before( + self, + beforeName: str, + ruleName: str, + fn: RuleFuncTv, + options: RuleOptionsType | None = None, + ) -> None: + """Add new rule to chain before one with given name. + + :param beforeName: new rule will be added before this one. + :param ruleName: new rule will be added before this one. + :param fn: new rule function. + :param options: new rule options (not mandatory). + :raises: KeyError if name not found + """ + index = self.__find__(beforeName) + options = options or {} + if index == -1: + raise KeyError(f"Parser rule not found: {beforeName}") + self.__rules__.insert( + index, Rule[RuleFuncTv](ruleName, True, fn, options.get("alt", [])) + ) + self.__cache__ = None + + def after( + self, + afterName: str, + ruleName: str, + fn: RuleFuncTv, + options: RuleOptionsType | None = None, + ) -> None: + """Add new rule to chain after one with given name. + + :param afterName: new rule will be added after this one. + :param ruleName: new rule will be added after this one. + :param fn: new rule function. + :param options: new rule options (not mandatory). + :raises: KeyError if name not found + """ + index = self.__find__(afterName) + options = options or {} + if index == -1: + raise KeyError(f"Parser rule not found: {afterName}") + self.__rules__.insert( + index + 1, Rule[RuleFuncTv](ruleName, True, fn, options.get("alt", [])) + ) + self.__cache__ = None + + def push( + self, ruleName: str, fn: RuleFuncTv, options: RuleOptionsType | None = None + ) -> None: + """Push new rule to the end of chain. + + :param ruleName: new rule will be added to the end of chain. + :param fn: new rule function. + :param options: new rule options (not mandatory). + + """ + self.__rules__.append( + Rule[RuleFuncTv](ruleName, True, fn, (options or {}).get("alt", [])) + ) + self.__cache__ = None + + def enable( + self, names: str | Iterable[str], ignoreInvalid: bool = False + ) -> list[str]: + """Enable rules with given names. + + :param names: name or list of rule names to enable. + :param ignoreInvalid: ignore errors when rule not found + :raises: KeyError if name not found and not ignoreInvalid + :return: list of found rule names + """ + if isinstance(names, str): + names = [names] + result: list[str] = [] + for name in names: + idx = self.__find__(name) + if (idx < 0) and ignoreInvalid: + continue + if (idx < 0) and not ignoreInvalid: + raise KeyError(f"Rules manager: invalid rule name {name}") + self.__rules__[idx].enabled = True + result.append(name) + self.__cache__ = None + return result + + def enableOnly( + self, names: str | Iterable[str], ignoreInvalid: bool = False + ) -> list[str]: + """Enable rules with given names, and disable everything else. + + :param names: name or list of rule names to enable. + :param ignoreInvalid: ignore errors when rule not found + :raises: KeyError if name not found and not ignoreInvalid + :return: list of found rule names + """ + if isinstance(names, str): + names = [names] + for rule in self.__rules__: + rule.enabled = False + return self.enable(names, ignoreInvalid) + + def disable( + self, names: str | Iterable[str], ignoreInvalid: bool = False + ) -> list[str]: + """Disable rules with given names. + + :param names: name or list of rule names to enable. + :param ignoreInvalid: ignore errors when rule not found + :raises: KeyError if name not found and not ignoreInvalid + :return: list of found rule names + """ + if isinstance(names, str): + names = [names] + result = [] + for name in names: + idx = self.__find__(name) + if (idx < 0) and ignoreInvalid: + continue + if (idx < 0) and not ignoreInvalid: + raise KeyError(f"Rules manager: invalid rule name {name}") + self.__rules__[idx].enabled = False + result.append(name) + self.__cache__ = None + return result + + def getRules(self, chainName: str = "") -> list[RuleFuncTv]: + """Return array of active functions (rules) for given chain name. + It analyzes rules configuration, compiles caches if not exists and returns result. + + Default chain name is `''` (empty string). It can't be skipped. + That's done intentionally, to keep signature monomorphic for high speed. + + """ + if self.__cache__ is None: + self.__compile__() + assert self.__cache__ is not None + # Chain can be empty, if rules disabled. But we still have to return Array. + return self.__cache__.get(chainName, []) or [] + + def get_all_rules(self) -> list[str]: + """Return all available rule names.""" + return [r.name for r in self.__rules__] + + def get_active_rules(self) -> list[str]: + """Return the active rule names.""" + return [r.name for r in self.__rules__ if r.enabled] diff --git a/lib/markdown_it/rules_block/__init__.py b/lib/markdown_it/rules_block/__init__.py new file mode 100644 index 0000000..517da23 --- /dev/null +++ b/lib/markdown_it/rules_block/__init__.py @@ -0,0 +1,27 @@ +__all__ = ( + "StateBlock", + "blockquote", + "code", + "fence", + "heading", + "hr", + "html_block", + "lheading", + "list_block", + "paragraph", + "reference", + "table", +) + +from .blockquote import blockquote +from .code import code +from .fence import fence +from .heading import heading +from .hr import hr +from .html_block import html_block +from .lheading import lheading +from .list import list_block +from .paragraph import paragraph +from .reference import reference +from .state_block import StateBlock +from .table import table diff --git a/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..ff71ac5 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc new file mode 100644 index 0000000..69140c2 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc new file mode 100644 index 0000000..889656b Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc new file mode 100644 index 0000000..c3e9215 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc new file mode 100644 index 0000000..12af1dd Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc new file mode 100644 index 0000000..9203733 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc new file mode 100644 index 0000000..f938621 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc new file mode 100644 index 0000000..86bd1e0 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc new file mode 100644 index 0000000..e9c03f7 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc new file mode 100644 index 0000000..aca8006 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc new file mode 100644 index 0000000..05b5e88 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc new file mode 100644 index 0000000..1b7ce84 Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc b/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc new file mode 100644 index 0000000..b72dd2a Binary files /dev/null and b/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_block/blockquote.py b/lib/markdown_it/rules_block/blockquote.py new file mode 100644 index 0000000..0c9081b --- /dev/null +++ b/lib/markdown_it/rules_block/blockquote.py @@ -0,0 +1,299 @@ +# Block quotes +from __future__ import annotations + +import logging + +from ..common.utils import isStrSpace +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def blockquote(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug( + "entering blockquote: %s, %s, %s, %s", state, startLine, endLine, silent + ) + + oldLineMax = state.lineMax + pos = state.bMarks[startLine] + state.tShift[startLine] + max = state.eMarks[startLine] + + if state.is_code_block(startLine): + return False + + # check the block quote marker + try: + if state.src[pos] != ">": + return False + except IndexError: + return False + pos += 1 + + # we know that it's going to be a valid blockquote, + # so no point trying to find the end of it in silent mode + if silent: + return True + + # set offset past spaces and ">" + initial = offset = state.sCount[startLine] + 1 + + try: + second_char: str | None = state.src[pos] + except IndexError: + second_char = None + + # skip one optional space after '>' + if second_char == " ": + # ' > test ' + # ^ -- position start of line here: + pos += 1 + initial += 1 + offset += 1 + adjustTab = False + spaceAfterMarker = True + elif second_char == "\t": + spaceAfterMarker = True + + if (state.bsCount[startLine] + offset) % 4 == 3: + # ' >\t test ' + # ^ -- position start of line here (tab has width==1) + pos += 1 + initial += 1 + offset += 1 + adjustTab = False + else: + # ' >\t test ' + # ^ -- position start of line here + shift bsCount slightly + # to make extra space appear + adjustTab = True + + else: + spaceAfterMarker = False + + oldBMarks = [state.bMarks[startLine]] + state.bMarks[startLine] = pos + + while pos < max: + ch = state.src[pos] + + if isStrSpace(ch): + if ch == "\t": + offset += ( + 4 + - (offset + state.bsCount[startLine] + (1 if adjustTab else 0)) % 4 + ) + else: + offset += 1 + + else: + break + + pos += 1 + + oldBSCount = [state.bsCount[startLine]] + state.bsCount[startLine] = ( + state.sCount[startLine] + 1 + (1 if spaceAfterMarker else 0) + ) + + lastLineEmpty = pos >= max + + oldSCount = [state.sCount[startLine]] + state.sCount[startLine] = offset - initial + + oldTShift = [state.tShift[startLine]] + state.tShift[startLine] = pos - state.bMarks[startLine] + + terminatorRules = state.md.block.ruler.getRules("blockquote") + + oldParentType = state.parentType + state.parentType = "blockquote" + + # Search the end of the block + # + # Block ends with either: + # 1. an empty line outside: + # ``` + # > test + # + # ``` + # 2. an empty line inside: + # ``` + # > + # test + # ``` + # 3. another tag: + # ``` + # > test + # - - - + # ``` + + # for (nextLine = startLine + 1; nextLine < endLine; nextLine++) { + nextLine = startLine + 1 + while nextLine < endLine: + # check if it's outdented, i.e. it's inside list item and indented + # less than said list item: + # + # ``` + # 1. anything + # > current blockquote + # 2. checking this line + # ``` + isOutdented = state.sCount[nextLine] < state.blkIndent + + pos = state.bMarks[nextLine] + state.tShift[nextLine] + max = state.eMarks[nextLine] + + if pos >= max: + # Case 1: line is not inside the blockquote, and this line is empty. + break + + evaluatesTrue = state.src[pos] == ">" and not isOutdented + pos += 1 + if evaluatesTrue: + # This line is inside the blockquote. + + # set offset past spaces and ">" + initial = offset = state.sCount[nextLine] + 1 + + try: + next_char: str | None = state.src[pos] + except IndexError: + next_char = None + + # skip one optional space after '>' + if next_char == " ": + # ' > test ' + # ^ -- position start of line here: + pos += 1 + initial += 1 + offset += 1 + adjustTab = False + spaceAfterMarker = True + elif next_char == "\t": + spaceAfterMarker = True + + if (state.bsCount[nextLine] + offset) % 4 == 3: + # ' >\t test ' + # ^ -- position start of line here (tab has width==1) + pos += 1 + initial += 1 + offset += 1 + adjustTab = False + else: + # ' >\t test ' + # ^ -- position start of line here + shift bsCount slightly + # to make extra space appear + adjustTab = True + + else: + spaceAfterMarker = False + + oldBMarks.append(state.bMarks[nextLine]) + state.bMarks[nextLine] = pos + + while pos < max: + ch = state.src[pos] + + if isStrSpace(ch): + if ch == "\t": + offset += ( + 4 + - ( + offset + + state.bsCount[nextLine] + + (1 if adjustTab else 0) + ) + % 4 + ) + else: + offset += 1 + else: + break + + pos += 1 + + lastLineEmpty = pos >= max + + oldBSCount.append(state.bsCount[nextLine]) + state.bsCount[nextLine] = ( + state.sCount[nextLine] + 1 + (1 if spaceAfterMarker else 0) + ) + + oldSCount.append(state.sCount[nextLine]) + state.sCount[nextLine] = offset - initial + + oldTShift.append(state.tShift[nextLine]) + state.tShift[nextLine] = pos - state.bMarks[nextLine] + + nextLine += 1 + continue + + # Case 2: line is not inside the blockquote, and the last line was empty. + if lastLineEmpty: + break + + # Case 3: another tag found. + terminate = False + + for terminatorRule in terminatorRules: + if terminatorRule(state, nextLine, endLine, True): + terminate = True + break + + if terminate: + # Quirk to enforce "hard termination mode" for paragraphs; + # normally if you call `tokenize(state, startLine, nextLine)`, + # paragraphs will look below nextLine for paragraph continuation, + # but if blockquote is terminated by another tag, they shouldn't + state.lineMax = nextLine + + if state.blkIndent != 0: + # state.blkIndent was non-zero, we now set it to zero, + # so we need to re-calculate all offsets to appear as + # if indent wasn't changed + oldBMarks.append(state.bMarks[nextLine]) + oldBSCount.append(state.bsCount[nextLine]) + oldTShift.append(state.tShift[nextLine]) + oldSCount.append(state.sCount[nextLine]) + state.sCount[nextLine] -= state.blkIndent + + break + + oldBMarks.append(state.bMarks[nextLine]) + oldBSCount.append(state.bsCount[nextLine]) + oldTShift.append(state.tShift[nextLine]) + oldSCount.append(state.sCount[nextLine]) + + # A negative indentation means that this is a paragraph continuation + # + state.sCount[nextLine] = -1 + + nextLine += 1 + + oldIndent = state.blkIndent + state.blkIndent = 0 + + token = state.push("blockquote_open", "blockquote", 1) + token.markup = ">" + token.map = lines = [startLine, 0] + + state.md.block.tokenize(state, startLine, nextLine) + + token = state.push("blockquote_close", "blockquote", -1) + token.markup = ">" + + state.lineMax = oldLineMax + state.parentType = oldParentType + lines[1] = state.line + + # Restore original tShift; this might not be necessary since the parser + # has already been here, but just to make sure we can do that. + for i, item in enumerate(oldTShift): + state.bMarks[i + startLine] = oldBMarks[i] + state.tShift[i + startLine] = item + state.sCount[i + startLine] = oldSCount[i] + state.bsCount[i + startLine] = oldBSCount[i] + + state.blkIndent = oldIndent + + return True diff --git a/lib/markdown_it/rules_block/code.py b/lib/markdown_it/rules_block/code.py new file mode 100644 index 0000000..af8a41c --- /dev/null +++ b/lib/markdown_it/rules_block/code.py @@ -0,0 +1,36 @@ +"""Code block (4 spaces padded).""" + +import logging + +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def code(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug("entering code: %s, %s, %s, %s", state, startLine, endLine, silent) + + if not state.is_code_block(startLine): + return False + + last = nextLine = startLine + 1 + + while nextLine < endLine: + if state.isEmpty(nextLine): + nextLine += 1 + continue + + if state.is_code_block(nextLine): + nextLine += 1 + last = nextLine + continue + + break + + state.line = last + + token = state.push("code_block", "code", 0) + token.content = state.getLines(startLine, last, 4 + state.blkIndent, False) + "\n" + token.map = [startLine, state.line] + + return True diff --git a/lib/markdown_it/rules_block/fence.py b/lib/markdown_it/rules_block/fence.py new file mode 100644 index 0000000..263f1b8 --- /dev/null +++ b/lib/markdown_it/rules_block/fence.py @@ -0,0 +1,101 @@ +# fences (``` lang, ~~~ lang) +import logging + +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def fence(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug("entering fence: %s, %s, %s, %s", state, startLine, endLine, silent) + + haveEndMarker = False + pos = state.bMarks[startLine] + state.tShift[startLine] + maximum = state.eMarks[startLine] + + if state.is_code_block(startLine): + return False + + if pos + 3 > maximum: + return False + + marker = state.src[pos] + + if marker not in ("~", "`"): + return False + + # scan marker length + mem = pos + pos = state.skipCharsStr(pos, marker) + + length = pos - mem + + if length < 3: + return False + + markup = state.src[mem:pos] + params = state.src[pos:maximum] + + if marker == "`" and marker in params: + return False + + # Since start is found, we can report success here in validation mode + if silent: + return True + + # search end of block + nextLine = startLine + + while True: + nextLine += 1 + if nextLine >= endLine: + # unclosed block should be autoclosed by end of document. + # also block seems to be autoclosed by end of parent + break + + pos = mem = state.bMarks[nextLine] + state.tShift[nextLine] + maximum = state.eMarks[nextLine] + + if pos < maximum and state.sCount[nextLine] < state.blkIndent: + # non-empty line with negative indent should stop the list: + # - ``` + # test + break + + try: + if state.src[pos] != marker: + continue + except IndexError: + break + + if state.is_code_block(nextLine): + continue + + pos = state.skipCharsStr(pos, marker) + + # closing code fence must be at least as long as the opening one + if pos - mem < length: + continue + + # make sure tail has spaces only + pos = state.skipSpaces(pos) + + if pos < maximum: + continue + + haveEndMarker = True + # found! + break + + # If a fence has heading spaces, they should be removed from its inner block + length = state.sCount[startLine] + + state.line = nextLine + (1 if haveEndMarker else 0) + + token = state.push("fence", "code", 0) + token.info = params + token.content = state.getLines(startLine + 1, nextLine, length, True) + token.markup = markup + token.map = [startLine, state.line] + + return True diff --git a/lib/markdown_it/rules_block/heading.py b/lib/markdown_it/rules_block/heading.py new file mode 100644 index 0000000..afcf9ed --- /dev/null +++ b/lib/markdown_it/rules_block/heading.py @@ -0,0 +1,69 @@ +"""Atex heading (#, ##, ...)""" + +from __future__ import annotations + +import logging + +from ..common.utils import isStrSpace +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug("entering heading: %s, %s, %s, %s", state, startLine, endLine, silent) + + pos = state.bMarks[startLine] + state.tShift[startLine] + maximum = state.eMarks[startLine] + + if state.is_code_block(startLine): + return False + + ch: str | None = state.src[pos] + + if ch != "#" or pos >= maximum: + return False + + # count heading level + level = 1 + pos += 1 + try: + ch = state.src[pos] + except IndexError: + ch = None + while ch == "#" and pos < maximum and level <= 6: + level += 1 + pos += 1 + try: + ch = state.src[pos] + except IndexError: + ch = None + + if level > 6 or (pos < maximum and not isStrSpace(ch)): + return False + + if silent: + return True + + # Let's cut tails like ' ### ' from the end of string + + maximum = state.skipSpacesBack(maximum, pos) + tmp = state.skipCharsStrBack(maximum, "#", pos) + if tmp > pos and isStrSpace(state.src[tmp - 1]): + maximum = tmp + + state.line = startLine + 1 + + token = state.push("heading_open", "h" + str(level), 1) + token.markup = "########"[:level] + token.map = [startLine, state.line] + + token = state.push("inline", "", 0) + token.content = state.src[pos:maximum].strip() + token.map = [startLine, state.line] + token.children = [] + + token = state.push("heading_close", "h" + str(level), -1) + token.markup = "########"[:level] + + return True diff --git a/lib/markdown_it/rules_block/hr.py b/lib/markdown_it/rules_block/hr.py new file mode 100644 index 0000000..fca7d79 --- /dev/null +++ b/lib/markdown_it/rules_block/hr.py @@ -0,0 +1,56 @@ +"""Horizontal rule + +At least 3 of these characters on a line * - _ +""" + +import logging + +from ..common.utils import isStrSpace +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def hr(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug("entering hr: %s, %s, %s, %s", state, startLine, endLine, silent) + + pos = state.bMarks[startLine] + state.tShift[startLine] + maximum = state.eMarks[startLine] + + if state.is_code_block(startLine): + return False + + try: + marker = state.src[pos] + except IndexError: + return False + pos += 1 + + # Check hr marker + if marker not in ("*", "-", "_"): + return False + + # markers can be mixed with spaces, but there should be at least 3 of them + + cnt = 1 + while pos < maximum: + ch = state.src[pos] + pos += 1 + if ch != marker and not isStrSpace(ch): + return False + if ch == marker: + cnt += 1 + + if cnt < 3: + return False + + if silent: + return True + + state.line = startLine + 1 + + token = state.push("hr", "hr", 0) + token.map = [startLine, state.line] + token.markup = marker * (cnt + 1) + + return True diff --git a/lib/markdown_it/rules_block/html_block.py b/lib/markdown_it/rules_block/html_block.py new file mode 100644 index 0000000..3d43f6e --- /dev/null +++ b/lib/markdown_it/rules_block/html_block.py @@ -0,0 +1,90 @@ +# HTML block +from __future__ import annotations + +import logging +import re + +from ..common.html_blocks import block_names +from ..common.html_re import HTML_OPEN_CLOSE_TAG_STR +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + +# An array of opening and corresponding closing sequences for html tags, +# last argument defines whether it can terminate a paragraph or not +HTML_SEQUENCES: list[tuple[re.Pattern[str], re.Pattern[str], bool]] = [ + ( + re.compile(r"^<(script|pre|style|textarea)(?=(\s|>|$))", re.IGNORECASE), + re.compile(r"<\/(script|pre|style|textarea)>", re.IGNORECASE), + True, + ), + (re.compile(r"^"), True), + (re.compile(r"^<\?"), re.compile(r"\?>"), True), + (re.compile(r"^"), True), + (re.compile(r"^"), True), + ( + re.compile("^|$))", re.IGNORECASE), + re.compile(r"^$"), + True, + ), + (re.compile(HTML_OPEN_CLOSE_TAG_STR + "\\s*$"), re.compile(r"^$"), False), +] + + +def html_block(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug( + "entering html_block: %s, %s, %s, %s", state, startLine, endLine, silent + ) + pos = state.bMarks[startLine] + state.tShift[startLine] + maximum = state.eMarks[startLine] + + if state.is_code_block(startLine): + return False + + if not state.md.options.get("html", None): + return False + + if state.src[pos] != "<": + return False + + lineText = state.src[pos:maximum] + + html_seq = None + for HTML_SEQUENCE in HTML_SEQUENCES: + if HTML_SEQUENCE[0].search(lineText): + html_seq = HTML_SEQUENCE + break + + if not html_seq: + return False + + if silent: + # true if this sequence can be a terminator, false otherwise + return html_seq[2] + + nextLine = startLine + 1 + + # If we are here - we detected HTML block. + # Let's roll down till block end. + if not html_seq[1].search(lineText): + while nextLine < endLine: + if state.sCount[nextLine] < state.blkIndent: + break + + pos = state.bMarks[nextLine] + state.tShift[nextLine] + maximum = state.eMarks[nextLine] + lineText = state.src[pos:maximum] + + if html_seq[1].search(lineText): + if len(lineText) != 0: + nextLine += 1 + break + nextLine += 1 + + state.line = nextLine + + token = state.push("html_block", "", 0) + token.map = [startLine, nextLine] + token.content = state.getLines(startLine, nextLine, state.blkIndent, True) + + return True diff --git a/lib/markdown_it/rules_block/lheading.py b/lib/markdown_it/rules_block/lheading.py new file mode 100644 index 0000000..3522207 --- /dev/null +++ b/lib/markdown_it/rules_block/lheading.py @@ -0,0 +1,86 @@ +# lheading (---, ==) +import logging + +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug("entering lheading: %s, %s, %s, %s", state, startLine, endLine, silent) + + level = None + nextLine = startLine + 1 + ruler = state.md.block.ruler + terminatorRules = ruler.getRules("paragraph") + + if state.is_code_block(startLine): + return False + + oldParentType = state.parentType + state.parentType = "paragraph" # use paragraph to match terminatorRules + + # jump line-by-line until empty one or EOF + while nextLine < endLine and not state.isEmpty(nextLine): + # this would be a code block normally, but after paragraph + # it's considered a lazy continuation regardless of what's there + if state.sCount[nextLine] - state.blkIndent > 3: + nextLine += 1 + continue + + # Check for underline in setext header + if state.sCount[nextLine] >= state.blkIndent: + pos = state.bMarks[nextLine] + state.tShift[nextLine] + maximum = state.eMarks[nextLine] + + if pos < maximum: + marker = state.src[pos] + + if marker in ("-", "="): + pos = state.skipCharsStr(pos, marker) + pos = state.skipSpaces(pos) + + # /* = */ + if pos >= maximum: + level = 1 if marker == "=" else 2 + break + + # quirk for blockquotes, this line should already be checked by that rule + if state.sCount[nextLine] < 0: + nextLine += 1 + continue + + # Some tags can terminate paragraph without empty line. + terminate = False + for terminatorRule in terminatorRules: + if terminatorRule(state, nextLine, endLine, True): + terminate = True + break + if terminate: + break + + nextLine += 1 + + if not level: + # Didn't find valid underline + return False + + content = state.getLines(startLine, nextLine, state.blkIndent, False).strip() + + state.line = nextLine + 1 + + token = state.push("heading_open", "h" + str(level), 1) + token.markup = marker + token.map = [startLine, state.line] + + token = state.push("inline", "", 0) + token.content = content + token.map = [startLine, state.line - 1] + token.children = [] + + token = state.push("heading_close", "h" + str(level), -1) + token.markup = marker + + state.parentType = oldParentType + + return True diff --git a/lib/markdown_it/rules_block/list.py b/lib/markdown_it/rules_block/list.py new file mode 100644 index 0000000..d8070d7 --- /dev/null +++ b/lib/markdown_it/rules_block/list.py @@ -0,0 +1,345 @@ +# Lists +import logging + +from ..common.utils import isStrSpace +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +# Search `[-+*][\n ]`, returns next pos after marker on success +# or -1 on fail. +def skipBulletListMarker(state: StateBlock, startLine: int) -> int: + pos = state.bMarks[startLine] + state.tShift[startLine] + maximum = state.eMarks[startLine] + + try: + marker = state.src[pos] + except IndexError: + return -1 + pos += 1 + + if marker not in ("*", "-", "+"): + return -1 + + if pos < maximum: + ch = state.src[pos] + + if not isStrSpace(ch): + # " -test " - is not a list item + return -1 + + return pos + + +# Search `\d+[.)][\n ]`, returns next pos after marker on success +# or -1 on fail. +def skipOrderedListMarker(state: StateBlock, startLine: int) -> int: + start = state.bMarks[startLine] + state.tShift[startLine] + pos = start + maximum = state.eMarks[startLine] + + # List marker should have at least 2 chars (digit + dot) + if pos + 1 >= maximum: + return -1 + + ch = state.src[pos] + pos += 1 + + ch_ord = ord(ch) + # /* 0 */ /* 9 */ + if ch_ord < 0x30 or ch_ord > 0x39: + return -1 + + while True: + # EOL -> fail + if pos >= maximum: + return -1 + + ch = state.src[pos] + pos += 1 + + # /* 0 */ /* 9 */ + ch_ord = ord(ch) + if ch_ord >= 0x30 and ch_ord <= 0x39: + # List marker should have no more than 9 digits + # (prevents integer overflow in browsers) + if pos - start >= 10: + return -1 + + continue + + # found valid marker + if ch in (")", "."): + break + + return -1 + + if pos < maximum: + ch = state.src[pos] + + if not isStrSpace(ch): + # " 1.test " - is not a list item + return -1 + + return pos + + +def markTightParagraphs(state: StateBlock, idx: int) -> None: + level = state.level + 2 + + i = idx + 2 + length = len(state.tokens) - 2 + while i < length: + if state.tokens[i].level == level and state.tokens[i].type == "paragraph_open": + state.tokens[i + 2].hidden = True + state.tokens[i].hidden = True + i += 2 + i += 1 + + +def list_block(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug("entering list: %s, %s, %s, %s", state, startLine, endLine, silent) + + isTerminatingParagraph = False + tight = True + + if state.is_code_block(startLine): + return False + + # Special case: + # - item 1 + # - item 2 + # - item 3 + # - item 4 + # - this one is a paragraph continuation + if ( + state.listIndent >= 0 + and state.sCount[startLine] - state.listIndent >= 4 + and state.sCount[startLine] < state.blkIndent + ): + return False + + # limit conditions when list can interrupt + # a paragraph (validation mode only) + # Next list item should still terminate previous list item + # + # This code can fail if plugins use blkIndent as well as lists, + # but I hope the spec gets fixed long before that happens. + # + if ( + silent + and state.parentType == "paragraph" + and state.sCount[startLine] >= state.blkIndent + ): + isTerminatingParagraph = True + + # Detect list type and position after marker + posAfterMarker = skipOrderedListMarker(state, startLine) + if posAfterMarker >= 0: + isOrdered = True + start = state.bMarks[startLine] + state.tShift[startLine] + markerValue = int(state.src[start : posAfterMarker - 1]) + + # If we're starting a new ordered list right after + # a paragraph, it should start with 1. + if isTerminatingParagraph and markerValue != 1: + return False + else: + posAfterMarker = skipBulletListMarker(state, startLine) + if posAfterMarker >= 0: + isOrdered = False + else: + return False + + # If we're starting a new unordered list right after + # a paragraph, first line should not be empty. + if ( + isTerminatingParagraph + and state.skipSpaces(posAfterMarker) >= state.eMarks[startLine] + ): + return False + + # We should terminate list on style change. Remember first one to compare. + markerChar = state.src[posAfterMarker - 1] + + # For validation mode we can terminate immediately + if silent: + return True + + # Start list + listTokIdx = len(state.tokens) + + if isOrdered: + token = state.push("ordered_list_open", "ol", 1) + if markerValue != 1: + token.attrs = {"start": markerValue} + + else: + token = state.push("bullet_list_open", "ul", 1) + + token.map = listLines = [startLine, 0] + token.markup = markerChar + + # + # Iterate list items + # + + nextLine = startLine + prevEmptyEnd = False + terminatorRules = state.md.block.ruler.getRules("list") + + oldParentType = state.parentType + state.parentType = "list" + + while nextLine < endLine: + pos = posAfterMarker + maximum = state.eMarks[nextLine] + + initial = offset = ( + state.sCount[nextLine] + + posAfterMarker + - (state.bMarks[startLine] + state.tShift[startLine]) + ) + + while pos < maximum: + ch = state.src[pos] + + if ch == "\t": + offset += 4 - (offset + state.bsCount[nextLine]) % 4 + elif ch == " ": + offset += 1 + else: + break + + pos += 1 + + contentStart = pos + + # trimming space in "- \n 3" case, indent is 1 here + indentAfterMarker = 1 if contentStart >= maximum else offset - initial + + # If we have more than 4 spaces, the indent is 1 + # (the rest is just indented code block) + if indentAfterMarker > 4: + indentAfterMarker = 1 + + # " - test" + # ^^^^^ - calculating total length of this thing + indent = initial + indentAfterMarker + + # Run subparser & write tokens + token = state.push("list_item_open", "li", 1) + token.markup = markerChar + token.map = itemLines = [startLine, 0] + if isOrdered: + token.info = state.src[start : posAfterMarker - 1] + + # change current state, then restore it after parser subcall + oldTight = state.tight + oldTShift = state.tShift[startLine] + oldSCount = state.sCount[startLine] + + # - example list + # ^ listIndent position will be here + # ^ blkIndent position will be here + # + oldListIndent = state.listIndent + state.listIndent = state.blkIndent + state.blkIndent = indent + + state.tight = True + state.tShift[startLine] = contentStart - state.bMarks[startLine] + state.sCount[startLine] = offset + + if contentStart >= maximum and state.isEmpty(startLine + 1): + # workaround for this case + # (list item is empty, list terminates before "foo"): + # ~~~~~~~~ + # - + # + # foo + # ~~~~~~~~ + state.line = min(state.line + 2, endLine) + else: + # NOTE in list.js this was: + # state.md.block.tokenize(state, startLine, endLine, True) + # but tokeniz does not take the final parameter + state.md.block.tokenize(state, startLine, endLine) + + # If any of list item is tight, mark list as tight + if (not state.tight) or prevEmptyEnd: + tight = False + + # Item become loose if finish with empty line, + # but we should filter last element, because it means list finish + prevEmptyEnd = (state.line - startLine) > 1 and state.isEmpty(state.line - 1) + + state.blkIndent = state.listIndent + state.listIndent = oldListIndent + state.tShift[startLine] = oldTShift + state.sCount[startLine] = oldSCount + state.tight = oldTight + + token = state.push("list_item_close", "li", -1) + token.markup = markerChar + + nextLine = startLine = state.line + itemLines[1] = nextLine + + if nextLine >= endLine: + break + + contentStart = state.bMarks[startLine] + + # + # Try to check if list is terminated or continued. + # + if state.sCount[nextLine] < state.blkIndent: + break + + if state.is_code_block(startLine): + break + + # fail if terminating block found + terminate = False + for terminatorRule in terminatorRules: + if terminatorRule(state, nextLine, endLine, True): + terminate = True + break + + if terminate: + break + + # fail if list has another type + if isOrdered: + posAfterMarker = skipOrderedListMarker(state, nextLine) + if posAfterMarker < 0: + break + start = state.bMarks[nextLine] + state.tShift[nextLine] + else: + posAfterMarker = skipBulletListMarker(state, nextLine) + if posAfterMarker < 0: + break + + if markerChar != state.src[posAfterMarker - 1]: + break + + # Finalize list + if isOrdered: + token = state.push("ordered_list_close", "ol", -1) + else: + token = state.push("bullet_list_close", "ul", -1) + + token.markup = markerChar + + listLines[1] = nextLine + state.line = nextLine + + state.parentType = oldParentType + + # mark paragraphs tight if needed + if tight: + markTightParagraphs(state, listTokIdx) + + return True diff --git a/lib/markdown_it/rules_block/paragraph.py b/lib/markdown_it/rules_block/paragraph.py new file mode 100644 index 0000000..30ba877 --- /dev/null +++ b/lib/markdown_it/rules_block/paragraph.py @@ -0,0 +1,66 @@ +"""Paragraph.""" + +import logging + +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def paragraph(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + LOGGER.debug( + "entering paragraph: %s, %s, %s, %s", state, startLine, endLine, silent + ) + + nextLine = startLine + 1 + ruler = state.md.block.ruler + terminatorRules = ruler.getRules("paragraph") + endLine = state.lineMax + + oldParentType = state.parentType + state.parentType = "paragraph" + + # jump line-by-line until empty one or EOF + while nextLine < endLine: + if state.isEmpty(nextLine): + break + # this would be a code block normally, but after paragraph + # it's considered a lazy continuation regardless of what's there + if state.sCount[nextLine] - state.blkIndent > 3: + nextLine += 1 + continue + + # quirk for blockquotes, this line should already be checked by that rule + if state.sCount[nextLine] < 0: + nextLine += 1 + continue + + # Some tags can terminate paragraph without empty line. + terminate = False + for terminatorRule in terminatorRules: + if terminatorRule(state, nextLine, endLine, True): + terminate = True + break + + if terminate: + break + + nextLine += 1 + + content = state.getLines(startLine, nextLine, state.blkIndent, False).strip() + + state.line = nextLine + + token = state.push("paragraph_open", "p", 1) + token.map = [startLine, state.line] + + token = state.push("inline", "", 0) + token.content = content + token.map = [startLine, state.line] + token.children = [] + + token = state.push("paragraph_close", "p", -1) + + state.parentType = oldParentType + + return True diff --git a/lib/markdown_it/rules_block/reference.py b/lib/markdown_it/rules_block/reference.py new file mode 100644 index 0000000..ad94d40 --- /dev/null +++ b/lib/markdown_it/rules_block/reference.py @@ -0,0 +1,235 @@ +import logging + +from ..common.utils import charCodeAt, isSpace, normalizeReference +from .state_block import StateBlock + +LOGGER = logging.getLogger(__name__) + + +def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) -> bool: + LOGGER.debug( + "entering reference: %s, %s, %s, %s", state, startLine, _endLine, silent + ) + + pos = state.bMarks[startLine] + state.tShift[startLine] + maximum = state.eMarks[startLine] + nextLine = startLine + 1 + + if state.is_code_block(startLine): + return False + + if state.src[pos] != "[": + return False + + string = state.src[pos : maximum + 1] + + # string = state.getLines(startLine, nextLine, state.blkIndent, False).strip() + maximum = len(string) + + labelEnd = None + pos = 1 + while pos < maximum: + ch = charCodeAt(string, pos) + if ch == 0x5B: # /* [ */ + return False + elif ch == 0x5D: # /* ] */ + labelEnd = pos + break + elif ch == 0x0A: # /* \n */ + if (lineContent := getNextLine(state, nextLine)) is not None: + string += lineContent + maximum = len(string) + nextLine += 1 + elif ch == 0x5C: # /* \ */ + pos += 1 + if ( + pos < maximum + and charCodeAt(string, pos) == 0x0A + and (lineContent := getNextLine(state, nextLine)) is not None + ): + string += lineContent + maximum = len(string) + nextLine += 1 + pos += 1 + + if ( + labelEnd is None or labelEnd < 0 or charCodeAt(string, labelEnd + 1) != 0x3A + ): # /* : */ + return False + + # [label]: destination 'title' + # ^^^ skip optional whitespace here + pos = labelEnd + 2 + while pos < maximum: + ch = charCodeAt(string, pos) + if ch == 0x0A: + if (lineContent := getNextLine(state, nextLine)) is not None: + string += lineContent + maximum = len(string) + nextLine += 1 + elif isSpace(ch): + pass + else: + break + pos += 1 + + # [label]: destination 'title' + # ^^^^^^^^^^^ parse this + destRes = state.md.helpers.parseLinkDestination(string, pos, maximum) + if not destRes.ok: + return False + + href = state.md.normalizeLink(destRes.str) + if not state.md.validateLink(href): + return False + + pos = destRes.pos + + # save cursor state, we could require to rollback later + destEndPos = pos + destEndLineNo = nextLine + + # [label]: destination 'title' + # ^^^ skipping those spaces + start = pos + while pos < maximum: + ch = charCodeAt(string, pos) + if ch == 0x0A: + if (lineContent := getNextLine(state, nextLine)) is not None: + string += lineContent + maximum = len(string) + nextLine += 1 + elif isSpace(ch): + pass + else: + break + pos += 1 + + # [label]: destination 'title' + # ^^^^^^^ parse this + titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, None) + while titleRes.can_continue: + if (lineContent := getNextLine(state, nextLine)) is None: + break + string += lineContent + pos = maximum + maximum = len(string) + nextLine += 1 + titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, titleRes) + + if pos < maximum and start != pos and titleRes.ok: + title = titleRes.str + pos = titleRes.pos + else: + title = "" + pos = destEndPos + nextLine = destEndLineNo + + # skip trailing spaces until the rest of the line + while pos < maximum: + ch = charCodeAt(string, pos) + if not isSpace(ch): + break + pos += 1 + + if pos < maximum and charCodeAt(string, pos) != 0x0A and title: + # garbage at the end of the line after title, + # but it could still be a valid reference if we roll back + title = "" + pos = destEndPos + nextLine = destEndLineNo + while pos < maximum: + ch = charCodeAt(string, pos) + if not isSpace(ch): + break + pos += 1 + + if pos < maximum and charCodeAt(string, pos) != 0x0A: + # garbage at the end of the line + return False + + label = normalizeReference(string[1:labelEnd]) + if not label: + # CommonMark 0.20 disallows empty labels + return False + + # Reference can not terminate anything. This check is for safety only. + if silent: + return True + + if "references" not in state.env: + state.env["references"] = {} + + state.line = nextLine + + # note, this is not part of markdown-it JS, but is useful for renderers + if state.md.options.get("inline_definitions", False): + token = state.push("definition", "", 0) + token.meta = { + "id": label, + "title": title, + "url": href, + "label": string[1:labelEnd], + } + token.map = [startLine, state.line] + + if label not in state.env["references"]: + state.env["references"][label] = { + "title": title, + "href": href, + "map": [startLine, state.line], + } + else: + state.env.setdefault("duplicate_refs", []).append( + { + "title": title, + "href": href, + "label": label, + "map": [startLine, state.line], + } + ) + + return True + + +def getNextLine(state: StateBlock, nextLine: int) -> None | str: + endLine = state.lineMax + + if nextLine >= endLine or state.isEmpty(nextLine): + # empty line or end of input + return None + + isContinuation = False + + # this would be a code block normally, but after paragraph + # it's considered a lazy continuation regardless of what's there + if state.is_code_block(nextLine): + isContinuation = True + + # quirk for blockquotes, this line should already be checked by that rule + if state.sCount[nextLine] < 0: + isContinuation = True + + if not isContinuation: + terminatorRules = state.md.block.ruler.getRules("reference") + oldParentType = state.parentType + state.parentType = "reference" + + # Some tags can terminate paragraph without empty line. + terminate = False + for terminatorRule in terminatorRules: + if terminatorRule(state, nextLine, endLine, True): + terminate = True + break + + state.parentType = oldParentType + + if terminate: + # terminated by another block + return None + + pos = state.bMarks[nextLine] + state.tShift[nextLine] + maximum = state.eMarks[nextLine] + + # max + 1 explicitly includes the newline + return state.src[pos : maximum + 1] diff --git a/lib/markdown_it/rules_block/state_block.py b/lib/markdown_it/rules_block/state_block.py new file mode 100644 index 0000000..445ad26 --- /dev/null +++ b/lib/markdown_it/rules_block/state_block.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal + +from ..common.utils import isStrSpace +from ..ruler import StateBase +from ..token import Token +from ..utils import EnvType + +if TYPE_CHECKING: + from markdown_it.main import MarkdownIt + + +class StateBlock(StateBase): + def __init__( + self, src: str, md: MarkdownIt, env: EnvType, tokens: list[Token] + ) -> None: + self.src = src + + # link to parser instance + self.md = md + + self.env = env + + # + # Internal state variables + # + + self.tokens = tokens + + self.bMarks: list[int] = [] # line begin offsets for fast jumps + self.eMarks: list[int] = [] # line end offsets for fast jumps + # offsets of the first non-space characters (tabs not expanded) + self.tShift: list[int] = [] + self.sCount: list[int] = [] # indents for each line (tabs expanded) + + # An amount of virtual spaces (tabs expanded) between beginning + # of each line (bMarks) and real beginning of that line. + # + # It exists only as a hack because blockquotes override bMarks + # losing information in the process. + # + # It's used only when expanding tabs, you can think about it as + # an initial tab length, e.g. bsCount=21 applied to string `\t123` + # means first tab should be expanded to 4-21%4 === 3 spaces. + # + self.bsCount: list[int] = [] + + # block parser variables + self.blkIndent = 0 # required block content indent (for example, if we are + # inside a list, it would be positioned after list marker) + self.line = 0 # line index in src + self.lineMax = 0 # lines count + self.tight = False # loose/tight mode for lists + self.ddIndent = -1 # indent of the current dd block (-1 if there isn't any) + self.listIndent = -1 # indent of the current list block (-1 if there isn't any) + + # can be 'blockquote', 'list', 'root', 'paragraph' or 'reference' + # used in lists to determine if they interrupt a paragraph + self.parentType = "root" + + self.level = 0 + + # renderer + self.result = "" + + # Create caches + # Generate markers. + indent_found = False + + start = pos = indent = offset = 0 + length = len(self.src) + + for pos, character in enumerate(self.src): + if not indent_found: + if isStrSpace(character): + indent += 1 + + if character == "\t": + offset += 4 - offset % 4 + else: + offset += 1 + continue + else: + indent_found = True + + if character == "\n" or pos == length - 1: + if character != "\n": + pos += 1 + self.bMarks.append(start) + self.eMarks.append(pos) + self.tShift.append(indent) + self.sCount.append(offset) + self.bsCount.append(0) + + indent_found = False + indent = 0 + offset = 0 + start = pos + 1 + + # Push fake entry to simplify cache bounds checks + self.bMarks.append(length) + self.eMarks.append(length) + self.tShift.append(0) + self.sCount.append(0) + self.bsCount.append(0) + + self.lineMax = len(self.bMarks) - 1 # don't count last fake line + + # pre-check if code blocks are enabled, to speed up is_code_block method + self._code_enabled = "code" in self.md["block"].ruler.get_active_rules() + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(line={self.line},level={self.level},tokens={len(self.tokens)})" + ) + + def push(self, ttype: str, tag: str, nesting: Literal[-1, 0, 1]) -> Token: + """Push new token to "stream".""" + token = Token(ttype, tag, nesting) + token.block = True + if nesting < 0: + self.level -= 1 # closing tag + token.level = self.level + if nesting > 0: + self.level += 1 # opening tag + self.tokens.append(token) + return token + + def isEmpty(self, line: int) -> bool: + """.""" + return (self.bMarks[line] + self.tShift[line]) >= self.eMarks[line] + + def skipEmptyLines(self, from_pos: int) -> int: + """.""" + while from_pos < self.lineMax: + try: + if (self.bMarks[from_pos] + self.tShift[from_pos]) < self.eMarks[ + from_pos + ]: + break + except IndexError: + pass + from_pos += 1 + return from_pos + + def skipSpaces(self, pos: int) -> int: + """Skip spaces from given position.""" + while True: + try: + current = self.src[pos] + except IndexError: + break + if not isStrSpace(current): + break + pos += 1 + return pos + + def skipSpacesBack(self, pos: int, minimum: int) -> int: + """Skip spaces from given position in reverse.""" + if pos <= minimum: + return pos + while pos > minimum: + pos -= 1 + if not isStrSpace(self.src[pos]): + return pos + 1 + return pos + + def skipChars(self, pos: int, code: int) -> int: + """Skip character code from given position.""" + while True: + try: + current = self.srcCharCode[pos] + except IndexError: + break + if current != code: + break + pos += 1 + return pos + + def skipCharsStr(self, pos: int, ch: str) -> int: + """Skip character string from given position.""" + while True: + try: + current = self.src[pos] + except IndexError: + break + if current != ch: + break + pos += 1 + return pos + + def skipCharsBack(self, pos: int, code: int, minimum: int) -> int: + """Skip character code reverse from given position - 1.""" + if pos <= minimum: + return pos + while pos > minimum: + pos -= 1 + if code != self.srcCharCode[pos]: + return pos + 1 + return pos + + def skipCharsStrBack(self, pos: int, ch: str, minimum: int) -> int: + """Skip character string reverse from given position - 1.""" + if pos <= minimum: + return pos + while pos > minimum: + pos -= 1 + if ch != self.src[pos]: + return pos + 1 + return pos + + def getLines(self, begin: int, end: int, indent: int, keepLastLF: bool) -> str: + """Cut lines range from source.""" + line = begin + if begin >= end: + return "" + + queue = [""] * (end - begin) + + i = 1 + while line < end: + lineIndent = 0 + lineStart = first = self.bMarks[line] + last = ( + self.eMarks[line] + 1 + if line + 1 < end or keepLastLF + else self.eMarks[line] + ) + + while (first < last) and (lineIndent < indent): + ch = self.src[first] + if isStrSpace(ch): + if ch == "\t": + lineIndent += 4 - (lineIndent + self.bsCount[line]) % 4 + else: + lineIndent += 1 + elif first - lineStart < self.tShift[line]: + lineIndent += 1 + else: + break + first += 1 + + if lineIndent > indent: + # partially expanding tabs in code blocks, e.g '\t\tfoobar' + # with indent=2 becomes ' \tfoobar' + queue[i - 1] = (" " * (lineIndent - indent)) + self.src[first:last] + else: + queue[i - 1] = self.src[first:last] + + line += 1 + i += 1 + + return "".join(queue) + + def is_code_block(self, line: int) -> bool: + """Check if line is a code block, + i.e. the code block rule is enabled and text is indented by more than 3 spaces. + """ + return self._code_enabled and (self.sCount[line] - self.blkIndent) >= 4 diff --git a/lib/markdown_it/rules_block/table.py b/lib/markdown_it/rules_block/table.py new file mode 100644 index 0000000..c52553d --- /dev/null +++ b/lib/markdown_it/rules_block/table.py @@ -0,0 +1,250 @@ +# GFM table, https://github.github.com/gfm/#tables-extension- +from __future__ import annotations + +import re + +from ..common.utils import charStrAt, isStrSpace +from .state_block import StateBlock + +headerLineRe = re.compile(r"^:?-+:?$") +enclosingPipesRe = re.compile(r"^\||\|$") + +# Limit the amount of empty autocompleted cells in a table, +# see https://github.com/markdown-it/markdown-it/issues/1000, +# Both pulldown-cmark and commonmark-hs limit the number of cells this way to ~200k. +# We set it to 65k, which can expand user input by a factor of x370 +# (256x256 square is 1.8kB expanded into 650kB). +MAX_AUTOCOMPLETED_CELLS = 0x10000 + + +def getLine(state: StateBlock, line: int) -> str: + pos = state.bMarks[line] + state.tShift[line] + maximum = state.eMarks[line] + + # return state.src.substr(pos, max - pos) + return state.src[pos:maximum] + + +def escapedSplit(string: str) -> list[str]: + result: list[str] = [] + pos = 0 + max = len(string) + isEscaped = False + lastPos = 0 + current = "" + ch = charStrAt(string, pos) + + while pos < max: + if ch == "|": + if not isEscaped: + # pipe separating cells, '|' + result.append(current + string[lastPos:pos]) + current = "" + lastPos = pos + 1 + else: + # escaped pipe, '\|' + current += string[lastPos : pos - 1] + lastPos = pos + + isEscaped = ch == "\\" + pos += 1 + + ch = charStrAt(string, pos) + + result.append(current + string[lastPos:]) + + return result + + +def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool: + tbodyLines = None + + # should have at least two lines + if startLine + 2 > endLine: + return False + + nextLine = startLine + 1 + + if state.sCount[nextLine] < state.blkIndent: + return False + + if state.is_code_block(nextLine): + return False + + # first character of the second line should be '|', '-', ':', + # and no other characters are allowed but spaces; + # basically, this is the equivalent of /^[-:|][-:|\s]*$/ regexp + + pos = state.bMarks[nextLine] + state.tShift[nextLine] + if pos >= state.eMarks[nextLine]: + return False + first_ch = state.src[pos] + pos += 1 + if first_ch not in ("|", "-", ":"): + return False + + if pos >= state.eMarks[nextLine]: + return False + second_ch = state.src[pos] + pos += 1 + if second_ch not in ("|", "-", ":") and not isStrSpace(second_ch): + return False + + # if first character is '-', then second character must not be a space + # (due to parsing ambiguity with list) + if first_ch == "-" and isStrSpace(second_ch): + return False + + while pos < state.eMarks[nextLine]: + ch = state.src[pos] + + if ch not in ("|", "-", ":") and not isStrSpace(ch): + return False + + pos += 1 + + lineText = getLine(state, startLine + 1) + + columns = lineText.split("|") + aligns = [] + for i in range(len(columns)): + t = columns[i].strip() + if not t: + # allow empty columns before and after table, but not in between columns; + # e.g. allow ` |---| `, disallow ` ---||--- ` + if i == 0 or i == len(columns) - 1: + continue + else: + return False + + if not headerLineRe.search(t): + return False + if charStrAt(t, len(t) - 1) == ":": + aligns.append("center" if charStrAt(t, 0) == ":" else "right") + elif charStrAt(t, 0) == ":": + aligns.append("left") + else: + aligns.append("") + + lineText = getLine(state, startLine).strip() + if "|" not in lineText: + return False + if state.is_code_block(startLine): + return False + columns = escapedSplit(lineText) + if columns and columns[0] == "": + columns.pop(0) + if columns and columns[-1] == "": + columns.pop() + + # header row will define an amount of columns in the entire table, + # and align row should be exactly the same (the rest of the rows can differ) + columnCount = len(columns) + if columnCount == 0 or columnCount != len(aligns): + return False + + if silent: + return True + + oldParentType = state.parentType + state.parentType = "table" + + # use 'blockquote' lists for termination because it's + # the most similar to tables + terminatorRules = state.md.block.ruler.getRules("blockquote") + + token = state.push("table_open", "table", 1) + token.map = tableLines = [startLine, 0] + + token = state.push("thead_open", "thead", 1) + token.map = [startLine, startLine + 1] + + token = state.push("tr_open", "tr", 1) + token.map = [startLine, startLine + 1] + + for i in range(len(columns)): + token = state.push("th_open", "th", 1) + if aligns[i]: + token.attrs = {"style": "text-align:" + aligns[i]} + + token = state.push("inline", "", 0) + # note in markdown-it this map was removed in v12.0.0 however, we keep it, + # since it is helpful to propagate to children tokens + token.map = [startLine, startLine + 1] + token.content = columns[i].strip() + token.children = [] + + token = state.push("th_close", "th", -1) + + token = state.push("tr_close", "tr", -1) + token = state.push("thead_close", "thead", -1) + + autocompleted_cells = 0 + nextLine = startLine + 2 + while nextLine < endLine: + if state.sCount[nextLine] < state.blkIndent: + break + + terminate = False + for i in range(len(terminatorRules)): + if terminatorRules[i](state, nextLine, endLine, True): + terminate = True + break + + if terminate: + break + lineText = getLine(state, nextLine).strip() + if not lineText: + break + if state.is_code_block(nextLine): + break + columns = escapedSplit(lineText) + if columns and columns[0] == "": + columns.pop(0) + if columns and columns[-1] == "": + columns.pop() + + # note: autocomplete count can be negative if user specifies more columns than header, + # but that does not affect intended use (which is limiting expansion) + autocompleted_cells += columnCount - len(columns) + if autocompleted_cells > MAX_AUTOCOMPLETED_CELLS: + break + + if nextLine == startLine + 2: + token = state.push("tbody_open", "tbody", 1) + token.map = tbodyLines = [startLine + 2, 0] + + token = state.push("tr_open", "tr", 1) + token.map = [nextLine, nextLine + 1] + + for i in range(columnCount): + token = state.push("td_open", "td", 1) + if aligns[i]: + token.attrs = {"style": "text-align:" + aligns[i]} + + token = state.push("inline", "", 0) + # note in markdown-it this map was removed in v12.0.0 however, we keep it, + # since it is helpful to propagate to children tokens + token.map = [nextLine, nextLine + 1] + try: + token.content = columns[i].strip() if columns[i] else "" + except IndexError: + token.content = "" + token.children = [] + + token = state.push("td_close", "td", -1) + + token = state.push("tr_close", "tr", -1) + + nextLine += 1 + + if tbodyLines: + token = state.push("tbody_close", "tbody", -1) + tbodyLines[1] = nextLine + + token = state.push("table_close", "table", -1) + + tableLines[1] = nextLine + state.parentType = oldParentType + state.line = nextLine + return True diff --git a/lib/markdown_it/rules_core/__init__.py b/lib/markdown_it/rules_core/__init__.py new file mode 100644 index 0000000..e7d7753 --- /dev/null +++ b/lib/markdown_it/rules_core/__init__.py @@ -0,0 +1,19 @@ +__all__ = ( + "StateCore", + "block", + "inline", + "linkify", + "normalize", + "replace", + "smartquotes", + "text_join", +) + +from .block import block +from .inline import inline +from .linkify import linkify +from .normalize import normalize +from .replacements import replace +from .smartquotes import smartquotes +from .state_core import StateCore +from .text_join import text_join diff --git a/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..c197c7e Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc new file mode 100644 index 0000000..af5505c Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc new file mode 100644 index 0000000..9d3b6bd Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc new file mode 100644 index 0000000..e48cabf Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc new file mode 100644 index 0000000..3aa299d Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc new file mode 100644 index 0000000..1bdeba6 Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc new file mode 100644 index 0000000..c5b9a31 Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc new file mode 100644 index 0000000..bbe2789 Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc b/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc new file mode 100644 index 0000000..9190b35 Binary files /dev/null and b/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_core/block.py b/lib/markdown_it/rules_core/block.py new file mode 100644 index 0000000..a6c3bb8 --- /dev/null +++ b/lib/markdown_it/rules_core/block.py @@ -0,0 +1,13 @@ +from ..token import Token +from .state_core import StateCore + + +def block(state: StateCore) -> None: + if state.inlineMode: + token = Token("inline", "", 0) + token.content = state.src + token.map = [0, 1] + token.children = [] + state.tokens.append(token) + else: + state.md.block.parse(state.src, state.md, state.env, state.tokens) diff --git a/lib/markdown_it/rules_core/inline.py b/lib/markdown_it/rules_core/inline.py new file mode 100644 index 0000000..c3fd0b5 --- /dev/null +++ b/lib/markdown_it/rules_core/inline.py @@ -0,0 +1,10 @@ +from .state_core import StateCore + + +def inline(state: StateCore) -> None: + """Parse inlines""" + for token in state.tokens: + if token.type == "inline": + if token.children is None: + token.children = [] + state.md.inline.parse(token.content, state.md, state.env, token.children) diff --git a/lib/markdown_it/rules_core/linkify.py b/lib/markdown_it/rules_core/linkify.py new file mode 100644 index 0000000..efbc9d4 --- /dev/null +++ b/lib/markdown_it/rules_core/linkify.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import re +from typing import Protocol + +from ..common.utils import arrayReplaceAt, isLinkClose, isLinkOpen +from ..token import Token +from .state_core import StateCore + +HTTP_RE = re.compile(r"^http://") +MAILTO_RE = re.compile(r"^mailto:") +TEST_MAILTO_RE = re.compile(r"^mailto:", flags=re.IGNORECASE) + + +def linkify(state: StateCore) -> None: + """Rule for identifying plain-text links.""" + if not state.md.options.linkify: + return + + if not state.md.linkify: + raise ModuleNotFoundError("Linkify enabled but not installed.") + + for inline_token in state.tokens: + if inline_token.type != "inline" or not state.md.linkify.pretest( + inline_token.content + ): + continue + + tokens = inline_token.children + + htmlLinkLevel = 0 + + # We scan from the end, to keep position when new tags added. + # Use reversed logic in links start/end match + assert tokens is not None + i = len(tokens) + while i >= 1: + i -= 1 + assert isinstance(tokens, list) + currentToken = tokens[i] + + # Skip content of markdown links + if currentToken.type == "link_close": + i -= 1 + while ( + tokens[i].level != currentToken.level + and tokens[i].type != "link_open" + ): + i -= 1 + continue + + # Skip content of html tag links + if currentToken.type == "html_inline": + if isLinkOpen(currentToken.content) and htmlLinkLevel > 0: + htmlLinkLevel -= 1 + if isLinkClose(currentToken.content): + htmlLinkLevel += 1 + if htmlLinkLevel > 0: + continue + + if currentToken.type == "text" and state.md.linkify.test( + currentToken.content + ): + text = currentToken.content + links: list[_LinkType] = state.md.linkify.match(text) or [] + + # Now split string to nodes + nodes = [] + level = currentToken.level + lastPos = 0 + + # forbid escape sequence at the start of the string, + # this avoids http\://example.com/ from being linkified as + # http://example.com/ + if ( + links + and links[0].index == 0 + and i > 0 + and tokens[i - 1].type == "text_special" + ): + links = links[1:] + + for link in links: + url = link.url + fullUrl = state.md.normalizeLink(url) + if not state.md.validateLink(fullUrl): + continue + + urlText = link.text + + # Linkifier might send raw hostnames like "example.com", where url + # starts with domain name. So we prepend http:// in those cases, + # and remove it afterwards. + if not link.schema: + urlText = HTTP_RE.sub( + "", state.md.normalizeLinkText("http://" + urlText) + ) + elif link.schema == "mailto:" and TEST_MAILTO_RE.search(urlText): + urlText = MAILTO_RE.sub( + "", state.md.normalizeLinkText("mailto:" + urlText) + ) + else: + urlText = state.md.normalizeLinkText(urlText) + + pos = link.index + + if pos > lastPos: + token = Token("text", "", 0) + token.content = text[lastPos:pos] + token.level = level + nodes.append(token) + + token = Token("link_open", "a", 1) + token.attrs = {"href": fullUrl} + token.level = level + level += 1 + token.markup = "linkify" + token.info = "auto" + nodes.append(token) + + token = Token("text", "", 0) + token.content = urlText + token.level = level + nodes.append(token) + + token = Token("link_close", "a", -1) + level -= 1 + token.level = level + token.markup = "linkify" + token.info = "auto" + nodes.append(token) + + lastPos = link.last_index + + if lastPos < len(text): + token = Token("text", "", 0) + token.content = text[lastPos:] + token.level = level + nodes.append(token) + + inline_token.children = tokens = arrayReplaceAt(tokens, i, nodes) + + +class _LinkType(Protocol): + url: str + text: str + index: int + last_index: int + schema: str | None diff --git a/lib/markdown_it/rules_core/normalize.py b/lib/markdown_it/rules_core/normalize.py new file mode 100644 index 0000000..3243924 --- /dev/null +++ b/lib/markdown_it/rules_core/normalize.py @@ -0,0 +1,19 @@ +"""Normalize input string.""" + +import re + +from .state_core import StateCore + +# https://spec.commonmark.org/0.29/#line-ending +NEWLINES_RE = re.compile(r"\r\n?|\n") +NULL_RE = re.compile(r"\0") + + +def normalize(state: StateCore) -> None: + # Normalize newlines + string = NEWLINES_RE.sub("\n", state.src) + + # Replace NULL characters + string = NULL_RE.sub("\ufffd", string) + + state.src = string diff --git a/lib/markdown_it/rules_core/replacements.py b/lib/markdown_it/rules_core/replacements.py new file mode 100644 index 0000000..bcc9980 --- /dev/null +++ b/lib/markdown_it/rules_core/replacements.py @@ -0,0 +1,127 @@ +"""Simple typographic replacements + +* ``(c)``, ``(C)`` → © +* ``(tm)``, ``(TM)`` → ™ +* ``(r)``, ``(R)`` → ® +* ``+-`` → ± +* ``...`` → … +* ``?....`` → ?.. +* ``!....`` → !.. +* ``????????`` → ??? +* ``!!!!!`` → !!! +* ``,,,`` → , +* ``--`` → &ndash +* ``---`` → &mdash +""" + +from __future__ import annotations + +import logging +import re + +from ..token import Token +from .state_core import StateCore + +LOGGER = logging.getLogger(__name__) + +# TODO: +# - fractionals 1/2, 1/4, 3/4 -> ½, ¼, ¾ +# - multiplication 2 x 4 -> 2 × 4 + +RARE_RE = re.compile(r"\+-|\.\.|\?\?\?\?|!!!!|,,|--") + +# Workaround for phantomjs - need regex without /g flag, +# or root check will fail every second time +# SCOPED_ABBR_TEST_RE = r"\((c|tm|r)\)" + +SCOPED_ABBR_RE = re.compile(r"\((c|tm|r)\)", flags=re.IGNORECASE) + +PLUS_MINUS_RE = re.compile(r"\+-") + +ELLIPSIS_RE = re.compile(r"\.{2,}") + +ELLIPSIS_QUESTION_EXCLAMATION_RE = re.compile(r"([?!])…") + +QUESTION_EXCLAMATION_RE = re.compile(r"([?!]){4,}") + +COMMA_RE = re.compile(r",{2,}") + +EM_DASH_RE = re.compile(r"(^|[^-])---(?=[^-]|$)", flags=re.MULTILINE) + +EN_DASH_RE = re.compile(r"(^|\s)--(?=\s|$)", flags=re.MULTILINE) + +EN_DASH_INDENT_RE = re.compile(r"(^|[^-\s])--(?=[^-\s]|$)", flags=re.MULTILINE) + + +SCOPED_ABBR = {"c": "©", "r": "®", "tm": "™"} + + +def replaceFn(match: re.Match[str]) -> str: + return SCOPED_ABBR[match.group(1).lower()] + + +def replace_scoped(inlineTokens: list[Token]) -> None: + inside_autolink = 0 + + for token in inlineTokens: + if token.type == "text" and not inside_autolink: + token.content = SCOPED_ABBR_RE.sub(replaceFn, token.content) + + if token.type == "link_open" and token.info == "auto": + inside_autolink -= 1 + + if token.type == "link_close" and token.info == "auto": + inside_autolink += 1 + + +def replace_rare(inlineTokens: list[Token]) -> None: + inside_autolink = 0 + + for token in inlineTokens: + if ( + token.type == "text" + and (not inside_autolink) + and RARE_RE.search(token.content) + ): + # +- -> ± + token.content = PLUS_MINUS_RE.sub("±", token.content) + + # .., ..., ....... -> … + token.content = ELLIPSIS_RE.sub("…", token.content) + + # but ?..... & !..... -> ?.. & !.. + token.content = ELLIPSIS_QUESTION_EXCLAMATION_RE.sub("\\1..", token.content) + token.content = QUESTION_EXCLAMATION_RE.sub("\\1\\1\\1", token.content) + + # ,, ,,, ,,,, -> , + token.content = COMMA_RE.sub(",", token.content) + + # em-dash + token.content = EM_DASH_RE.sub("\\1\u2014", token.content) + + # en-dash + token.content = EN_DASH_RE.sub("\\1\u2013", token.content) + token.content = EN_DASH_INDENT_RE.sub("\\1\u2013", token.content) + + if token.type == "link_open" and token.info == "auto": + inside_autolink -= 1 + + if token.type == "link_close" and token.info == "auto": + inside_autolink += 1 + + +def replace(state: StateCore) -> None: + if not state.md.options.typographer: + return + + for token in state.tokens: + if token.type != "inline": + continue + if token.children is None: + continue + + if SCOPED_ABBR_RE.search(token.content): + replace_scoped(token.children) + + if RARE_RE.search(token.content): + replace_rare(token.children) diff --git a/lib/markdown_it/rules_core/smartquotes.py b/lib/markdown_it/rules_core/smartquotes.py new file mode 100644 index 0000000..f9b8b45 --- /dev/null +++ b/lib/markdown_it/rules_core/smartquotes.py @@ -0,0 +1,202 @@ +"""Convert straight quotation marks to typographic ones""" + +from __future__ import annotations + +import re +from typing import Any + +from ..common.utils import charCodeAt, isMdAsciiPunct, isPunctChar, isWhiteSpace +from ..token import Token +from .state_core import StateCore + +QUOTE_TEST_RE = re.compile(r"['\"]") +QUOTE_RE = re.compile(r"['\"]") +APOSTROPHE = "\u2019" # ’ + + +def replaceAt(string: str, index: int, ch: str) -> str: + # When the index is negative, the behavior is different from the js version. + # But basically, the index will not be negative. + assert index >= 0 + return string[:index] + ch + string[index + 1 :] + + +def process_inlines(tokens: list[Token], state: StateCore) -> None: + stack: list[dict[str, Any]] = [] + + for i, token in enumerate(tokens): + thisLevel = token.level + + j = 0 + for j in range(len(stack))[::-1]: + if stack[j]["level"] <= thisLevel: + break + else: + # When the loop is terminated without a "break". + # Subtract 1 to get the same index as the js version. + j -= 1 + + stack = stack[: j + 1] + + if token.type != "text": + continue + + text = token.content + pos = 0 + maximum = len(text) + + while pos < maximum: + goto_outer = False + lastIndex = pos + t = QUOTE_RE.search(text[lastIndex:]) + if not t: + break + + canOpen = canClose = True + pos = t.start(0) + lastIndex + 1 + isSingle = t.group(0) == "'" + + # Find previous character, + # default to space if it's the beginning of the line + lastChar: None | int = 0x20 + + if t.start(0) + lastIndex - 1 >= 0: + lastChar = charCodeAt(text, t.start(0) + lastIndex - 1) + else: + for j in range(i)[::-1]: + if tokens[j].type == "softbreak" or tokens[j].type == "hardbreak": + break + # should skip all tokens except 'text', 'html_inline' or 'code_inline' + if not tokens[j].content: + continue + + lastChar = charCodeAt(tokens[j].content, len(tokens[j].content) - 1) + break + + # Find next character, + # default to space if it's the end of the line + nextChar: None | int = 0x20 + + if pos < maximum: + nextChar = charCodeAt(text, pos) + else: + for j in range(i + 1, len(tokens)): + # nextChar defaults to 0x20 + if tokens[j].type == "softbreak" or tokens[j].type == "hardbreak": + break + # should skip all tokens except 'text', 'html_inline' or 'code_inline' + if not tokens[j].content: + continue + + nextChar = charCodeAt(tokens[j].content, 0) + break + + isLastPunctChar = lastChar is not None and ( + isMdAsciiPunct(lastChar) or isPunctChar(chr(lastChar)) + ) + isNextPunctChar = nextChar is not None and ( + isMdAsciiPunct(nextChar) or isPunctChar(chr(nextChar)) + ) + + isLastWhiteSpace = lastChar is not None and isWhiteSpace(lastChar) + isNextWhiteSpace = nextChar is not None and isWhiteSpace(nextChar) + + if isNextWhiteSpace: # noqa: SIM114 + canOpen = False + elif isNextPunctChar and not (isLastWhiteSpace or isLastPunctChar): + canOpen = False + + if isLastWhiteSpace: # noqa: SIM114 + canClose = False + elif isLastPunctChar and not (isNextWhiteSpace or isNextPunctChar): + canClose = False + + if nextChar == 0x22 and t.group(0) == '"': # 0x22: " # noqa: SIM102 + if ( + lastChar is not None and lastChar >= 0x30 and lastChar <= 0x39 + ): # 0x30: 0, 0x39: 9 + # special case: 1"" - count first quote as an inch + canClose = canOpen = False + + if canOpen and canClose: + # Replace quotes in the middle of punctuation sequence, but not + # in the middle of the words, i.e.: + # + # 1. foo " bar " baz - not replaced + # 2. foo-"-bar-"-baz - replaced + # 3. foo"bar"baz - not replaced + canOpen = isLastPunctChar + canClose = isNextPunctChar + + if not canOpen and not canClose: + # middle of word + if isSingle: + token.content = replaceAt( + token.content, t.start(0) + lastIndex, APOSTROPHE + ) + continue + + if canClose: + # this could be a closing quote, rewind the stack to get a match + for j in range(len(stack))[::-1]: + item = stack[j] + if stack[j]["level"] < thisLevel: + break + if item["single"] == isSingle and stack[j]["level"] == thisLevel: + item = stack[j] + + if isSingle: + openQuote = state.md.options.quotes[2] + closeQuote = state.md.options.quotes[3] + else: + openQuote = state.md.options.quotes[0] + closeQuote = state.md.options.quotes[1] + + # replace token.content *before* tokens[item.token].content, + # because, if they are pointing at the same token, replaceAt + # could mess up indices when quote length != 1 + token.content = replaceAt( + token.content, t.start(0) + lastIndex, closeQuote + ) + tokens[item["token"]].content = replaceAt( + tokens[item["token"]].content, item["pos"], openQuote + ) + + pos += len(closeQuote) - 1 + if item["token"] == i: + pos += len(openQuote) - 1 + + text = token.content + maximum = len(text) + + stack = stack[:j] + goto_outer = True + break + if goto_outer: + goto_outer = False + continue + + if canOpen: + stack.append( + { + "token": i, + "pos": t.start(0) + lastIndex, + "single": isSingle, + "level": thisLevel, + } + ) + elif canClose and isSingle: + token.content = replaceAt( + token.content, t.start(0) + lastIndex, APOSTROPHE + ) + + +def smartquotes(state: StateCore) -> None: + if not state.md.options.typographer: + return + + for token in state.tokens: + if token.type != "inline" or not QUOTE_RE.search(token.content): + continue + if token.children is not None: + process_inlines(token.children, state) diff --git a/lib/markdown_it/rules_core/state_core.py b/lib/markdown_it/rules_core/state_core.py new file mode 100644 index 0000000..a938041 --- /dev/null +++ b/lib/markdown_it/rules_core/state_core.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..ruler import StateBase +from ..token import Token +from ..utils import EnvType + +if TYPE_CHECKING: + from markdown_it import MarkdownIt + + +class StateCore(StateBase): + def __init__( + self, + src: str, + md: MarkdownIt, + env: EnvType, + tokens: list[Token] | None = None, + ) -> None: + self.src = src + self.md = md # link to parser instance + self.env = env + self.tokens: list[Token] = tokens or [] + self.inlineMode = False diff --git a/lib/markdown_it/rules_core/text_join.py b/lib/markdown_it/rules_core/text_join.py new file mode 100644 index 0000000..5379f6d --- /dev/null +++ b/lib/markdown_it/rules_core/text_join.py @@ -0,0 +1,35 @@ +"""Join raw text tokens with the rest of the text + +This is set as a separate rule to provide an opportunity for plugins +to run text replacements after text join, but before escape join. + +For example, `\\:)` shouldn't be replaced with an emoji. +""" + +from __future__ import annotations + +from ..token import Token +from .state_core import StateCore + + +def text_join(state: StateCore) -> None: + """Join raw text for escape sequences (`text_special`) tokens with the rest of the text""" + + for inline_token in state.tokens[:]: + if inline_token.type != "inline": + continue + + # convert text_special to text and join all adjacent text nodes + new_tokens: list[Token] = [] + for child_token in inline_token.children or []: + if child_token.type == "text_special": + child_token.type = "text" + if ( + child_token.type == "text" + and new_tokens + and new_tokens[-1].type == "text" + ): + new_tokens[-1].content += child_token.content + else: + new_tokens.append(child_token) + inline_token.children = new_tokens diff --git a/lib/markdown_it/rules_inline/__init__.py b/lib/markdown_it/rules_inline/__init__.py new file mode 100644 index 0000000..d82ef8f --- /dev/null +++ b/lib/markdown_it/rules_inline/__init__.py @@ -0,0 +1,31 @@ +__all__ = ( + "StateInline", + "autolink", + "backtick", + "emphasis", + "entity", + "escape", + "fragments_join", + "html_inline", + "image", + "link", + "link_pairs", + "linkify", + "newline", + "strikethrough", + "text", +) +from . import emphasis, strikethrough +from .autolink import autolink +from .backticks import backtick +from .balance_pairs import link_pairs +from .entity import entity +from .escape import escape +from .fragments_join import fragments_join +from .html_inline import html_inline +from .image import image +from .link import link +from .linkify import linkify +from .newline import newline +from .state_inline import StateInline +from .text import text diff --git a/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..d11fe81 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc new file mode 100644 index 0000000..170c101 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc new file mode 100644 index 0000000..9958038 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc new file mode 100644 index 0000000..bd1ab9d Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc new file mode 100644 index 0000000..05eabeb Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc new file mode 100644 index 0000000..8f1ad41 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc new file mode 100644 index 0000000..1650fd2 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc new file mode 100644 index 0000000..b7a02d6 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc new file mode 100644 index 0000000..cbdcc08 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc new file mode 100644 index 0000000..40bd7ac Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc new file mode 100644 index 0000000..853e646 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc new file mode 100644 index 0000000..ad57602 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc new file mode 100644 index 0000000..e34886b Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc new file mode 100644 index 0000000..5ba02aa Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc new file mode 100644 index 0000000..4aab224 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc b/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc new file mode 100644 index 0000000..ea97dd1 Binary files /dev/null and b/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc differ diff --git a/lib/markdown_it/rules_inline/autolink.py b/lib/markdown_it/rules_inline/autolink.py new file mode 100644 index 0000000..6546e25 --- /dev/null +++ b/lib/markdown_it/rules_inline/autolink.py @@ -0,0 +1,77 @@ +# Process autolinks '' +import re + +from .state_inline import StateInline + +EMAIL_RE = re.compile( + r"^([a-zA-Z0-9.!#$%&\'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)$" +) +AUTOLINK_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.\-]{1,31}):([^<>\x00-\x20]*)$") + + +def autolink(state: StateInline, silent: bool) -> bool: + pos = state.pos + + if state.src[pos] != "<": + return False + + start = state.pos + maximum = state.posMax + + while True: + pos += 1 + if pos >= maximum: + return False + + ch = state.src[pos] + + if ch == "<": + return False + if ch == ">": + break + + url = state.src[start + 1 : pos] + + if AUTOLINK_RE.search(url) is not None: + fullUrl = state.md.normalizeLink(url) + if not state.md.validateLink(fullUrl): + return False + + if not silent: + token = state.push("link_open", "a", 1) + token.attrs = {"href": fullUrl} + token.markup = "autolink" + token.info = "auto" + + token = state.push("text", "", 0) + token.content = state.md.normalizeLinkText(url) + + token = state.push("link_close", "a", -1) + token.markup = "autolink" + token.info = "auto" + + state.pos += len(url) + 2 + return True + + if EMAIL_RE.search(url) is not None: + fullUrl = state.md.normalizeLink("mailto:" + url) + if not state.md.validateLink(fullUrl): + return False + + if not silent: + token = state.push("link_open", "a", 1) + token.attrs = {"href": fullUrl} + token.markup = "autolink" + token.info = "auto" + + token = state.push("text", "", 0) + token.content = state.md.normalizeLinkText(url) + + token = state.push("link_close", "a", -1) + token.markup = "autolink" + token.info = "auto" + + state.pos += len(url) + 2 + return True + + return False diff --git a/lib/markdown_it/rules_inline/backticks.py b/lib/markdown_it/rules_inline/backticks.py new file mode 100644 index 0000000..fc60d6b --- /dev/null +++ b/lib/markdown_it/rules_inline/backticks.py @@ -0,0 +1,72 @@ +# Parse backticks +import re + +from .state_inline import StateInline + +regex = re.compile("^ (.+) $") + + +def backtick(state: StateInline, silent: bool) -> bool: + pos = state.pos + + if state.src[pos] != "`": + return False + + start = pos + pos += 1 + maximum = state.posMax + + # scan marker length + while pos < maximum and (state.src[pos] == "`"): + pos += 1 + + marker = state.src[start:pos] + openerLength = len(marker) + + if state.backticksScanned and state.backticks.get(openerLength, 0) <= start: + if not silent: + state.pending += marker + state.pos += openerLength + return True + + matchStart = matchEnd = pos + + # Nothing found in the cache, scan until the end of the line (or until marker is found) + while True: + try: + matchStart = state.src.index("`", matchEnd) + except ValueError: + break + matchEnd = matchStart + 1 + + # scan marker length + while matchEnd < maximum and (state.src[matchEnd] == "`"): + matchEnd += 1 + + closerLength = matchEnd - matchStart + + if closerLength == openerLength: + # Found matching closer length. + if not silent: + token = state.push("code_inline", "code", 0) + token.markup = marker + token.content = state.src[pos:matchStart].replace("\n", " ") + if ( + token.content.startswith(" ") + and token.content.endswith(" ") + and len(token.content.strip()) > 0 + ): + token.content = token.content[1:-1] + state.pos = matchEnd + return True + + # Some different length found, put it in cache as upper limit of where closer can be found + state.backticks[closerLength] = matchStart + + # Scanned through the end, didn't find anything + state.backticksScanned = True + + if not silent: + state.pending += marker + state.pos += openerLength + return True diff --git a/lib/markdown_it/rules_inline/balance_pairs.py b/lib/markdown_it/rules_inline/balance_pairs.py new file mode 100644 index 0000000..9c63b27 --- /dev/null +++ b/lib/markdown_it/rules_inline/balance_pairs.py @@ -0,0 +1,138 @@ +"""Balance paired characters (*, _, etc) in inline tokens.""" + +from __future__ import annotations + +from .state_inline import Delimiter, StateInline + + +def processDelimiters(state: StateInline, delimiters: list[Delimiter]) -> None: + """For each opening emphasis-like marker find a matching closing one.""" + if not delimiters: + return + + openersBottom = {} + maximum = len(delimiters) + + # headerIdx is the first delimiter of the current (where closer is) delimiter run + headerIdx = 0 + lastTokenIdx = -2 # needs any value lower than -1 + jumps: list[int] = [] + closerIdx = 0 + while closerIdx < maximum: + closer = delimiters[closerIdx] + + jumps.append(0) + + # markers belong to same delimiter run if: + # - they have adjacent tokens + # - AND markers are the same + # + if ( + delimiters[headerIdx].marker != closer.marker + or lastTokenIdx != closer.token - 1 + ): + headerIdx = closerIdx + lastTokenIdx = closer.token + + # Length is only used for emphasis-specific "rule of 3", + # if it's not defined (in strikethrough or 3rd party plugins), + # we can default it to 0 to disable those checks. + # + closer.length = closer.length or 0 + + if not closer.close: + closerIdx += 1 + continue + + # Previously calculated lower bounds (previous fails) + # for each marker, each delimiter length modulo 3, + # and for whether this closer can be an opener; + # https://github.com/commonmark/cmark/commit/34250e12ccebdc6372b8b49c44fab57c72443460 + if closer.marker not in openersBottom: + openersBottom[closer.marker] = [-1, -1, -1, -1, -1, -1] + + minOpenerIdx = openersBottom[closer.marker][ + (3 if closer.open else 0) + (closer.length % 3) + ] + + openerIdx = headerIdx - jumps[headerIdx] - 1 + + newMinOpenerIdx = openerIdx + + while openerIdx > minOpenerIdx: + opener = delimiters[openerIdx] + + if opener.marker != closer.marker: + openerIdx -= jumps[openerIdx] + 1 + continue + + if opener.open and opener.end < 0: + isOddMatch = False + + # from spec: + # + # If one of the delimiters can both open and close emphasis, then the + # sum of the lengths of the delimiter runs containing the opening and + # closing delimiters must not be a multiple of 3 unless both lengths + # are multiples of 3. + # + if ( + (opener.close or closer.open) + and ((opener.length + closer.length) % 3 == 0) + and (opener.length % 3 != 0 or closer.length % 3 != 0) + ): + isOddMatch = True + + if not isOddMatch: + # If previous delimiter cannot be an opener, we can safely skip + # the entire sequence in future checks. This is required to make + # sure algorithm has linear complexity (see *_*_*_*_*_... case). + # + if openerIdx > 0 and not delimiters[openerIdx - 1].open: + lastJump = jumps[openerIdx - 1] + 1 + else: + lastJump = 0 + + jumps[closerIdx] = closerIdx - openerIdx + lastJump + jumps[openerIdx] = lastJump + + closer.open = False + opener.end = closerIdx + opener.close = False + newMinOpenerIdx = -1 + + # treat next token as start of run, + # it optimizes skips in **<...>**a**<...>** pathological case + lastTokenIdx = -2 + + break + + openerIdx -= jumps[openerIdx] + 1 + + if newMinOpenerIdx != -1: + # If match for this delimiter run failed, we want to set lower bound for + # future lookups. This is required to make sure algorithm has linear + # complexity. + # + # See details here: + # https:#github.com/commonmark/cmark/issues/178#issuecomment-270417442 + # + openersBottom[closer.marker][ + (3 if closer.open else 0) + ((closer.length or 0) % 3) + ] = newMinOpenerIdx + + closerIdx += 1 + + +def link_pairs(state: StateInline) -> None: + tokens_meta = state.tokens_meta + maximum = len(state.tokens_meta) + + processDelimiters(state, state.delimiters) + + curr = 0 + while curr < maximum: + curr_meta = tokens_meta[curr] + if curr_meta and "delimiters" in curr_meta: + processDelimiters(state, curr_meta["delimiters"]) + curr += 1 diff --git a/lib/markdown_it/rules_inline/emphasis.py b/lib/markdown_it/rules_inline/emphasis.py new file mode 100644 index 0000000..9a98f9e --- /dev/null +++ b/lib/markdown_it/rules_inline/emphasis.py @@ -0,0 +1,102 @@ +# Process *this* and _that_ +# +from __future__ import annotations + +from .state_inline import Delimiter, StateInline + + +def tokenize(state: StateInline, silent: bool) -> bool: + """Insert each marker as a separate text token, and add it to delimiter list""" + start = state.pos + marker = state.src[start] + + if silent: + return False + + if marker not in ("_", "*"): + return False + + scanned = state.scanDelims(state.pos, marker == "*") + + for _ in range(scanned.length): + token = state.push("text", "", 0) + token.content = marker + state.delimiters.append( + Delimiter( + marker=ord(marker), + length=scanned.length, + token=len(state.tokens) - 1, + end=-1, + open=scanned.can_open, + close=scanned.can_close, + ) + ) + + state.pos += scanned.length + + return True + + +def _postProcess(state: StateInline, delimiters: list[Delimiter]) -> None: + i = len(delimiters) - 1 + while i >= 0: + startDelim = delimiters[i] + + # /* _ */ /* * */ + if startDelim.marker != 0x5F and startDelim.marker != 0x2A: + i -= 1 + continue + + # Process only opening markers + if startDelim.end == -1: + i -= 1 + continue + + endDelim = delimiters[startDelim.end] + + # If the previous delimiter has the same marker and is adjacent to this one, + # merge those into one strong delimiter. + # + # `whatever` -> `whatever` + # + isStrong = ( + i > 0 + and delimiters[i - 1].end == startDelim.end + 1 + # check that first two markers match and adjacent + and delimiters[i - 1].marker == startDelim.marker + and delimiters[i - 1].token == startDelim.token - 1 + # check that last two markers are adjacent (we can safely assume they match) + and delimiters[startDelim.end + 1].token == endDelim.token + 1 + ) + + ch = chr(startDelim.marker) + + token = state.tokens[startDelim.token] + token.type = "strong_open" if isStrong else "em_open" + token.tag = "strong" if isStrong else "em" + token.nesting = 1 + token.markup = ch + ch if isStrong else ch + token.content = "" + + token = state.tokens[endDelim.token] + token.type = "strong_close" if isStrong else "em_close" + token.tag = "strong" if isStrong else "em" + token.nesting = -1 + token.markup = ch + ch if isStrong else ch + token.content = "" + + if isStrong: + state.tokens[delimiters[i - 1].token].content = "" + state.tokens[delimiters[startDelim.end + 1].token].content = "" + i -= 1 + + i -= 1 + + +def postProcess(state: StateInline) -> None: + """Walk through delimiter list and replace text tokens with tags.""" + _postProcess(state, state.delimiters) + + for token in state.tokens_meta: + if token and "delimiters" in token: + _postProcess(state, token["delimiters"]) diff --git a/lib/markdown_it/rules_inline/entity.py b/lib/markdown_it/rules_inline/entity.py new file mode 100644 index 0000000..ec9d396 --- /dev/null +++ b/lib/markdown_it/rules_inline/entity.py @@ -0,0 +1,53 @@ +# Process html entity - {, ¯, ", ... +import re + +from ..common.entities import entities +from ..common.utils import fromCodePoint, isValidEntityCode +from .state_inline import StateInline + +DIGITAL_RE = re.compile(r"^&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));", re.IGNORECASE) +NAMED_RE = re.compile(r"^&([a-z][a-z0-9]{1,31});", re.IGNORECASE) + + +def entity(state: StateInline, silent: bool) -> bool: + pos = state.pos + maximum = state.posMax + + if state.src[pos] != "&": + return False + + if pos + 1 >= maximum: + return False + + if state.src[pos + 1] == "#": + if match := DIGITAL_RE.search(state.src[pos:]): + if not silent: + match1 = match.group(1) + code = ( + int(match1[1:], 16) if match1[0].lower() == "x" else int(match1, 10) + ) + + token = state.push("text_special", "", 0) + token.content = ( + fromCodePoint(code) + if isValidEntityCode(code) + else fromCodePoint(0xFFFD) + ) + token.markup = match.group(0) + token.info = "entity" + + state.pos += len(match.group(0)) + return True + + else: + if (match := NAMED_RE.search(state.src[pos:])) and match.group(1) in entities: + if not silent: + token = state.push("text_special", "", 0) + token.content = entities[match.group(1)] + token.markup = match.group(0) + token.info = "entity" + + state.pos += len(match.group(0)) + return True + + return False diff --git a/lib/markdown_it/rules_inline/escape.py b/lib/markdown_it/rules_inline/escape.py new file mode 100644 index 0000000..0fca6c8 --- /dev/null +++ b/lib/markdown_it/rules_inline/escape.py @@ -0,0 +1,93 @@ +""" +Process escaped chars and hardbreaks +""" + +from ..common.utils import isStrSpace +from .state_inline import StateInline + + +def escape(state: StateInline, silent: bool) -> bool: + """Process escaped chars and hardbreaks.""" + pos = state.pos + maximum = state.posMax + + if state.src[pos] != "\\": + return False + + pos += 1 + + # '\' at the end of the inline block + if pos >= maximum: + return False + + ch1 = state.src[pos] + ch1_ord = ord(ch1) + if ch1 == "\n": + if not silent: + state.push("hardbreak", "br", 0) + pos += 1 + # skip leading whitespaces from next line + while pos < maximum: + ch = state.src[pos] + if not isStrSpace(ch): + break + pos += 1 + + state.pos = pos + return True + + escapedStr = state.src[pos] + + if ch1_ord >= 0xD800 and ch1_ord <= 0xDBFF and pos + 1 < maximum: + ch2 = state.src[pos + 1] + ch2_ord = ord(ch2) + if ch2_ord >= 0xDC00 and ch2_ord <= 0xDFFF: + escapedStr += ch2 + pos += 1 + + origStr = "\\" + escapedStr + + if not silent: + token = state.push("text_special", "", 0) + token.content = escapedStr if ch1 in _ESCAPED else origStr + token.markup = origStr + token.info = "escape" + + state.pos = pos + 1 + return True + + +_ESCAPED = { + "!", + '"', + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "[", + "\\", + "]", + "^", + "_", + "`", + "{", + "|", + "}", + "~", +} diff --git a/lib/markdown_it/rules_inline/fragments_join.py b/lib/markdown_it/rules_inline/fragments_join.py new file mode 100644 index 0000000..f795c13 --- /dev/null +++ b/lib/markdown_it/rules_inline/fragments_join.py @@ -0,0 +1,43 @@ +from .state_inline import StateInline + + +def fragments_join(state: StateInline) -> None: + """ + Clean up tokens after emphasis and strikethrough postprocessing: + merge adjacent text nodes into one and re-calculate all token levels + + This is necessary because initially emphasis delimiter markers (``*, _, ~``) + are treated as their own separate text tokens. Then emphasis rule either + leaves them as text (needed to merge with adjacent text) or turns them + into opening/closing tags (which messes up levels inside). + """ + level = 0 + maximum = len(state.tokens) + + curr = last = 0 + while curr < maximum: + # re-calculate levels after emphasis/strikethrough turns some text nodes + # into opening/closing tags + if state.tokens[curr].nesting < 0: + level -= 1 # closing tag + state.tokens[curr].level = level + if state.tokens[curr].nesting > 0: + level += 1 # opening tag + + if ( + state.tokens[curr].type == "text" + and curr + 1 < maximum + and state.tokens[curr + 1].type == "text" + ): + # collapse two adjacent text nodes + state.tokens[curr + 1].content = ( + state.tokens[curr].content + state.tokens[curr + 1].content + ) + else: + if curr != last: + state.tokens[last] = state.tokens[curr] + last += 1 + curr += 1 + + if curr != last: + del state.tokens[last:] diff --git a/lib/markdown_it/rules_inline/html_inline.py b/lib/markdown_it/rules_inline/html_inline.py new file mode 100644 index 0000000..9065e1d --- /dev/null +++ b/lib/markdown_it/rules_inline/html_inline.py @@ -0,0 +1,43 @@ +# Process html tags +from ..common.html_re import HTML_TAG_RE +from ..common.utils import isLinkClose, isLinkOpen +from .state_inline import StateInline + + +def isLetter(ch: int) -> bool: + lc = ch | 0x20 # to lower case + # /* a */ and /* z */ + return (lc >= 0x61) and (lc <= 0x7A) + + +def html_inline(state: StateInline, silent: bool) -> bool: + pos = state.pos + + if not state.md.options.get("html", None): + return False + + # Check start + maximum = state.posMax + if state.src[pos] != "<" or pos + 2 >= maximum: + return False + + # Quick fail on second char + ch = state.src[pos + 1] + if ch not in ("!", "?", "/") and not isLetter(ord(ch)): # /* / */ + return False + + match = HTML_TAG_RE.search(state.src[pos:]) + if not match: + return False + + if not silent: + token = state.push("html_inline", "", 0) + token.content = state.src[pos : pos + len(match.group(0))] + + if isLinkOpen(token.content): + state.linkLevel += 1 + if isLinkClose(token.content): + state.linkLevel -= 1 + + state.pos += len(match.group(0)) + return True diff --git a/lib/markdown_it/rules_inline/image.py b/lib/markdown_it/rules_inline/image.py new file mode 100644 index 0000000..005105b --- /dev/null +++ b/lib/markdown_it/rules_inline/image.py @@ -0,0 +1,148 @@ +# Process ![image]( "title") +from __future__ import annotations + +from ..common.utils import isStrSpace, normalizeReference +from ..token import Token +from .state_inline import StateInline + + +def image(state: StateInline, silent: bool) -> bool: + label = None + href = "" + oldPos = state.pos + max = state.posMax + + if state.src[state.pos] != "!": + return False + + if state.pos + 1 < state.posMax and state.src[state.pos + 1] != "[": + return False + + labelStart = state.pos + 2 + labelEnd = state.md.helpers.parseLinkLabel(state, state.pos + 1, False) + + # parser failed to find ']', so it's not a valid link + if labelEnd < 0: + return False + + pos = labelEnd + 1 + + if pos < max and state.src[pos] == "(": + # + # Inline link + # + + # [link]( "title" ) + # ^^ skipping these spaces + pos += 1 + while pos < max: + ch = state.src[pos] + if not isStrSpace(ch) and ch != "\n": + break + pos += 1 + + if pos >= max: + return False + + # [link]( "title" ) + # ^^^^^^ parsing link destination + start = pos + res = state.md.helpers.parseLinkDestination(state.src, pos, state.posMax) + if res.ok: + href = state.md.normalizeLink(res.str) + if state.md.validateLink(href): + pos = res.pos + else: + href = "" + + # [link]( "title" ) + # ^^ skipping these spaces + start = pos + while pos < max: + ch = state.src[pos] + if not isStrSpace(ch) and ch != "\n": + break + pos += 1 + + # [link]( "title" ) + # ^^^^^^^ parsing link title + res = state.md.helpers.parseLinkTitle(state.src, pos, state.posMax, None) + if pos < max and start != pos and res.ok: + title = res.str + pos = res.pos + + # [link]( "title" ) + # ^^ skipping these spaces + while pos < max: + ch = state.src[pos] + if not isStrSpace(ch) and ch != "\n": + break + pos += 1 + else: + title = "" + + if pos >= max or state.src[pos] != ")": + state.pos = oldPos + return False + + pos += 1 + + else: + # + # Link reference + # + if "references" not in state.env: + return False + + # /* [ */ + if pos < max and state.src[pos] == "[": + start = pos + 1 + pos = state.md.helpers.parseLinkLabel(state, pos) + if pos >= 0: + label = state.src[start:pos] + pos += 1 + else: + pos = labelEnd + 1 + else: + pos = labelEnd + 1 + + # covers label == '' and label == undefined + # (collapsed reference link and shortcut reference link respectively) + if not label: + label = state.src[labelStart:labelEnd] + + label = normalizeReference(label) + + ref = state.env["references"].get(label, None) + if not ref: + state.pos = oldPos + return False + + href = ref["href"] + title = ref["title"] + + # + # We found the end of the link, and know for a fact it's a valid link + # so all that's left to do is to call tokenizer. + # + if not silent: + content = state.src[labelStart:labelEnd] + + tokens: list[Token] = [] + state.md.inline.parse(content, state.md, state.env, tokens) + + token = state.push("image", "img", 0) + token.attrs = {"src": href, "alt": ""} + token.children = tokens or None + token.content = content + + if title: + token.attrSet("title", title) + + # note, this is not part of markdown-it JS, but is useful for renderers + if label and state.md.options.get("store_labels", False): + token.meta["label"] = label + + state.pos = pos + state.posMax = max + return True diff --git a/lib/markdown_it/rules_inline/link.py b/lib/markdown_it/rules_inline/link.py new file mode 100644 index 0000000..2e92c7d --- /dev/null +++ b/lib/markdown_it/rules_inline/link.py @@ -0,0 +1,149 @@ +# Process [link]( "stuff") + +from ..common.utils import isStrSpace, normalizeReference +from .state_inline import StateInline + + +def link(state: StateInline, silent: bool) -> bool: + href = "" + title = "" + label = None + oldPos = state.pos + maximum = state.posMax + start = state.pos + parseReference = True + + if state.src[state.pos] != "[": + return False + + labelStart = state.pos + 1 + labelEnd = state.md.helpers.parseLinkLabel(state, state.pos, True) + + # parser failed to find ']', so it's not a valid link + if labelEnd < 0: + return False + + pos = labelEnd + 1 + + if pos < maximum and state.src[pos] == "(": + # + # Inline link + # + + # might have found a valid shortcut link, disable reference parsing + parseReference = False + + # [link]( "title" ) + # ^^ skipping these spaces + pos += 1 + while pos < maximum: + ch = state.src[pos] + if not isStrSpace(ch) and ch != "\n": + break + pos += 1 + + if pos >= maximum: + return False + + # [link]( "title" ) + # ^^^^^^ parsing link destination + start = pos + res = state.md.helpers.parseLinkDestination(state.src, pos, state.posMax) + if res.ok: + href = state.md.normalizeLink(res.str) + if state.md.validateLink(href): + pos = res.pos + else: + href = "" + + # [link]( "title" ) + # ^^ skipping these spaces + start = pos + while pos < maximum: + ch = state.src[pos] + if not isStrSpace(ch) and ch != "\n": + break + pos += 1 + + # [link]( "title" ) + # ^^^^^^^ parsing link title + res = state.md.helpers.parseLinkTitle(state.src, pos, state.posMax) + if pos < maximum and start != pos and res.ok: + title = res.str + pos = res.pos + + # [link]( "title" ) + # ^^ skipping these spaces + while pos < maximum: + ch = state.src[pos] + if not isStrSpace(ch) and ch != "\n": + break + pos += 1 + + if pos >= maximum or state.src[pos] != ")": + # parsing a valid shortcut link failed, fallback to reference + parseReference = True + + pos += 1 + + if parseReference: + # + # Link reference + # + if "references" not in state.env: + return False + + if pos < maximum and state.src[pos] == "[": + start = pos + 1 + pos = state.md.helpers.parseLinkLabel(state, pos) + if pos >= 0: + label = state.src[start:pos] + pos += 1 + else: + pos = labelEnd + 1 + + else: + pos = labelEnd + 1 + + # covers label == '' and label == undefined + # (collapsed reference link and shortcut reference link respectively) + if not label: + label = state.src[labelStart:labelEnd] + + label = normalizeReference(label) + + ref = state.env["references"].get(label, None) + if not ref: + state.pos = oldPos + return False + + href = ref["href"] + title = ref["title"] + + # + # We found the end of the link, and know for a fact it's a valid link + # so all that's left to do is to call tokenizer. + # + if not silent: + state.pos = labelStart + state.posMax = labelEnd + + token = state.push("link_open", "a", 1) + token.attrs = {"href": href} + + if title: + token.attrSet("title", title) + + # note, this is not part of markdown-it JS, but is useful for renderers + if label and state.md.options.get("store_labels", False): + token.meta["label"] = label + + state.linkLevel += 1 + state.md.inline.tokenize(state) + state.linkLevel -= 1 + + token = state.push("link_close", "a", -1) + + state.pos = pos + state.posMax = maximum + return True diff --git a/lib/markdown_it/rules_inline/linkify.py b/lib/markdown_it/rules_inline/linkify.py new file mode 100644 index 0000000..3669396 --- /dev/null +++ b/lib/markdown_it/rules_inline/linkify.py @@ -0,0 +1,62 @@ +"""Process links like https://example.org/""" + +import re + +from .state_inline import StateInline + +# RFC3986: scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) +SCHEME_RE = re.compile(r"(?:^|[^a-z0-9.+-])([a-z][a-z0-9.+-]*)$", re.IGNORECASE) + + +def linkify(state: StateInline, silent: bool) -> bool: + """Rule for identifying plain-text links.""" + if not state.md.options.linkify: + return False + if state.linkLevel > 0: + return False + if not state.md.linkify: + raise ModuleNotFoundError("Linkify enabled but not installed.") + + pos = state.pos + maximum = state.posMax + + if ( + (pos + 3) > maximum + or state.src[pos] != ":" + or state.src[pos + 1] != "/" + or state.src[pos + 2] != "/" + ): + return False + + if not (match := SCHEME_RE.search(state.pending)): + return False + + proto = match.group(1) + if not (link := state.md.linkify.match_at_start(state.src[pos - len(proto) :])): + return False + url: str = link.url + + # disallow '*' at the end of the link (conflicts with emphasis) + url = url.rstrip("*") + + full_url = state.md.normalizeLink(url) + if not state.md.validateLink(full_url): + return False + + if not silent: + state.pending = state.pending[: -len(proto)] + + token = state.push("link_open", "a", 1) + token.attrs = {"href": full_url} + token.markup = "linkify" + token.info = "auto" + + token = state.push("text", "", 0) + token.content = state.md.normalizeLinkText(url) + + token = state.push("link_close", "a", -1) + token.markup = "linkify" + token.info = "auto" + + state.pos += len(url) - len(proto) + return True diff --git a/lib/markdown_it/rules_inline/newline.py b/lib/markdown_it/rules_inline/newline.py new file mode 100644 index 0000000..d05ee6d --- /dev/null +++ b/lib/markdown_it/rules_inline/newline.py @@ -0,0 +1,44 @@ +"""Proceess '\n'.""" + +from ..common.utils import charStrAt, isStrSpace +from .state_inline import StateInline + + +def newline(state: StateInline, silent: bool) -> bool: + pos = state.pos + + if state.src[pos] != "\n": + return False + + pmax = len(state.pending) - 1 + maximum = state.posMax + + # ' \n' -> hardbreak + # Lookup in pending chars is bad practice! Don't copy to other rules! + # Pending string is stored in concat mode, indexed lookups will cause + # conversion to flat mode. + if not silent: + if pmax >= 0 and charStrAt(state.pending, pmax) == " ": + if pmax >= 1 and charStrAt(state.pending, pmax - 1) == " ": + # Find whitespaces tail of pending chars. + ws = pmax - 1 + while ws >= 1 and charStrAt(state.pending, ws - 1) == " ": + ws -= 1 + state.pending = state.pending[:ws] + + state.push("hardbreak", "br", 0) + else: + state.pending = state.pending[:-1] + state.push("softbreak", "br", 0) + + else: + state.push("softbreak", "br", 0) + + pos += 1 + + # skip heading spaces for next line + while pos < maximum and isStrSpace(state.src[pos]): + pos += 1 + + state.pos = pos + return True diff --git a/lib/markdown_it/rules_inline/state_inline.py b/lib/markdown_it/rules_inline/state_inline.py new file mode 100644 index 0000000..50dc412 --- /dev/null +++ b/lib/markdown_it/rules_inline/state_inline.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from collections import namedtuple +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +from ..common.utils import isMdAsciiPunct, isPunctChar, isWhiteSpace +from ..ruler import StateBase +from ..token import Token +from ..utils import EnvType + +if TYPE_CHECKING: + from markdown_it import MarkdownIt + + +@dataclass(slots=True) +class Delimiter: + # Char code of the starting marker (number). + marker: int + + # Total length of these series of delimiters. + length: int + + # A position of the token this delimiter corresponds to. + token: int + + # If this delimiter is matched as a valid opener, `end` will be + # equal to its position, otherwise it's `-1`. + end: int + + # Boolean flags that determine if this delimiter could open or close + # an emphasis. + open: bool + close: bool + + level: bool | None = None + + +Scanned = namedtuple("Scanned", ["can_open", "can_close", "length"]) + + +class StateInline(StateBase): + def __init__( + self, src: str, md: MarkdownIt, env: EnvType, outTokens: list[Token] + ) -> None: + self.src = src + self.env = env + self.md = md + self.tokens = outTokens + self.tokens_meta: list[dict[str, Any] | None] = [None] * len(outTokens) + + self.pos = 0 + self.posMax = len(self.src) + self.level = 0 + self.pending = "" + self.pendingLevel = 0 + + # Stores { start: end } pairs. Useful for backtrack + # optimization of pairs parse (emphasis, strikes). + self.cache: dict[int, int] = {} + + # List of emphasis-like delimiters for current tag + self.delimiters: list[Delimiter] = [] + + # Stack of delimiter lists for upper level tags + self._prev_delimiters: list[list[Delimiter]] = [] + + # backticklength => last seen position + self.backticks: dict[int, int] = {} + self.backticksScanned = False + + # Counter used to disable inline linkify-it execution + # inside and markdown links + self.linkLevel = 0 + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}" + f"(pos=[{self.pos} of {self.posMax}], token={len(self.tokens)})" + ) + + def pushPending(self) -> Token: + token = Token("text", "", 0) + token.content = self.pending + token.level = self.pendingLevel + self.tokens.append(token) + self.pending = "" + return token + + def push(self, ttype: str, tag: str, nesting: Literal[-1, 0, 1]) -> Token: + """Push new token to "stream". + If pending text exists - flush it as text token + """ + if self.pending: + self.pushPending() + + token = Token(ttype, tag, nesting) + token_meta = None + + if nesting < 0: + # closing tag + self.level -= 1 + self.delimiters = self._prev_delimiters.pop() + + token.level = self.level + + if nesting > 0: + # opening tag + self.level += 1 + self._prev_delimiters.append(self.delimiters) + self.delimiters = [] + token_meta = {"delimiters": self.delimiters} + + self.pendingLevel = self.level + self.tokens.append(token) + self.tokens_meta.append(token_meta) + return token + + def scanDelims(self, start: int, canSplitWord: bool) -> Scanned: + """ + Scan a sequence of emphasis-like markers, and determine whether + it can start an emphasis sequence or end an emphasis sequence. + + - start - position to scan from (it should point at a valid marker); + - canSplitWord - determine if these markers can be found inside a word + + """ + pos = start + maximum = self.posMax + marker = self.src[start] + + # treat beginning of the line as a whitespace + lastChar = self.src[start - 1] if start > 0 else " " + + while pos < maximum and self.src[pos] == marker: + pos += 1 + + count = pos - start + + # treat end of the line as a whitespace + nextChar = self.src[pos] if pos < maximum else " " + + isLastPunctChar = isMdAsciiPunct(ord(lastChar)) or isPunctChar(lastChar) + isNextPunctChar = isMdAsciiPunct(ord(nextChar)) or isPunctChar(nextChar) + + isLastWhiteSpace = isWhiteSpace(ord(lastChar)) + isNextWhiteSpace = isWhiteSpace(ord(nextChar)) + + left_flanking = not ( + isNextWhiteSpace + or (isNextPunctChar and not (isLastWhiteSpace or isLastPunctChar)) + ) + right_flanking = not ( + isLastWhiteSpace + or (isLastPunctChar and not (isNextWhiteSpace or isNextPunctChar)) + ) + + can_open = left_flanking and ( + canSplitWord or (not right_flanking) or isLastPunctChar + ) + can_close = right_flanking and ( + canSplitWord or (not left_flanking) or isNextPunctChar + ) + + return Scanned(can_open, can_close, count) diff --git a/lib/markdown_it/rules_inline/strikethrough.py b/lib/markdown_it/rules_inline/strikethrough.py new file mode 100644 index 0000000..ec81628 --- /dev/null +++ b/lib/markdown_it/rules_inline/strikethrough.py @@ -0,0 +1,127 @@ +# ~~strike through~~ +from __future__ import annotations + +from .state_inline import Delimiter, StateInline + + +def tokenize(state: StateInline, silent: bool) -> bool: + """Insert each marker as a separate text token, and add it to delimiter list""" + start = state.pos + ch = state.src[start] + + if silent: + return False + + if ch != "~": + return False + + scanned = state.scanDelims(state.pos, True) + length = scanned.length + + if length < 2: + return False + + if length % 2: + token = state.push("text", "", 0) + token.content = ch + length -= 1 + + i = 0 + while i < length: + token = state.push("text", "", 0) + token.content = ch + ch + state.delimiters.append( + Delimiter( + marker=ord(ch), + length=0, # disable "rule of 3" length checks meant for emphasis + token=len(state.tokens) - 1, + end=-1, + open=scanned.can_open, + close=scanned.can_close, + ) + ) + + i += 2 + + state.pos += scanned.length + + return True + + +def _postProcess(state: StateInline, delimiters: list[Delimiter]) -> None: + loneMarkers = [] + maximum = len(delimiters) + + i = 0 + while i < maximum: + startDelim = delimiters[i] + + if startDelim.marker != 0x7E: # /* ~ */ + i += 1 + continue + + if startDelim.end == -1: + i += 1 + continue + + endDelim = delimiters[startDelim.end] + + token = state.tokens[startDelim.token] + token.type = "s_open" + token.tag = "s" + token.nesting = 1 + token.markup = "~~" + token.content = "" + + token = state.tokens[endDelim.token] + token.type = "s_close" + token.tag = "s" + token.nesting = -1 + token.markup = "~~" + token.content = "" + + if ( + state.tokens[endDelim.token - 1].type == "text" + and state.tokens[endDelim.token - 1].content == "~" + ): + loneMarkers.append(endDelim.token - 1) + + i += 1 + + # If a marker sequence has an odd number of characters, it's split + # like this: `~~~~~` -> `~` + `~~` + `~~`, leaving one marker at the + # start of the sequence. + # + # So, we have to move all those markers after subsequent s_close tags. + # + while loneMarkers: + i = loneMarkers.pop() + j = i + 1 + + while (j < len(state.tokens)) and (state.tokens[j].type == "s_close"): + j += 1 + + j -= 1 + + if i != j: + token = state.tokens[j] + state.tokens[j] = state.tokens[i] + state.tokens[i] = token + + +def postProcess(state: StateInline) -> None: + """Walk through delimiter list and replace text tokens with tags.""" + tokens_meta = state.tokens_meta + maximum = len(state.tokens_meta) + _postProcess(state, state.delimiters) + + curr = 0 + while curr < maximum: + try: + curr_meta = tokens_meta[curr] + except IndexError: + pass + else: + if curr_meta and "delimiters" in curr_meta: + _postProcess(state, curr_meta["delimiters"]) + curr += 1 diff --git a/lib/markdown_it/rules_inline/text.py b/lib/markdown_it/rules_inline/text.py new file mode 100644 index 0000000..18b2fcc --- /dev/null +++ b/lib/markdown_it/rules_inline/text.py @@ -0,0 +1,62 @@ +import functools +import re + +# Skip text characters for text token, place those to pending buffer +# and increment current pos +from .state_inline import StateInline + +# Rule to skip pure text +# '{}$%@~+=:' reserved for extensions + +# !!!! Don't confuse with "Markdown ASCII Punctuation" chars +# http://spec.commonmark.org/0.15/#ascii-punctuation-character + + +_TerminatorChars = { + "\n", + "!", + "#", + "$", + "%", + "&", + "*", + "+", + "-", + ":", + "<", + "=", + ">", + "@", + "[", + "\\", + "]", + "^", + "_", + "`", + "{", + "}", + "~", +} + + +@functools.cache +def _terminator_char_regex() -> re.Pattern[str]: + return re.compile("[" + re.escape("".join(_TerminatorChars)) + "]") + + +def text(state: StateInline, silent: bool) -> bool: + pos = state.pos + posMax = state.posMax + + terminator_char = _terminator_char_regex().search(state.src, pos) + pos = terminator_char.start() if terminator_char else posMax + + if pos == state.pos: + return False + + if not silent: + state.pending += state.src[state.pos : pos] + + state.pos = pos + + return True diff --git a/lib/markdown_it/token.py b/lib/markdown_it/token.py new file mode 100644 index 0000000..d6d0b45 --- /dev/null +++ b/lib/markdown_it/token.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from collections.abc import Callable, MutableMapping +import dataclasses as dc +from typing import Any, Literal +import warnings + + +def convert_attrs(value: Any) -> Any: + """Convert Token.attrs set as ``None`` or ``[[key, value], ...]`` to a dict. + + This improves compatibility with upstream markdown-it. + """ + if not value: + return {} + if isinstance(value, list): + return dict(value) + return value + + +@dc.dataclass(slots=True) +class Token: + type: str + """Type of the token (string, e.g. "paragraph_open")""" + + tag: str + """HTML tag name, e.g. 'p'""" + + nesting: Literal[-1, 0, 1] + """Level change (number in {-1, 0, 1} set), where: + - `1` means the tag is opening + - `0` means the tag is self-closing + - `-1` means the tag is closing + """ + + attrs: dict[str, str | int | float] = dc.field(default_factory=dict) + """HTML attributes. + Note this differs from the upstream "list of lists" format, + although than an instance can still be initialised with this format. + """ + + map: list[int] | None = None + """Source map info. Format: `[ line_begin, line_end ]`""" + + level: int = 0 + """Nesting level, the same as `state.level`""" + + children: list[Token] | None = None + """Array of child nodes (inline and img tokens).""" + + content: str = "" + """Inner content, in the case of a self-closing tag (code, html, fence, etc.),""" + + markup: str = "" + """'*' or '_' for emphasis, fence string for fence, etc.""" + + info: str = "" + """Additional information: + - Info string for "fence" tokens + - The value "auto" for autolink "link_open" and "link_close" tokens + - The string value of the item marker for ordered-list "list_item_open" tokens + """ + + meta: dict[Any, Any] = dc.field(default_factory=dict) + """A place for plugins to store any arbitrary data""" + + block: bool = False + """True for block-level tokens, false for inline tokens. + Used in renderer to calculate line breaks + """ + + hidden: bool = False + """If true, ignore this element when rendering. + Used for tight lists to hide paragraphs. + """ + + def __post_init__(self) -> None: + self.attrs = convert_attrs(self.attrs) + + def attrIndex(self, name: str) -> int: + warnings.warn( # noqa: B028 + "Token.attrIndex should not be used, since Token.attrs is a dictionary", + UserWarning, + ) + if name not in self.attrs: + return -1 + return list(self.attrs.keys()).index(name) + + def attrItems(self) -> list[tuple[str, str | int | float]]: + """Get (key, value) list of attrs.""" + return list(self.attrs.items()) + + def attrPush(self, attrData: tuple[str, str | int | float]) -> None: + """Add `[ name, value ]` attribute to list. Init attrs if necessary.""" + name, value = attrData + self.attrSet(name, value) + + def attrSet(self, name: str, value: str | int | float) -> None: + """Set `name` attribute to `value`. Override old value if exists.""" + self.attrs[name] = value + + def attrGet(self, name: str) -> None | str | int | float: + """Get the value of attribute `name`, or null if it does not exist.""" + return self.attrs.get(name, None) + + def attrJoin(self, name: str, value: str) -> None: + """Join value to existing attribute via space. + Or create new attribute if not exists. + Useful to operate with token classes. + """ + if name in self.attrs: + current = self.attrs[name] + if not isinstance(current, str): + raise TypeError( + f"existing attr 'name' is not a str: {self.attrs[name]}" + ) + self.attrs[name] = f"{current} {value}" + else: + self.attrs[name] = value + + def copy(self, **changes: Any) -> Token: + """Return a shallow copy of the instance.""" + return dc.replace(self, **changes) + + def as_dict( + self, + *, + children: bool = True, + as_upstream: bool = True, + meta_serializer: Callable[[dict[Any, Any]], Any] | None = None, + filter: Callable[[str, Any], bool] | None = None, + dict_factory: Callable[..., MutableMapping[str, Any]] = dict, + ) -> MutableMapping[str, Any]: + """Return the token as a dictionary. + + :param children: Also convert children to dicts + :param as_upstream: Ensure the output dictionary is equal to that created by markdown-it + For example, attrs are converted to null or lists + :param meta_serializer: hook for serializing ``Token.meta`` + :param filter: A callable whose return code determines whether an + attribute or element is included (``True``) or dropped (``False``). + Is called with the (key, value) pair. + :param dict_factory: A callable to produce dictionaries from. + For example, to produce ordered dictionaries instead of normal Python + dictionaries, pass in ``collections.OrderedDict``. + + """ + mapping = dict_factory((f.name, getattr(self, f.name)) for f in dc.fields(self)) + if filter: + mapping = dict_factory((k, v) for k, v in mapping.items() if filter(k, v)) + if as_upstream and "attrs" in mapping: + mapping["attrs"] = ( + None + if not mapping["attrs"] + else [[k, v] for k, v in mapping["attrs"].items()] + ) + if meta_serializer and "meta" in mapping: + mapping["meta"] = meta_serializer(mapping["meta"]) + if children and mapping.get("children", None): + mapping["children"] = [ + child.as_dict( + children=children, + filter=filter, + dict_factory=dict_factory, + as_upstream=as_upstream, + meta_serializer=meta_serializer, + ) + for child in mapping["children"] + ] + return mapping + + @classmethod + def from_dict(cls, dct: MutableMapping[str, Any]) -> Token: + """Convert a dict to a Token.""" + token = cls(**dct) + if token.children: + token.children = [cls.from_dict(c) for c in token.children] # type: ignore[arg-type] + return token diff --git a/lib/markdown_it/tree.py b/lib/markdown_it/tree.py new file mode 100644 index 0000000..5369157 --- /dev/null +++ b/lib/markdown_it/tree.py @@ -0,0 +1,333 @@ +"""A tree representation of a linear markdown-it token stream. + +This module is not part of upstream JavaScript markdown-it. +""" + +from __future__ import annotations + +from collections.abc import Generator, Sequence +import textwrap +from typing import Any, NamedTuple, TypeVar, overload + +from .token import Token + + +class _NesterTokens(NamedTuple): + opening: Token + closing: Token + + +_NodeType = TypeVar("_NodeType", bound="SyntaxTreeNode") + + +class SyntaxTreeNode: + """A Markdown syntax tree node. + + A class that can be used to construct a tree representation of a linear + `markdown-it-py` token stream. + + Each node in the tree represents either: + - root of the Markdown document + - a single unnested `Token` + - a `Token` "_open" and "_close" token pair, and the tokens nested in + between + """ + + def __init__( + self, tokens: Sequence[Token] = (), *, create_root: bool = True + ) -> None: + """Initialize a `SyntaxTreeNode` from a token stream. + + If `create_root` is True, create a root node for the document. + """ + # Only nodes representing an unnested token have self.token + self.token: Token | None = None + + # Only containers have nester tokens + self.nester_tokens: _NesterTokens | None = None + + # Root node does not have self.parent + self._parent: Any = None + + # Empty list unless a non-empty container, or unnested token that has + # children (i.e. inline or img) + self._children: list[Any] = [] + + if create_root: + self._set_children_from_tokens(tokens) + return + + if not tokens: + raise ValueError( + "Can only create root from empty token sequence." + " Set `create_root=True`." + ) + elif len(tokens) == 1: + inline_token = tokens[0] + if inline_token.nesting: + raise ValueError( + "Unequal nesting level at the start and end of token stream." + ) + self.token = inline_token + if inline_token.children: + self._set_children_from_tokens(inline_token.children) + else: + self.nester_tokens = _NesterTokens(tokens[0], tokens[-1]) + self._set_children_from_tokens(tokens[1:-1]) + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.type})" + + @overload + def __getitem__(self: _NodeType, item: int) -> _NodeType: ... + + @overload + def __getitem__(self: _NodeType, item: slice) -> list[_NodeType]: ... + + def __getitem__(self: _NodeType, item: int | slice) -> _NodeType | list[_NodeType]: + return self.children[item] + + def to_tokens(self: _NodeType) -> list[Token]: + """Recover the linear token stream.""" + + def recursive_collect_tokens(node: _NodeType, token_list: list[Token]) -> None: + if node.type == "root": + for child in node.children: + recursive_collect_tokens(child, token_list) + elif node.token: + token_list.append(node.token) + else: + assert node.nester_tokens + token_list.append(node.nester_tokens.opening) + for child in node.children: + recursive_collect_tokens(child, token_list) + token_list.append(node.nester_tokens.closing) + + tokens: list[Token] = [] + recursive_collect_tokens(self, tokens) + return tokens + + @property + def children(self: _NodeType) -> list[_NodeType]: + return self._children + + @children.setter + def children(self: _NodeType, value: list[_NodeType]) -> None: + self._children = value + + @property + def parent(self: _NodeType) -> _NodeType | None: + return self._parent # type: ignore + + @parent.setter + def parent(self: _NodeType, value: _NodeType | None) -> None: + self._parent = value + + @property + def is_root(self) -> bool: + """Is the node a special root node?""" + return not (self.token or self.nester_tokens) + + @property + def is_nested(self) -> bool: + """Is this node nested?. + + Returns `True` if the node represents a `Token` pair and tokens in the + sequence between them, where `Token.nesting` of the first `Token` in + the pair is 1 and nesting of the other `Token` is -1. + """ + return bool(self.nester_tokens) + + @property + def siblings(self: _NodeType) -> Sequence[_NodeType]: + """Get siblings of the node. + + Gets the whole group of siblings, including self. + """ + if not self.parent: + return [self] + return self.parent.children + + @property + def type(self) -> str: + """Get a string type of the represented syntax. + + - "root" for root nodes + - `Token.type` if the node represents an unnested token + - `Token.type` of the opening token, with "_open" suffix stripped, if + the node represents a nester token pair + """ + if self.is_root: + return "root" + if self.token: + return self.token.type + assert self.nester_tokens + return self.nester_tokens.opening.type.removesuffix("_open") + + @property + def next_sibling(self: _NodeType) -> _NodeType | None: + """Get the next node in the sequence of siblings. + + Returns `None` if this is the last sibling. + """ + self_index = self.siblings.index(self) + if self_index + 1 < len(self.siblings): + return self.siblings[self_index + 1] + return None + + @property + def previous_sibling(self: _NodeType) -> _NodeType | None: + """Get the previous node in the sequence of siblings. + + Returns `None` if this is the first sibling. + """ + self_index = self.siblings.index(self) + if self_index - 1 >= 0: + return self.siblings[self_index - 1] + return None + + def _add_child( + self, + tokens: Sequence[Token], + ) -> None: + """Make a child node for `self`.""" + child = type(self)(tokens, create_root=False) + child.parent = self + self.children.append(child) + + def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None: + """Convert the token stream to a tree structure and set the resulting + nodes as children of `self`.""" + reversed_tokens = list(reversed(tokens)) + while reversed_tokens: + token = reversed_tokens.pop() + + if not token.nesting: + self._add_child([token]) + continue + if token.nesting != 1: + raise ValueError("Invalid token nesting") + + nested_tokens = [token] + nesting = 1 + while reversed_tokens and nesting: + token = reversed_tokens.pop() + nested_tokens.append(token) + nesting += token.nesting + if nesting: + raise ValueError(f"unclosed tokens starting {nested_tokens[0]}") + + self._add_child(nested_tokens) + + def pretty( + self, *, indent: int = 2, show_text: bool = False, _current: int = 0 + ) -> str: + """Create an XML style string of the tree.""" + prefix = " " * _current + text = prefix + f"<{self.type}" + if not self.is_root and self.attrs: + text += " " + " ".join(f"{k}={v!r}" for k, v in self.attrs.items()) + text += ">" + if ( + show_text + and not self.is_root + and self.type in ("text", "text_special") + and self.content + ): + text += "\n" + textwrap.indent(self.content, prefix + " " * indent) + for child in self.children: + text += "\n" + child.pretty( + indent=indent, show_text=show_text, _current=_current + indent + ) + return text + + def walk( + self: _NodeType, *, include_self: bool = True + ) -> Generator[_NodeType, None, None]: + """Recursively yield all descendant nodes in the tree starting at self. + + The order mimics the order of the underlying linear token + stream (i.e. depth first). + """ + if include_self: + yield self + for child in self.children: + yield from child.walk(include_self=True) + + # NOTE: + # The values of the properties defined below directly map to properties + # of the underlying `Token`s. A root node does not translate to a `Token` + # object, so calling these property getters on a root node will raise an + # `AttributeError`. + # + # There is no mapping for `Token.nesting` because the `is_nested` property + # provides that data, and can be called on any node type, including root. + + def _attribute_token(self) -> Token: + """Return the `Token` that is used as the data source for the + properties defined below.""" + if self.token: + return self.token + if self.nester_tokens: + return self.nester_tokens.opening + raise AttributeError("Root node does not have the accessed attribute") + + @property + def tag(self) -> str: + """html tag name, e.g. \"p\" """ + return self._attribute_token().tag + + @property + def attrs(self) -> dict[str, str | int | float]: + """Html attributes.""" + return self._attribute_token().attrs + + def attrGet(self, name: str) -> None | str | int | float: + """Get the value of attribute `name`, or null if it does not exist.""" + return self._attribute_token().attrGet(name) + + @property + def map(self) -> tuple[int, int] | None: + """Source map info. Format: `tuple[ line_begin, line_end ]`""" + map_ = self._attribute_token().map + if map_: + # Type ignore because `Token`s attribute types are not perfect + return tuple(map_) # type: ignore + return None + + @property + def level(self) -> int: + """nesting level, the same as `state.level`""" + return self._attribute_token().level + + @property + def content(self) -> str: + """In a case of self-closing tag (code, html, fence, etc.), it + has contents of this tag.""" + return self._attribute_token().content + + @property + def markup(self) -> str: + """'*' or '_' for emphasis, fence string for fence, etc.""" + return self._attribute_token().markup + + @property + def info(self) -> str: + """fence infostring""" + return self._attribute_token().info + + @property + def meta(self) -> dict[Any, Any]: + """A place for plugins to store an arbitrary data.""" + return self._attribute_token().meta + + @property + def block(self) -> bool: + """True for block-level tokens, false for inline tokens.""" + return self._attribute_token().block + + @property + def hidden(self) -> bool: + """If it's true, ignore this element when rendering. + Used for tight lists to hide paragraphs.""" + return self._attribute_token().hidden diff --git a/lib/markdown_it/utils.py b/lib/markdown_it/utils.py new file mode 100644 index 0000000..2571a15 --- /dev/null +++ b/lib/markdown_it/utils.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from collections.abc import Callable, Iterable, MutableMapping +from collections.abc import MutableMapping as MutableMappingABC +from pathlib import Path +from typing import TYPE_CHECKING, Any, TypedDict, cast + +if TYPE_CHECKING: + from typing_extensions import NotRequired + + +EnvType = MutableMapping[str, Any] # note: could use TypeAlias in python 3.10 +"""Type for the environment sandbox used in parsing and rendering, +which stores mutable variables for use by plugins and rules. +""" + + +class OptionsType(TypedDict): + """Options for parsing.""" + + maxNesting: int + """Internal protection, recursion limit.""" + html: bool + """Enable HTML tags in source.""" + linkify: bool + """Enable autoconversion of URL-like texts to links.""" + typographer: bool + """Enable smartquotes and replacements.""" + quotes: str + """Quote characters.""" + xhtmlOut: bool + """Use '/' to close single tags (
).""" + breaks: bool + """Convert newlines in paragraphs into
.""" + langPrefix: str + """CSS language prefix for fenced blocks.""" + highlight: Callable[[str, str, str], str] | None + """Highlighter function: (content, lang, attrs) -> str.""" + store_labels: NotRequired[bool] + """Store link label in link/image token's metadata (under Token.meta['label']). + + This is a Python only option, and is intended for the use of round-trip parsing. + """ + + +class PresetType(TypedDict): + """Preset configuration for markdown-it.""" + + options: OptionsType + """Options for parsing.""" + components: MutableMapping[str, MutableMapping[str, list[str]]] + """Components for parsing and rendering.""" + + +class OptionsDict(MutableMappingABC): # type: ignore + """A dictionary, with attribute access to core markdownit configuration options.""" + + # Note: ideally we would probably just remove attribute access entirely, + # but we keep it for backwards compatibility. + + def __init__(self, options: OptionsType) -> None: + self._options = cast(OptionsType, dict(options)) + + def __getitem__(self, key: str) -> Any: + return self._options[key] # type: ignore[literal-required] + + def __setitem__(self, key: str, value: Any) -> None: + self._options[key] = value # type: ignore[literal-required] + + def __delitem__(self, key: str) -> None: + del self._options[key] # type: ignore + + def __iter__(self) -> Iterable[str]: # type: ignore + return iter(self._options) + + def __len__(self) -> int: + return len(self._options) + + def __repr__(self) -> str: + return repr(self._options) + + def __str__(self) -> str: + return str(self._options) + + @property + def maxNesting(self) -> int: + """Internal protection, recursion limit.""" + return self._options["maxNesting"] + + @maxNesting.setter + def maxNesting(self, value: int) -> None: + self._options["maxNesting"] = value + + @property + def html(self) -> bool: + """Enable HTML tags in source.""" + return self._options["html"] + + @html.setter + def html(self, value: bool) -> None: + self._options["html"] = value + + @property + def linkify(self) -> bool: + """Enable autoconversion of URL-like texts to links.""" + return self._options["linkify"] + + @linkify.setter + def linkify(self, value: bool) -> None: + self._options["linkify"] = value + + @property + def typographer(self) -> bool: + """Enable smartquotes and replacements.""" + return self._options["typographer"] + + @typographer.setter + def typographer(self, value: bool) -> None: + self._options["typographer"] = value + + @property + def quotes(self) -> str: + """Quote characters.""" + return self._options["quotes"] + + @quotes.setter + def quotes(self, value: str) -> None: + self._options["quotes"] = value + + @property + def xhtmlOut(self) -> bool: + """Use '/' to close single tags (
).""" + return self._options["xhtmlOut"] + + @xhtmlOut.setter + def xhtmlOut(self, value: bool) -> None: + self._options["xhtmlOut"] = value + + @property + def breaks(self) -> bool: + """Convert newlines in paragraphs into
.""" + return self._options["breaks"] + + @breaks.setter + def breaks(self, value: bool) -> None: + self._options["breaks"] = value + + @property + def langPrefix(self) -> str: + """CSS language prefix for fenced blocks.""" + return self._options["langPrefix"] + + @langPrefix.setter + def langPrefix(self, value: str) -> None: + self._options["langPrefix"] = value + + @property + def highlight(self) -> Callable[[str, str, str], str] | None: + """Highlighter function: (content, langName, langAttrs) -> escaped HTML.""" + return self._options["highlight"] + + @highlight.setter + def highlight(self, value: Callable[[str, str, str], str] | None) -> None: + self._options["highlight"] = value + + +def read_fixture_file(path: str | Path) -> list[list[Any]]: + text = Path(path).read_text(encoding="utf-8") + tests = [] + section = 0 + last_pos = 0 + lines = text.splitlines(keepends=True) + for i in range(len(lines)): + if lines[i].rstrip() == ".": + if section == 0: + tests.append([i, lines[i - 1].strip()]) + section = 1 + elif section == 1: + tests[-1].append("".join(lines[last_pos + 1 : i])) + section = 2 + elif section == 2: + tests[-1].append("".join(lines[last_pos + 1 : i])) + section = 0 + + last_pos = i + return tests diff --git a/lib/markdown_it_py-4.0.0.dist-info/INSTALLER b/lib/markdown_it_py-4.0.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/markdown_it_py-4.0.0.dist-info/METADATA b/lib/markdown_it_py-4.0.0.dist-info/METADATA new file mode 100644 index 0000000..0f2b466 --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/METADATA @@ -0,0 +1,219 @@ +Metadata-Version: 2.4 +Name: markdown-it-py +Version: 4.0.0 +Summary: Python port of markdown-it. Markdown parsing, done right! +Keywords: markdown,lexer,parser,commonmark,markdown-it +Author-email: Chris Sewell +Requires-Python: >=3.10 +Description-Content-Type: text/markdown +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup +License-File: LICENSE +License-File: LICENSE.markdown-it +Requires-Dist: mdurl~=0.1 +Requires-Dist: psutil ; extra == "benchmarking" +Requires-Dist: pytest ; extra == "benchmarking" +Requires-Dist: pytest-benchmark ; extra == "benchmarking" +Requires-Dist: commonmark~=0.9 ; extra == "compare" +Requires-Dist: markdown~=3.4 ; extra == "compare" +Requires-Dist: mistletoe~=1.0 ; extra == "compare" +Requires-Dist: mistune~=3.0 ; extra == "compare" +Requires-Dist: panflute~=2.3 ; extra == "compare" +Requires-Dist: markdown-it-pyrs ; extra == "compare" +Requires-Dist: linkify-it-py>=1,<3 ; extra == "linkify" +Requires-Dist: mdit-py-plugins>=0.5.0 ; extra == "plugins" +Requires-Dist: gprof2dot ; extra == "profiling" +Requires-Dist: mdit-py-plugins>=0.5.0 ; extra == "rtd" +Requires-Dist: myst-parser ; extra == "rtd" +Requires-Dist: pyyaml ; extra == "rtd" +Requires-Dist: sphinx ; extra == "rtd" +Requires-Dist: sphinx-copybutton ; extra == "rtd" +Requires-Dist: sphinx-design ; extra == "rtd" +Requires-Dist: sphinx-book-theme~=1.0 ; extra == "rtd" +Requires-Dist: jupyter_sphinx ; extra == "rtd" +Requires-Dist: ipykernel ; extra == "rtd" +Requires-Dist: coverage ; extra == "testing" +Requires-Dist: pytest ; extra == "testing" +Requires-Dist: pytest-cov ; extra == "testing" +Requires-Dist: pytest-regressions ; extra == "testing" +Requires-Dist: requests ; extra == "testing" +Project-URL: Documentation, https://markdown-it-py.readthedocs.io +Project-URL: Homepage, https://github.com/executablebooks/markdown-it-py +Provides-Extra: benchmarking +Provides-Extra: compare +Provides-Extra: linkify +Provides-Extra: plugins +Provides-Extra: profiling +Provides-Extra: rtd +Provides-Extra: testing + +# markdown-it-py + +[![Github-CI][github-ci]][github-link] +[![Coverage Status][codecov-badge]][codecov-link] +[![PyPI][pypi-badge]][pypi-link] +[![Conda][conda-badge]][conda-link] +[![PyPI - Downloads][install-badge]][install-link] + +

+ markdown-it-py icon +

+ +> Markdown parser done right. + +- Follows the __[CommonMark spec](http://spec.commonmark.org/)__ for baseline parsing +- Configurable syntax: you can add new rules and even replace existing ones. +- Pluggable: Adds syntax extensions to extend the parser (see the [plugin list][md-plugins]). +- High speed (see our [benchmarking tests][md-performance]) +- Easy to configure for [security][md-security] +- Member of [Google's Assured Open Source Software](https://cloud.google.com/assured-open-source-software/docs/supported-packages) + +This is a Python port of [markdown-it], and some of its associated plugins. +For more details see: . + +For details on [markdown-it] itself, see: + +- The __[Live demo](https://markdown-it.github.io)__ +- [The markdown-it README][markdown-it-readme] + +**See also:** [markdown-it-pyrs](https://github.com/chrisjsewell/markdown-it-pyrs) for an experimental Rust binding, +for even more speed! + +## Installation + +### PIP + +```bash +pip install markdown-it-py[plugins] +``` + +or with extras + +```bash +pip install markdown-it-py[linkify,plugins] +``` + +### Conda + +```bash +conda install -c conda-forge markdown-it-py +``` + +or with extras + +```bash +conda install -c conda-forge markdown-it-py linkify-it-py mdit-py-plugins +``` + +## Usage + +### Python API Usage + +Render markdown to HTML with markdown-it-py and a custom configuration +with and without plugins and features: + +```python +from markdown_it import MarkdownIt +from mdit_py_plugins.front_matter import front_matter_plugin +from mdit_py_plugins.footnote import footnote_plugin + +md = ( + MarkdownIt('commonmark', {'breaks':True,'html':True}) + .use(front_matter_plugin) + .use(footnote_plugin) + .enable('table') +) +text = (""" +--- +a: 1 +--- + +a | b +- | - +1 | 2 + +A footnote [^1] + +[^1]: some details +""") +tokens = md.parse(text) +html_text = md.render(text) + +## To export the html to a file, uncomment the lines below: +# from pathlib import Path +# Path("output.html").write_text(html_text) +``` + +### Command-line Usage + +Render markdown to HTML with markdown-it-py from the +command-line: + +```console +usage: markdown-it [-h] [-v] [filenames [filenames ...]] + +Parse one or more markdown files, convert each to HTML, and print to stdout + +positional arguments: + filenames specify an optional list of files to convert + +optional arguments: + -h, --help show this help message and exit + -v, --version show program's version number and exit + +Interactive: + + $ markdown-it + markdown-it-py [version 0.0.0] (interactive) + Type Ctrl-D to complete input, or Ctrl-C to exit. + >>> # Example + ... > markdown *input* + ... +

Example

+
+

markdown input

+
+ +Batch: + + $ markdown-it README.md README.footer.md > index.html + +``` + +## References / Thanks + +Big thanks to the authors of [markdown-it]: + +- Alex Kocharin [github/rlidwka](https://github.com/rlidwka) +- Vitaly Puzrin [github/puzrin](https://github.com/puzrin) + +Also [John MacFarlane](https://github.com/jgm) for his work on the CommonMark spec and reference implementations. + +[github-ci]: https://github.com/executablebooks/markdown-it-py/actions/workflows/tests.yml/badge.svg?branch=master +[github-link]: https://github.com/executablebooks/markdown-it-py +[pypi-badge]: https://img.shields.io/pypi/v/markdown-it-py.svg +[pypi-link]: https://pypi.org/project/markdown-it-py +[conda-badge]: https://anaconda.org/conda-forge/markdown-it-py/badges/version.svg +[conda-link]: https://anaconda.org/conda-forge/markdown-it-py +[codecov-badge]: https://codecov.io/gh/executablebooks/markdown-it-py/branch/master/graph/badge.svg +[codecov-link]: https://codecov.io/gh/executablebooks/markdown-it-py +[install-badge]: https://img.shields.io/pypi/dw/markdown-it-py?label=pypi%20installs +[install-link]: https://pypistats.org/packages/markdown-it-py + +[CommonMark spec]: http://spec.commonmark.org/ +[markdown-it]: https://github.com/markdown-it/markdown-it +[markdown-it-readme]: https://github.com/markdown-it/markdown-it/blob/master/README.md +[md-security]: https://markdown-it-py.readthedocs.io/en/latest/security.html +[md-performance]: https://markdown-it-py.readthedocs.io/en/latest/performance.html +[md-plugins]: https://markdown-it-py.readthedocs.io/en/latest/plugins.html + diff --git a/lib/markdown_it_py-4.0.0.dist-info/RECORD b/lib/markdown_it_py-4.0.0.dist-info/RECORD new file mode 100644 index 0000000..55a1948 --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/RECORD @@ -0,0 +1,142 @@ +../../bin/markdown-it,sha256=W3AzqoMVc-K93MTDhze1VTLE-VkAK11bh8a7c3bE6EI,192 +markdown_it/__init__.py,sha256=R7fMvDxageYJ4Q6doBcimogy1ctcV1eBuCFu5Pr8bbA,114 +markdown_it/__pycache__/__init__.cpython-314.pyc,, +markdown_it/__pycache__/_compat.cpython-314.pyc,, +markdown_it/__pycache__/_punycode.cpython-314.pyc,, +markdown_it/__pycache__/main.cpython-314.pyc,, +markdown_it/__pycache__/parser_block.cpython-314.pyc,, +markdown_it/__pycache__/parser_core.cpython-314.pyc,, +markdown_it/__pycache__/parser_inline.cpython-314.pyc,, +markdown_it/__pycache__/renderer.cpython-314.pyc,, +markdown_it/__pycache__/ruler.cpython-314.pyc,, +markdown_it/__pycache__/token.cpython-314.pyc,, +markdown_it/__pycache__/tree.cpython-314.pyc,, +markdown_it/__pycache__/utils.cpython-314.pyc,, +markdown_it/_compat.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35 +markdown_it/_punycode.py,sha256=JvSOZJ4VKr58z7unFGM0KhfTxqHMk2w8gglxae2QszM,2373 +markdown_it/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +markdown_it/cli/__pycache__/__init__.cpython-314.pyc,, +markdown_it/cli/__pycache__/parse.cpython-314.pyc,, +markdown_it/cli/parse.py,sha256=Un3N7fyGHhZAQouGVnRx-WZcpKwEK2OF08rzVAEBie8,2881 +markdown_it/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +markdown_it/common/__pycache__/__init__.cpython-314.pyc,, +markdown_it/common/__pycache__/entities.cpython-314.pyc,, +markdown_it/common/__pycache__/html_blocks.cpython-314.pyc,, +markdown_it/common/__pycache__/html_re.cpython-314.pyc,, +markdown_it/common/__pycache__/normalize_url.cpython-314.pyc,, +markdown_it/common/__pycache__/utils.cpython-314.pyc,, +markdown_it/common/entities.py,sha256=EYRCmUL7ZU1FRGLSXQlPx356lY8EUBdFyx96eSGc6d0,157 +markdown_it/common/html_blocks.py,sha256=QXbUDMoN9lXLgYFk2DBYllnLiFukL6dHn2X98Y6Wews,986 +markdown_it/common/html_re.py,sha256=FggAEv9IL8gHQqsGTkHcf333rTojwG0DQJMH9oVu0fU,926 +markdown_it/common/normalize_url.py,sha256=avOXnLd9xw5jU1q5PLftjAM9pvGx8l9QDEkmZSyrMgg,2568 +markdown_it/common/utils.py,sha256=pMgvMOE3ZW-BdJ7HfuzlXNKyD1Ivk7jHErc2J_B8J5M,8734 +markdown_it/helpers/__init__.py,sha256=YH2z7dS0WUc_9l51MWPvrLtFoBPh4JLGw58OuhGRCK0,253 +markdown_it/helpers/__pycache__/__init__.cpython-314.pyc,, +markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc,, +markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc,, +markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc,, +markdown_it/helpers/parse_link_destination.py,sha256=u-xxWVP3g1s7C1bQuQItiYyDrYoYHJzXaZXPgr-o6mY,1906 +markdown_it/helpers/parse_link_label.py,sha256=PIHG6ZMm3BUw0a2m17lCGqNrl3vaz911tuoGviWD3I4,1037 +markdown_it/helpers/parse_link_title.py,sha256=jkLoYQMKNeX9bvWQHkaSroiEo27HylkEUNmj8xBRlp4,2273 +markdown_it/main.py,sha256=vzuT23LJyKrPKNyHKKAbOHkNWpwIldOGUM-IGsv2DHM,12732 +markdown_it/parser_block.py,sha256=-MyugXB63Te71s4NcSQZiK5bE6BHkdFyZv_bviuatdI,3939 +markdown_it/parser_core.py,sha256=SRmJjqe8dC6GWzEARpWba59cBmxjCr3Gsg8h29O8sQk,1016 +markdown_it/parser_inline.py,sha256=y0jCig8CJxQO7hBz0ZY3sGvPlAKTohOwIgaqnlSaS5A,5024 +markdown_it/port.yaml,sha256=jt_rdwOnfocOV5nc35revTybAAQMIp_-1fla_527sVE,2447 +markdown_it/presets/__init__.py,sha256=22vFtwJEY7iqFRtgVZ-pJthcetfpr1Oig8XOF9x1328,970 +markdown_it/presets/__pycache__/__init__.cpython-314.pyc,, +markdown_it/presets/__pycache__/commonmark.cpython-314.pyc,, +markdown_it/presets/__pycache__/default.cpython-314.pyc,, +markdown_it/presets/__pycache__/zero.cpython-314.pyc,, +markdown_it/presets/commonmark.py,sha256=ygfb0R7WQ_ZoyQP3df-B0EnYMqNXCVOSw9SAdMjsGow,2869 +markdown_it/presets/default.py,sha256=FfKVUI0HH3M-_qy6RwotLStdC4PAaAxE7Dq0_KQtRtc,1811 +markdown_it/presets/zero.py,sha256=okXWTBEI-2nmwx5XKeCjxInRf65oC11gahtRl-QNtHM,2113 +markdown_it/py.typed,sha256=8PjyZ1aVoQpRVvt71muvuq5qE-jTFZkK-GLHkhdebmc,26 +markdown_it/renderer.py,sha256=Lzr0glqd5oxFL10DOfjjW8kg4Gp41idQ4viEQaE47oA,9947 +markdown_it/ruler.py,sha256=eMAtWGRAfSM33aiJed0k5923BEkuMVsMq1ct8vU-ql4,9142 +markdown_it/rules_block/__init__.py,sha256=SQpg0ocmsHeILPAWRHhzgLgJMKIcNkQyELH13o_6Ktc,553 +markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/code.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/fence.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/heading.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/hr.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/list.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/reference.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc,, +markdown_it/rules_block/__pycache__/table.cpython-314.pyc,, +markdown_it/rules_block/blockquote.py,sha256=7uymS36dcrned3DsIaRcqcbFU1NlymhvsZpEXTD3_n8,8887 +markdown_it/rules_block/code.py,sha256=iTAxv0U1-MDhz88M1m1pi2vzOhEMSEROsXMo2Qq--kU,860 +markdown_it/rules_block/fence.py,sha256=BJgU-PqZ4vAlCqGcrc8UtdLpJJyMeRWN-G-Op-zxrMc,2537 +markdown_it/rules_block/heading.py,sha256=4Lh15rwoVsQjE1hVhpbhidQ0k9xKHihgjAeYSbwgO5k,1745 +markdown_it/rules_block/hr.py,sha256=QCoY5kImaQRvF7PyP8OoWft6A8JVH1v6MN-0HR9Ikpg,1227 +markdown_it/rules_block/html_block.py,sha256=wA8pb34LtZr1BkIATgGKQBIGX5jQNOkwZl9UGEqvb5M,2721 +markdown_it/rules_block/lheading.py,sha256=fWoEuUo7S2svr5UMKmyQMkh0hheYAHg2gMM266Mogs4,2625 +markdown_it/rules_block/list.py,sha256=gIodkAJFyOIyKCZCj5lAlL7jIj5kAzrDb-K-2MFNplY,9668 +markdown_it/rules_block/paragraph.py,sha256=9pmCwA7eMu4LBdV4fWKzC4EdwaOoaGw2kfeYSQiLye8,1819 +markdown_it/rules_block/reference.py,sha256=ue1qZbUaUP0GIvwTjh6nD1UtCij8uwsIMuYW1xBkckc,6983 +markdown_it/rules_block/state_block.py,sha256=HowsQyy5hGUibH4HRZWKfLIlXeDUnuWL7kpF0-rSwoM,8422 +markdown_it/rules_block/table.py,sha256=8nMd9ONGOffER7BXmc9kbbhxkLjtpX79dVLR0iatGnM,7682 +markdown_it/rules_core/__init__.py,sha256=QFGBe9TUjnRQJDU7xY4SQYpxyTHNwg8beTSwXpNGRjE,394 +markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/block.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/inline.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc,, +markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc,, +markdown_it/rules_core/block.py,sha256=0_JY1CUy-H2OooFtIEZAACtuoGUMohgxo4Z6A_UinSg,372 +markdown_it/rules_core/inline.py,sha256=9oWmeBhJHE7x47oJcN9yp6UsAZtrEY_A-VmfoMvKld4,325 +markdown_it/rules_core/linkify.py,sha256=mjQqpk_lHLh2Nxw4UFaLxa47Fgi-OHnmDamlgXnhmv0,5141 +markdown_it/rules_core/normalize.py,sha256=AJm4femtFJ_QBnM0dzh0UNqTTJk9K6KMtwRPaioZFqM,403 +markdown_it/rules_core/replacements.py,sha256=CH75mie-tdzdLKQtMBuCTcXAl1ijegdZGfbV_Vk7st0,3471 +markdown_it/rules_core/smartquotes.py,sha256=izK9fSyuTzA-zAUGkRkz9KwwCQWo40iRqcCKqOhFbEE,7443 +markdown_it/rules_core/state_core.py,sha256=HqWZCUr5fW7xG6jeQZDdO0hE9hxxyl3_-bawgOy57HY,570 +markdown_it/rules_core/text_join.py,sha256=rLXxNuLh_es5RvH31GsXi7en8bMNO9UJ5nbJMDBPltY,1173 +markdown_it/rules_inline/__init__.py,sha256=qqHZk6-YE8Rc12q6PxvVKBaxv2wmZeeo45H1XMR_Vxs,696 +markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/image.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/link.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc,, +markdown_it/rules_inline/__pycache__/text.cpython-314.pyc,, +markdown_it/rules_inline/autolink.py,sha256=pPoqJY8i99VtFn7KgUzMackMeq1hytzioVvWs-VQPRo,2065 +markdown_it/rules_inline/backticks.py,sha256=J7bezjjNxiXlKqvHc0fJkHZwH7-2nBsXVjcKydk8E4M,2037 +markdown_it/rules_inline/balance_pairs.py,sha256=5zgBiGidqdiWmt7Io_cuZOYh5EFEfXrYRce8RXg5m7o,4852 +markdown_it/rules_inline/emphasis.py,sha256=7aDLZx0Jlekuvbu3uEUTDhJp00Z0Pj6g4C3-VLhI8Co,3123 +markdown_it/rules_inline/entity.py,sha256=CE8AIGMi5isEa24RNseo0wRmTTaj5YLbgTFdDmBesAU,1651 +markdown_it/rules_inline/escape.py,sha256=KGulwrP5FnqZM7GXY8lf7pyVv0YkR59taZDeHb5cmKg,1659 +markdown_it/rules_inline/fragments_join.py,sha256=_3JbwWYJz74gRHeZk6T8edVJT2IVSsi7FfmJJlieQlA,1493 +markdown_it/rules_inline/html_inline.py,sha256=SBg6HR0HRqCdrkkec0dfOYuQdAqyfeLRFLeQggtgjvg,1130 +markdown_it/rules_inline/image.py,sha256=Wbsg7jgnOtKXIwXGNJOlG7ORThkMkBVolxItC0ph6C0,4141 +markdown_it/rules_inline/link.py,sha256=2oD-fAdB0xyxDRtZLTjzLeWbzJ1k9bbPVQmohb58RuI,4258 +markdown_it/rules_inline/linkify.py,sha256=ifH6sb5wE8PGMWEw9Sr4x0DhMVfNOEBCfFSwKll2O-s,1706 +markdown_it/rules_inline/newline.py,sha256=329r0V3aDjzNtJcvzA3lsFYjzgBrShLAV5uf9hwQL_M,1297 +markdown_it/rules_inline/state_inline.py,sha256=d-menFzbz5FDy1JNgGBF-BASasnVI-9RuOxWz9PnKn4,5003 +markdown_it/rules_inline/strikethrough.py,sha256=pwcPlyhkh5pqFVxRCSrdW5dNCIOtU4eDit7TVDTPIVA,3214 +markdown_it/rules_inline/text.py,sha256=FQqaQRUqbnMLO9ZSWPWQUMEKH6JqWSSSmlZ5Ii9P48o,1119 +markdown_it/token.py,sha256=cWrt9kodfPdizHq_tYrzyIZNtJYNMN1813DPNlunwTg,6381 +markdown_it/tree.py,sha256=56Cdbwu2Aiks7kNYqO_fQZWpPb_n48CUllzjQQfgu1Y,11111 +markdown_it/utils.py,sha256=lVLeX7Af3GaNFfxmMgUbsn5p7cXbwhLq7RSf56UWuRE,5687 +markdown_it_py-4.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +markdown_it_py-4.0.0.dist-info/METADATA,sha256=6fyqHi2vP5bYQKCfuqo5T-qt83o22Ip7a2tnJIfGW_s,7288 +markdown_it_py-4.0.0.dist-info/RECORD,, +markdown_it_py-4.0.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 +markdown_it_py-4.0.0.dist-info/entry_points.txt,sha256=T81l7fHQ3pllpQ4wUtQK6a8g_p6wxQbnjKVHCk2WMG4,58 +markdown_it_py-4.0.0.dist-info/licenses/LICENSE,sha256=SiJg1uLND1oVGh6G2_59PtVSseK-q_mUHBulxJy85IQ,1078 +markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it,sha256=eSxIxahJoV_fnjfovPnm0d0TsytGxkKnSKCkapkZ1HM,1073 diff --git a/lib/markdown_it_py-4.0.0.dist-info/WHEEL b/lib/markdown_it_py-4.0.0.dist-info/WHEEL new file mode 100644 index 0000000..d8b9936 --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.12.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt b/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt new file mode 100644 index 0000000..7d829cd --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +markdown-it=markdown_it.cli.parse:main + diff --git a/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE b/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000..582ddf5 --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 ExecutableBookProject + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it b/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it new file mode 100644 index 0000000..7ffa058 --- /dev/null +++ b/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it @@ -0,0 +1,22 @@ +Copyright (c) 2014 Vitaly Puzrin, Alex Kocharin. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/lib/mdurl-0.1.2.dist-info/INSTALLER b/lib/mdurl-0.1.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/mdurl-0.1.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/mdurl-0.1.2.dist-info/LICENSE b/lib/mdurl-0.1.2.dist-info/LICENSE new file mode 100644 index 0000000..2a920c5 --- /dev/null +++ b/lib/mdurl-0.1.2.dist-info/LICENSE @@ -0,0 +1,46 @@ +Copyright (c) 2015 Vitaly Puzrin, Alex Kocharin. +Copyright (c) 2021 Taneli Hukkinen + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +.parse() is based on Joyent's node.js `url` code: + +Copyright Joyent, Inc. and other Node contributors. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/lib/mdurl-0.1.2.dist-info/METADATA b/lib/mdurl-0.1.2.dist-info/METADATA new file mode 100644 index 0000000..b4670e8 --- /dev/null +++ b/lib/mdurl-0.1.2.dist-info/METADATA @@ -0,0 +1,32 @@ +Metadata-Version: 2.1 +Name: mdurl +Version: 0.1.2 +Summary: Markdown URL utilities +Keywords: markdown,commonmark +Author-email: Taneli Hukkinen +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Typing :: Typed +Project-URL: Homepage, https://github.com/executablebooks/mdurl + +# mdurl + +[![Build Status](https://github.com/executablebooks/mdurl/workflows/Tests/badge.svg?branch=master)](https://github.com/executablebooks/mdurl/actions?query=workflow%3ATests+branch%3Amaster+event%3Apush) +[![codecov.io](https://codecov.io/gh/executablebooks/mdurl/branch/master/graph/badge.svg)](https://codecov.io/gh/executablebooks/mdurl) +[![PyPI version](https://img.shields.io/pypi/v/mdurl)](https://pypi.org/project/mdurl) + +This is a Python port of the JavaScript [mdurl](https://www.npmjs.com/package/mdurl) package. +See the [upstream README.md file](https://github.com/markdown-it/mdurl/blob/master/README.md) for API documentation. + diff --git a/lib/mdurl-0.1.2.dist-info/RECORD b/lib/mdurl-0.1.2.dist-info/RECORD new file mode 100644 index 0000000..355bb2f --- /dev/null +++ b/lib/mdurl-0.1.2.dist-info/RECORD @@ -0,0 +1,18 @@ +mdurl-0.1.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +mdurl-0.1.2.dist-info/LICENSE,sha256=fGBd9uKGZ6lgMRjpgnT2SknOPu0NJvzM6VNKNF4O-VU,2338 +mdurl-0.1.2.dist-info/METADATA,sha256=tTsp1I9Jk2cFP9o8gefOJ9JVg4Drv4PmYCOwLrfd0l0,1638 +mdurl-0.1.2.dist-info/RECORD,, +mdurl-0.1.2.dist-info/WHEEL,sha256=4TfKIB_xu-04bc2iKz6_zFt-gEFEEDU_31HGhqzOCE8,81 +mdurl/__init__.py,sha256=1vpE89NyXniIRZNC_4f6BPm3Ub4bPntjfyyhLRR7opU,547 +mdurl/__pycache__/__init__.cpython-314.pyc,, +mdurl/__pycache__/_decode.cpython-314.pyc,, +mdurl/__pycache__/_encode.cpython-314.pyc,, +mdurl/__pycache__/_format.cpython-314.pyc,, +mdurl/__pycache__/_parse.cpython-314.pyc,, +mdurl/__pycache__/_url.cpython-314.pyc,, +mdurl/_decode.py,sha256=3Q_gDQqU__TvDbu7x-b9LjbVl4QWy5g_qFwljcuvN_Y,3004 +mdurl/_encode.py,sha256=goJLUFt1h4rVZNqqm9t15Nw2W-bFXYQEy3aR01ImWvs,2602 +mdurl/_format.py,sha256=xZct0mdePXA0H3kAqxjGtlB5O86G35DAYMGkA44CmB4,626 +mdurl/_parse.py,sha256=ezZSkM2_4NQ2Zx047sEdcJG7NYQRFHiZK7Y8INHFzwY,11374 +mdurl/_url.py,sha256=5kQnRQN2A_G4svLnRzZcG0bfoD9AbBrYDXousDHZ3z0,284 +mdurl/py.typed,sha256=8PjyZ1aVoQpRVvt71muvuq5qE-jTFZkK-GLHkhdebmc,26 diff --git a/lib/mdurl-0.1.2.dist-info/WHEEL b/lib/mdurl-0.1.2.dist-info/WHEEL new file mode 100644 index 0000000..668ba4d --- /dev/null +++ b/lib/mdurl-0.1.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.7.1 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/mdurl/__init__.py b/lib/mdurl/__init__.py new file mode 100644 index 0000000..cdbb640 --- /dev/null +++ b/lib/mdurl/__init__.py @@ -0,0 +1,18 @@ +__all__ = ( + "decode", + "DECODE_DEFAULT_CHARS", + "DECODE_COMPONENT_CHARS", + "encode", + "ENCODE_DEFAULT_CHARS", + "ENCODE_COMPONENT_CHARS", + "format", + "parse", + "URL", +) +__version__ = "0.1.2" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +from mdurl._decode import DECODE_COMPONENT_CHARS, DECODE_DEFAULT_CHARS, decode +from mdurl._encode import ENCODE_COMPONENT_CHARS, ENCODE_DEFAULT_CHARS, encode +from mdurl._format import format +from mdurl._parse import url_parse as parse +from mdurl._url import URL diff --git a/lib/mdurl/__pycache__/__init__.cpython-314.pyc b/lib/mdurl/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..e946a91 Binary files /dev/null and b/lib/mdurl/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/mdurl/__pycache__/_decode.cpython-314.pyc b/lib/mdurl/__pycache__/_decode.cpython-314.pyc new file mode 100644 index 0000000..ccb5d52 Binary files /dev/null and b/lib/mdurl/__pycache__/_decode.cpython-314.pyc differ diff --git a/lib/mdurl/__pycache__/_encode.cpython-314.pyc b/lib/mdurl/__pycache__/_encode.cpython-314.pyc new file mode 100644 index 0000000..491fae2 Binary files /dev/null and b/lib/mdurl/__pycache__/_encode.cpython-314.pyc differ diff --git a/lib/mdurl/__pycache__/_format.cpython-314.pyc b/lib/mdurl/__pycache__/_format.cpython-314.pyc new file mode 100644 index 0000000..8867605 Binary files /dev/null and b/lib/mdurl/__pycache__/_format.cpython-314.pyc differ diff --git a/lib/mdurl/__pycache__/_parse.cpython-314.pyc b/lib/mdurl/__pycache__/_parse.cpython-314.pyc new file mode 100644 index 0000000..6cc96fe Binary files /dev/null and b/lib/mdurl/__pycache__/_parse.cpython-314.pyc differ diff --git a/lib/mdurl/__pycache__/_url.cpython-314.pyc b/lib/mdurl/__pycache__/_url.cpython-314.pyc new file mode 100644 index 0000000..7d01f4e Binary files /dev/null and b/lib/mdurl/__pycache__/_url.cpython-314.pyc differ diff --git a/lib/mdurl/_decode.py b/lib/mdurl/_decode.py new file mode 100644 index 0000000..9b50a2d --- /dev/null +++ b/lib/mdurl/_decode.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from collections.abc import Sequence +import functools +import re + +DECODE_DEFAULT_CHARS = ";/?:@&=+$,#" +DECODE_COMPONENT_CHARS = "" + +decode_cache: dict[str, list[str]] = {} + + +def get_decode_cache(exclude: str) -> Sequence[str]: + if exclude in decode_cache: + return decode_cache[exclude] + + cache: list[str] = [] + decode_cache[exclude] = cache + + for i in range(128): + ch = chr(i) + cache.append(ch) + + for i in range(len(exclude)): + ch_code = ord(exclude[i]) + cache[ch_code] = "%" + ("0" + hex(ch_code)[2:].upper())[-2:] + + return cache + + +# Decode percent-encoded string. +# +def decode(string: str, exclude: str = DECODE_DEFAULT_CHARS) -> str: + cache = get_decode_cache(exclude) + repl_func = functools.partial(repl_func_with_cache, cache=cache) + return re.sub(r"(%[a-f0-9]{2})+", repl_func, string, flags=re.IGNORECASE) + + +def repl_func_with_cache(match: re.Match, cache: Sequence[str]) -> str: + seq = match.group() + result = "" + + i = 0 + l = len(seq) # noqa: E741 + while i < l: + b1 = int(seq[i + 1 : i + 3], 16) + + if b1 < 0x80: + result += cache[b1] + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xE0) == 0xC0 and (i + 3 < l): + # 110xxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + + if (b2 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 2 + + i += 3 + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xF0) == 0xE0 and (i + 6 < l): + # 1110xxxx 10xxxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + b3 = int(seq[i + 7 : i + 9], 16) + + if (b2 & 0xC0) == 0x80 and (b3 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2, b3)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 3 + + i += 6 + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xF8) == 0xF0 and (i + 9 < l): + # 111110xx 10xxxxxx 10xxxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + b3 = int(seq[i + 7 : i + 9], 16) + b4 = int(seq[i + 10 : i + 12], 16) + + if (b2 & 0xC0) == 0x80 and (b3 & 0xC0) == 0x80 and (b4 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2, b3, b4)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 4 + + i += 9 + i += 3 # emulate JS for loop statement3 + continue + + result += "\ufffd" + i += 3 # emulate JS for loop statement3 + + return result diff --git a/lib/mdurl/_encode.py b/lib/mdurl/_encode.py new file mode 100644 index 0000000..bc2e5b9 --- /dev/null +++ b/lib/mdurl/_encode.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from collections.abc import Sequence +from string import ascii_letters, digits, hexdigits +from urllib.parse import quote as encode_uri_component + +ASCII_LETTERS_AND_DIGITS = ascii_letters + digits + +ENCODE_DEFAULT_CHARS = ";/?:@&=+$,-_.!~*'()#" +ENCODE_COMPONENT_CHARS = "-_.!~*'()" + +encode_cache: dict[str, list[str]] = {} + + +# Create a lookup array where anything but characters in `chars` string +# and alphanumeric chars is percent-encoded. +def get_encode_cache(exclude: str) -> Sequence[str]: + if exclude in encode_cache: + return encode_cache[exclude] + + cache: list[str] = [] + encode_cache[exclude] = cache + + for i in range(128): + ch = chr(i) + + if ch in ASCII_LETTERS_AND_DIGITS: + # always allow unencoded alphanumeric characters + cache.append(ch) + else: + cache.append("%" + ("0" + hex(i)[2:].upper())[-2:]) + + for i in range(len(exclude)): + cache[ord(exclude[i])] = exclude[i] + + return cache + + +# Encode unsafe characters with percent-encoding, skipping already +# encoded sequences. +# +# - string - string to encode +# - exclude - list of characters to ignore (in addition to a-zA-Z0-9) +# - keepEscaped - don't encode '%' in a correct escape sequence (default: true) +def encode( + string: str, exclude: str = ENCODE_DEFAULT_CHARS, *, keep_escaped: bool = True +) -> str: + result = "" + + cache = get_encode_cache(exclude) + + l = len(string) # noqa: E741 + i = 0 + while i < l: + code = ord(string[i]) + + # % + if keep_escaped and code == 0x25 and i + 2 < l: + if all(c in hexdigits for c in string[i + 1 : i + 3]): + result += string[i : i + 3] + i += 2 + i += 1 # JS for loop statement3 + continue + + if code < 128: + result += cache[code] + i += 1 # JS for loop statement3 + continue + + if code >= 0xD800 and code <= 0xDFFF: + if code >= 0xD800 and code <= 0xDBFF and i + 1 < l: + next_code = ord(string[i + 1]) + if next_code >= 0xDC00 and next_code <= 0xDFFF: + result += encode_uri_component(string[i] + string[i + 1]) + i += 1 + i += 1 # JS for loop statement3 + continue + result += "%EF%BF%BD" + i += 1 # JS for loop statement3 + continue + + result += encode_uri_component(string[i]) + i += 1 # JS for loop statement3 + + return result diff --git a/lib/mdurl/_format.py b/lib/mdurl/_format.py new file mode 100644 index 0000000..12524ca --- /dev/null +++ b/lib/mdurl/_format.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from mdurl._url import URL + + +def format(url: URL) -> str: # noqa: A001 + result = "" + + result += url.protocol or "" + result += "//" if url.slashes else "" + result += url.auth + "@" if url.auth else "" + + if url.hostname and ":" in url.hostname: + # ipv6 address + result += "[" + url.hostname + "]" + else: + result += url.hostname or "" + + result += ":" + url.port if url.port else "" + result += url.pathname or "" + result += url.search or "" + result += url.hash or "" + + return result diff --git a/lib/mdurl/_parse.py b/lib/mdurl/_parse.py new file mode 100644 index 0000000..ffeeac7 --- /dev/null +++ b/lib/mdurl/_parse.py @@ -0,0 +1,304 @@ +# Copyright Joyent, Inc. and other Node contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# Changes from joyent/node: +# +# 1. No leading slash in paths, +# e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/` +# +# 2. Backslashes are not replaced with slashes, +# so `http:\\example.org\` is treated like a relative path +# +# 3. Trailing colon is treated like a part of the path, +# i.e. in `http://example.org:foo` pathname is `:foo` +# +# 4. Nothing is URL-encoded in the resulting object, +# (in joyent/node some chars in auth and paths are encoded) +# +# 5. `url.parse()` does not have `parseQueryString` argument +# +# 6. Removed extraneous result properties: `host`, `path`, `query`, etc., +# which can be constructed using other parts of the url. + +from __future__ import annotations + +from collections import defaultdict +import re + +from mdurl._url import URL + +# Reference: RFC 3986, RFC 1808, RFC 2396 + +# define these here so at least they only have to be +# compiled once on the first module load. +PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE) +PORT_PATTERN = re.compile(r":[0-9]*$") + +# Special case for a simple path URL +SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$") + +# RFC 2396: characters reserved for delimiting URLs. +# We actually just auto-escape these. +DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t") + +# RFC 2396: characters not allowed for various reasons. +UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS + +# Allowed by RFCs, but cause of XSS attacks. Always escape these. +AUTO_ESCAPE = ("'",) + UNWISE +# Characters that are never ever allowed in a hostname. +# Note that any invalid chars are also handled, but these +# are the ones that are *expected* to be seen, so we fast-path +# them. +NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE +HOST_ENDING_CHARS = ("/", "?", "#") +HOSTNAME_MAX_LEN = 255 +HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$") +HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$") +# protocols that can allow "unsafe" and "unwise" chars. + +# protocols that never have a hostname. +HOSTLESS_PROTOCOL = defaultdict( + bool, + { + "javascript": True, + "javascript:": True, + }, +) +# protocols that always contain a // bit. +SLASHED_PROTOCOL = defaultdict( + bool, + { + "http": True, + "https": True, + "ftp": True, + "gopher": True, + "file": True, + "http:": True, + "https:": True, + "ftp:": True, + "gopher:": True, + "file:": True, + }, +) + + +class MutableURL: + def __init__(self) -> None: + self.protocol: str | None = None + self.slashes: bool = False + self.auth: str | None = None + self.port: str | None = None + self.hostname: str | None = None + self.hash: str | None = None + self.search: str | None = None + self.pathname: str | None = None + + def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL": + lower_proto = "" + slashes = False + rest = url + + # trim before proceeding. + # This is to support parse stuff like " http://foo.com \n" + rest = rest.strip() + + if not slashes_denote_host and len(url.split("#")) == 1: + # Try fast path regexp + simple_path = SIMPLE_PATH_PATTERN.match(rest) + if simple_path: + self.pathname = simple_path.group(1) + if simple_path.group(2): + self.search = simple_path.group(2) + return self + + proto = "" + proto_match = PROTOCOL_PATTERN.match(rest) + if proto_match: + proto = proto_match.group() + lower_proto = proto.lower() + self.protocol = proto + rest = rest[len(proto) :] + + # figure out if it's got a host + # user@server is *always* interpreted as a hostname, and url + # resolution will treat //foo/bar as host=foo,path=bar because that's + # how the browser resolves relative URLs. + if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest): + slashes = rest.startswith("//") + if slashes and not (proto and HOSTLESS_PROTOCOL[proto]): + rest = rest[2:] + self.slashes = True + + if not HOSTLESS_PROTOCOL[proto] and ( + slashes or (proto and not SLASHED_PROTOCOL[proto]) + ): + + # there's a hostname. + # the first instance of /, ?, ;, or # ends the host. + # + # If there is an @ in the hostname, then non-host chars *are* allowed + # to the left of the last @ sign, unless some host-ending character + # comes *before* the @-sign. + # URLs are obnoxious. + # + # ex: + # http://a@b@c/ => user:a@b host:c + # http://a@b?@c => user:a host:c path:/?@c + + # v0.12 TODO(isaacs): This is not quite how Chrome does things. + # Review our test case against browsers more comprehensively. + + # find the first instance of any hostEndingChars + host_end = -1 + for i in range(len(HOST_ENDING_CHARS)): + hec = rest.find(HOST_ENDING_CHARS[i]) + if hec != -1 and (host_end == -1 or hec < host_end): + host_end = hec + + # at this point, either we have an explicit point where the + # auth portion cannot go past, or the last @ char is the decider. + if host_end == -1: + # atSign can be anywhere. + at_sign = rest.rfind("@") + else: + # atSign must be in auth portion. + # http://a@b/c@d => host:b auth:a path:/c@d + at_sign = rest.rfind("@", 0, host_end + 1) + + # Now we have a portion which is definitely the auth. + # Pull that off. + if at_sign != -1: + auth = rest[:at_sign] + rest = rest[at_sign + 1 :] + self.auth = auth + + # the host is the remaining to the left of the first non-host char + host_end = -1 + for i in range(len(NON_HOST_CHARS)): + hec = rest.find(NON_HOST_CHARS[i]) + if hec != -1 and (host_end == -1 or hec < host_end): + host_end = hec + # if we still have not hit it, then the entire thing is a host. + if host_end == -1: + host_end = len(rest) + + if host_end > 0 and rest[host_end - 1] == ":": + host_end -= 1 + host = rest[:host_end] + rest = rest[host_end:] + + # pull out port. + self.parse_host(host) + + # we've indicated that there is a hostname, + # so even if it's empty, it has to be present. + self.hostname = self.hostname or "" + + # if hostname begins with [ and ends with ] + # assume that it's an IPv6 address. + ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith( + "]" + ) + + # validate a little. + if not ipv6_hostname: + hostparts = self.hostname.split(".") + l = len(hostparts) # noqa: E741 + i = 0 + while i < l: + part = hostparts[i] + if not part: + i += 1 # emulate statement3 in JS for loop + continue + if not HOSTNAME_PART_PATTERN.search(part): + newpart = "" + k = len(part) + j = 0 + while j < k: + if ord(part[j]) > 127: + # we replace non-ASCII char with a temporary placeholder + # we need this to make sure size of hostname is not + # broken by replacing non-ASCII by nothing + newpart += "x" + else: + newpart += part[j] + j += 1 # emulate statement3 in JS for loop + + # we test again with ASCII char only + if not HOSTNAME_PART_PATTERN.search(newpart): + valid_parts = hostparts[:i] + not_host = hostparts[i + 1 :] + bit = HOSTNAME_PART_START.search(part) + if bit: + valid_parts.append(bit.group(1)) + not_host.insert(0, bit.group(2)) + if not_host: + rest = ".".join(not_host) + rest + self.hostname = ".".join(valid_parts) + break + i += 1 # emulate statement3 in JS for loop + + if len(self.hostname) > HOSTNAME_MAX_LEN: + self.hostname = "" + + # strip [ and ] from the hostname + # the host field still retains them, though + if ipv6_hostname: + self.hostname = self.hostname[1:-1] + + # chop off from the tail first. + hash = rest.find("#") # noqa: A001 + if hash != -1: + # got a fragment string. + self.hash = rest[hash:] + rest = rest[:hash] + qm = rest.find("?") + if qm != -1: + self.search = rest[qm:] + rest = rest[:qm] + if rest: + self.pathname = rest + if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname: + self.pathname = "" + + return self + + def parse_host(self, host: str) -> None: + port_match = PORT_PATTERN.search(host) + if port_match: + port = port_match.group() + if port != ":": + self.port = port[1:] + host = host[: -len(port)] + if host: + self.hostname = host + + +def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL: + if isinstance(url, URL): + return url + u = MutableURL() + u.parse(url, slashes_denote_host) + return URL( + u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname + ) diff --git a/lib/mdurl/_url.py b/lib/mdurl/_url.py new file mode 100644 index 0000000..f866e7a --- /dev/null +++ b/lib/mdurl/_url.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import NamedTuple + + +class URL(NamedTuple): + protocol: str | None + slashes: bool + auth: str | None + port: str | None + hostname: str | None + hash: str | None # noqa: A003 + search: str | None + pathname: str | None diff --git a/lib/mdurl/py.typed b/lib/mdurl/py.typed new file mode 100644 index 0000000..7632ecf --- /dev/null +++ b/lib/mdurl/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/lib/nacl/__init__.py b/lib/nacl/__init__.py new file mode 100644 index 0000000..83aaacf --- /dev/null +++ b/lib/nacl/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__all__ = [ + "__uri__", + "__version__", + "__email__", +] + +__uri__ = "https://github.com/pyca/pynacl/" + +# Must be kept in sync with `pyproject.toml` +__version__ = "1.6.2" diff --git a/lib/nacl/__pycache__/__init__.cpython-314.pyc b/lib/nacl/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..22e167e Binary files /dev/null and b/lib/nacl/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/encoding.cpython-314.pyc b/lib/nacl/__pycache__/encoding.cpython-314.pyc new file mode 100644 index 0000000..da4c433 Binary files /dev/null and b/lib/nacl/__pycache__/encoding.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/exceptions.cpython-314.pyc b/lib/nacl/__pycache__/exceptions.cpython-314.pyc new file mode 100644 index 0000000..6c685f4 Binary files /dev/null and b/lib/nacl/__pycache__/exceptions.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/hash.cpython-314.pyc b/lib/nacl/__pycache__/hash.cpython-314.pyc new file mode 100644 index 0000000..1989636 Binary files /dev/null and b/lib/nacl/__pycache__/hash.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/hashlib.cpython-314.pyc b/lib/nacl/__pycache__/hashlib.cpython-314.pyc new file mode 100644 index 0000000..036846e Binary files /dev/null and b/lib/nacl/__pycache__/hashlib.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/public.cpython-314.pyc b/lib/nacl/__pycache__/public.cpython-314.pyc new file mode 100644 index 0000000..9573442 Binary files /dev/null and b/lib/nacl/__pycache__/public.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/secret.cpython-314.pyc b/lib/nacl/__pycache__/secret.cpython-314.pyc new file mode 100644 index 0000000..e0a547c Binary files /dev/null and b/lib/nacl/__pycache__/secret.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/signing.cpython-314.pyc b/lib/nacl/__pycache__/signing.cpython-314.pyc new file mode 100644 index 0000000..1eb46a7 Binary files /dev/null and b/lib/nacl/__pycache__/signing.cpython-314.pyc differ diff --git a/lib/nacl/__pycache__/utils.cpython-314.pyc b/lib/nacl/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..35aab69 Binary files /dev/null and b/lib/nacl/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/nacl/_sodium.abi3.so b/lib/nacl/_sodium.abi3.so new file mode 100755 index 0000000..8e5581b Binary files /dev/null and b/lib/nacl/_sodium.abi3.so differ diff --git a/lib/nacl/bindings/__init__.py b/lib/nacl/bindings/__init__.py new file mode 100644 index 0000000..2e07ba1 --- /dev/null +++ b/lib/nacl/bindings/__init__.py @@ -0,0 +1,508 @@ +# Copyright 2013-2019 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl.bindings.crypto_aead import ( + crypto_aead_aegis128l_ABYTES, + crypto_aead_aegis128l_KEYBYTES, + crypto_aead_aegis128l_MESSAGEBYTES_MAX, + crypto_aead_aegis128l_NPUBBYTES, + crypto_aead_aegis128l_NSECBYTES, + crypto_aead_aegis128l_decrypt, + crypto_aead_aegis128l_encrypt, + crypto_aead_aegis256_ABYTES, + crypto_aead_aegis256_KEYBYTES, + crypto_aead_aegis256_MESSAGEBYTES_MAX, + crypto_aead_aegis256_NPUBBYTES, + crypto_aead_aegis256_NSECBYTES, + crypto_aead_aegis256_decrypt, + crypto_aead_aegis256_encrypt, + crypto_aead_aes256gcm_ABYTES, + crypto_aead_aes256gcm_KEYBYTES, + crypto_aead_aes256gcm_MESSAGEBYTES_MAX, + crypto_aead_aes256gcm_NPUBBYTES, + crypto_aead_aes256gcm_NSECBYTES, + crypto_aead_aes256gcm_decrypt, + crypto_aead_aes256gcm_encrypt, + crypto_aead_chacha20poly1305_ABYTES, + crypto_aead_chacha20poly1305_KEYBYTES, + crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX, + crypto_aead_chacha20poly1305_NPUBBYTES, + crypto_aead_chacha20poly1305_NSECBYTES, + crypto_aead_chacha20poly1305_decrypt, + crypto_aead_chacha20poly1305_encrypt, + crypto_aead_chacha20poly1305_ietf_ABYTES, + crypto_aead_chacha20poly1305_ietf_KEYBYTES, + crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX, + crypto_aead_chacha20poly1305_ietf_NPUBBYTES, + crypto_aead_chacha20poly1305_ietf_NSECBYTES, + crypto_aead_chacha20poly1305_ietf_decrypt, + crypto_aead_chacha20poly1305_ietf_encrypt, + crypto_aead_xchacha20poly1305_ietf_ABYTES, + crypto_aead_xchacha20poly1305_ietf_KEYBYTES, + crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX, + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, + crypto_aead_xchacha20poly1305_ietf_NSECBYTES, + crypto_aead_xchacha20poly1305_ietf_decrypt, + crypto_aead_xchacha20poly1305_ietf_encrypt, +) +from nacl.bindings.crypto_box import ( + crypto_box, + crypto_box_BEFORENMBYTES, + crypto_box_BOXZEROBYTES, + crypto_box_NONCEBYTES, + crypto_box_PUBLICKEYBYTES, + crypto_box_SEALBYTES, + crypto_box_SECRETKEYBYTES, + crypto_box_SEEDBYTES, + crypto_box_ZEROBYTES, + crypto_box_afternm, + crypto_box_beforenm, + crypto_box_easy, + crypto_box_easy_afternm, + crypto_box_keypair, + crypto_box_open, + crypto_box_open_afternm, + crypto_box_open_easy, + crypto_box_open_easy_afternm, + crypto_box_seal, + crypto_box_seal_open, + crypto_box_seed_keypair, +) +from nacl.bindings.crypto_core import ( + crypto_core_ed25519_BYTES, + crypto_core_ed25519_NONREDUCEDSCALARBYTES, + crypto_core_ed25519_SCALARBYTES, + crypto_core_ed25519_add, + crypto_core_ed25519_from_uniform, + crypto_core_ed25519_is_valid_point, + crypto_core_ed25519_scalar_add, + crypto_core_ed25519_scalar_complement, + crypto_core_ed25519_scalar_invert, + crypto_core_ed25519_scalar_mul, + crypto_core_ed25519_scalar_negate, + crypto_core_ed25519_scalar_reduce, + crypto_core_ed25519_scalar_sub, + crypto_core_ed25519_sub, + has_crypto_core_ed25519, +) +from nacl.bindings.crypto_generichash import ( + crypto_generichash_BYTES, + crypto_generichash_BYTES_MAX, + crypto_generichash_BYTES_MIN, + crypto_generichash_KEYBYTES, + crypto_generichash_KEYBYTES_MAX, + crypto_generichash_KEYBYTES_MIN, + crypto_generichash_PERSONALBYTES, + crypto_generichash_SALTBYTES, + crypto_generichash_STATEBYTES, + generichash_blake2b_final as crypto_generichash_blake2b_final, + generichash_blake2b_init as crypto_generichash_blake2b_init, + generichash_blake2b_salt_personal as crypto_generichash_blake2b_salt_personal, + generichash_blake2b_update as crypto_generichash_blake2b_update, +) +from nacl.bindings.crypto_hash import ( + crypto_hash, + crypto_hash_BYTES, + crypto_hash_sha256, + crypto_hash_sha256_BYTES, + crypto_hash_sha512, + crypto_hash_sha512_BYTES, +) +from nacl.bindings.crypto_kx import ( + crypto_kx_PUBLIC_KEY_BYTES, + crypto_kx_SECRET_KEY_BYTES, + crypto_kx_SEED_BYTES, + crypto_kx_SESSION_KEY_BYTES, + crypto_kx_client_session_keys, + crypto_kx_keypair, + crypto_kx_seed_keypair, + crypto_kx_server_session_keys, +) +from nacl.bindings.crypto_pwhash import ( + crypto_pwhash_ALG_ARGON2I13, + crypto_pwhash_ALG_ARGON2ID13, + crypto_pwhash_ALG_DEFAULT, + crypto_pwhash_BYTES_MAX, + crypto_pwhash_BYTES_MIN, + crypto_pwhash_PASSWD_MAX, + crypto_pwhash_PASSWD_MIN, + crypto_pwhash_SALTBYTES, + crypto_pwhash_STRBYTES, + crypto_pwhash_alg, + crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE, + crypto_pwhash_argon2i_MEMLIMIT_MAX, + crypto_pwhash_argon2i_MEMLIMIT_MIN, + crypto_pwhash_argon2i_MEMLIMIT_MODERATE, + crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE, + crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE, + crypto_pwhash_argon2i_OPSLIMIT_MAX, + crypto_pwhash_argon2i_OPSLIMIT_MIN, + crypto_pwhash_argon2i_OPSLIMIT_MODERATE, + crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE, + crypto_pwhash_argon2i_STRPREFIX, + crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE, + crypto_pwhash_argon2id_MEMLIMIT_MAX, + crypto_pwhash_argon2id_MEMLIMIT_MIN, + crypto_pwhash_argon2id_MEMLIMIT_MODERATE, + crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE, + crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE, + crypto_pwhash_argon2id_OPSLIMIT_MAX, + crypto_pwhash_argon2id_OPSLIMIT_MIN, + crypto_pwhash_argon2id_OPSLIMIT_MODERATE, + crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE, + crypto_pwhash_argon2id_STRPREFIX, + crypto_pwhash_scryptsalsa208sha256_BYTES_MAX, + crypto_pwhash_scryptsalsa208sha256_BYTES_MIN, + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE, + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX, + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN, + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE, + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE, + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX, + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN, + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE, + crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX, + crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN, + crypto_pwhash_scryptsalsa208sha256_SALTBYTES, + crypto_pwhash_scryptsalsa208sha256_STRBYTES, + crypto_pwhash_scryptsalsa208sha256_STRPREFIX, + crypto_pwhash_scryptsalsa208sha256_ll, + crypto_pwhash_scryptsalsa208sha256_str, + crypto_pwhash_scryptsalsa208sha256_str_verify, + crypto_pwhash_str_alg, + crypto_pwhash_str_verify, + has_crypto_pwhash_scryptsalsa208sha256, + nacl_bindings_pick_scrypt_params, +) +from nacl.bindings.crypto_scalarmult import ( + crypto_scalarmult, + crypto_scalarmult_BYTES, + crypto_scalarmult_SCALARBYTES, + crypto_scalarmult_base, + crypto_scalarmult_ed25519, + crypto_scalarmult_ed25519_BYTES, + crypto_scalarmult_ed25519_SCALARBYTES, + crypto_scalarmult_ed25519_base, + crypto_scalarmult_ed25519_base_noclamp, + crypto_scalarmult_ed25519_noclamp, + has_crypto_scalarmult_ed25519, +) +from nacl.bindings.crypto_secretbox import ( + crypto_secretbox, + crypto_secretbox_BOXZEROBYTES, + crypto_secretbox_KEYBYTES, + crypto_secretbox_MACBYTES, + crypto_secretbox_MESSAGEBYTES_MAX, + crypto_secretbox_NONCEBYTES, + crypto_secretbox_ZEROBYTES, + crypto_secretbox_easy, + crypto_secretbox_open, + crypto_secretbox_open_easy, +) +from nacl.bindings.crypto_secretstream import ( + crypto_secretstream_xchacha20poly1305_ABYTES, + crypto_secretstream_xchacha20poly1305_HEADERBYTES, + crypto_secretstream_xchacha20poly1305_KEYBYTES, + crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX, + crypto_secretstream_xchacha20poly1305_STATEBYTES, + crypto_secretstream_xchacha20poly1305_TAG_FINAL, + crypto_secretstream_xchacha20poly1305_TAG_MESSAGE, + crypto_secretstream_xchacha20poly1305_TAG_PUSH, + crypto_secretstream_xchacha20poly1305_TAG_REKEY, + crypto_secretstream_xchacha20poly1305_init_pull, + crypto_secretstream_xchacha20poly1305_init_push, + crypto_secretstream_xchacha20poly1305_keygen, + crypto_secretstream_xchacha20poly1305_pull, + crypto_secretstream_xchacha20poly1305_push, + crypto_secretstream_xchacha20poly1305_rekey, + crypto_secretstream_xchacha20poly1305_state, +) +from nacl.bindings.crypto_shorthash import ( + BYTES as crypto_shorthash_siphash24_BYTES, + KEYBYTES as crypto_shorthash_siphash24_KEYBYTES, + XBYTES as crypto_shorthash_siphashx24_BYTES, + XKEYBYTES as crypto_shorthash_siphashx24_KEYBYTES, + crypto_shorthash_siphash24, + crypto_shorthash_siphashx24, + has_crypto_shorthash_siphashx24, +) +from nacl.bindings.crypto_sign import ( + crypto_sign, + crypto_sign_BYTES, + crypto_sign_PUBLICKEYBYTES, + crypto_sign_SECRETKEYBYTES, + crypto_sign_SEEDBYTES, + crypto_sign_ed25519_pk_to_curve25519, + crypto_sign_ed25519_sk_to_curve25519, + crypto_sign_ed25519_sk_to_pk, + crypto_sign_ed25519_sk_to_seed, + crypto_sign_ed25519ph_STATEBYTES, + crypto_sign_ed25519ph_final_create, + crypto_sign_ed25519ph_final_verify, + crypto_sign_ed25519ph_state, + crypto_sign_ed25519ph_update, + crypto_sign_keypair, + crypto_sign_open, + crypto_sign_seed_keypair, +) +from nacl.bindings.randombytes import ( + randombytes, + randombytes_buf_deterministic, +) +from nacl.bindings.sodium_core import sodium_init +from nacl.bindings.utils import ( + sodium_add, + sodium_increment, + sodium_memcmp, + sodium_pad, + sodium_unpad, +) + + +__all__ = [ + "crypto_aead_aegis128l_ABYTES", + "crypto_aead_aegis128l_KEYBYTES", + "crypto_aead_aegis128l_MESSAGEBYTES_MAX", + "crypto_aead_aegis128l_NPUBBYTES", + "crypto_aead_aegis128l_NSECBYTES", + "crypto_aead_aegis128l_decrypt", + "crypto_aead_aegis128l_encrypt", + "crypto_aead_aegis256_ABYTES", + "crypto_aead_aegis256_KEYBYTES", + "crypto_aead_aegis256_MESSAGEBYTES_MAX", + "crypto_aead_aegis256_NPUBBYTES", + "crypto_aead_aegis256_NSECBYTES", + "crypto_aead_aegis256_decrypt", + "crypto_aead_aegis256_encrypt", + "crypto_aead_aes256gcm_ABYTES", + "crypto_aead_aes256gcm_KEYBYTES", + "crypto_aead_aes256gcm_MESSAGEBYTES_MAX", + "crypto_aead_aes256gcm_NPUBBYTES", + "crypto_aead_aes256gcm_NSECBYTES", + "crypto_aead_aes256gcm_decrypt", + "crypto_aead_aes256gcm_encrypt", + "crypto_aead_chacha20poly1305_ABYTES", + "crypto_aead_chacha20poly1305_KEYBYTES", + "crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX", + "crypto_aead_chacha20poly1305_NPUBBYTES", + "crypto_aead_chacha20poly1305_NSECBYTES", + "crypto_aead_chacha20poly1305_decrypt", + "crypto_aead_chacha20poly1305_encrypt", + "crypto_aead_chacha20poly1305_ietf_ABYTES", + "crypto_aead_chacha20poly1305_ietf_KEYBYTES", + "crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX", + "crypto_aead_chacha20poly1305_ietf_NPUBBYTES", + "crypto_aead_chacha20poly1305_ietf_NSECBYTES", + "crypto_aead_chacha20poly1305_ietf_decrypt", + "crypto_aead_chacha20poly1305_ietf_encrypt", + "crypto_aead_xchacha20poly1305_ietf_ABYTES", + "crypto_aead_xchacha20poly1305_ietf_KEYBYTES", + "crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX", + "crypto_aead_xchacha20poly1305_ietf_NPUBBYTES", + "crypto_aead_xchacha20poly1305_ietf_NSECBYTES", + "crypto_aead_xchacha20poly1305_ietf_decrypt", + "crypto_aead_xchacha20poly1305_ietf_encrypt", + "crypto_box_SECRETKEYBYTES", + "crypto_box_PUBLICKEYBYTES", + "crypto_box_SEEDBYTES", + "crypto_box_NONCEBYTES", + "crypto_box_ZEROBYTES", + "crypto_box_BOXZEROBYTES", + "crypto_box_BEFORENMBYTES", + "crypto_box_SEALBYTES", + "crypto_box_keypair", + "crypto_box", + "crypto_box_open", + "crypto_box_beforenm", + "crypto_box_afternm", + "crypto_box_open_afternm", + "crypto_box_easy", + "crypto_box_easy_afternm", + "crypto_box_open_easy", + "crypto_box_open_easy_afternm", + "crypto_box_seal", + "crypto_box_seal_open", + "crypto_box_seed_keypair", + "has_crypto_core_ed25519", + "crypto_core_ed25519_BYTES", + "crypto_core_ed25519_UNIFORMBYTES", + "crypto_core_ed25519_SCALARBYTES", + "crypto_core_ed25519_NONREDUCEDSCALARBYTES", + "crypto_core_ed25519_add", + "crypto_core_ed25519_from_uniform", + "crypto_core_ed25519_is_valid_point", + "crypto_core_ed25519_sub", + "crypto_core_ed25519_scalar_invert", + "crypto_core_ed25519_scalar_negate", + "crypto_core_ed25519_scalar_complement", + "crypto_core_ed25519_scalar_add", + "crypto_core_ed25519_scalar_sub", + "crypto_core_ed25519_scalar_mul", + "crypto_core_ed25519_scalar_reduce", + "crypto_hash_BYTES", + "crypto_hash_sha256_BYTES", + "crypto_hash_sha512_BYTES", + "crypto_hash", + "crypto_hash_sha256", + "crypto_hash_sha512", + "crypto_generichash_BYTES", + "crypto_generichash_BYTES_MIN", + "crypto_generichash_BYTES_MAX", + "crypto_generichash_KEYBYTES", + "crypto_generichash_KEYBYTES_MIN", + "crypto_generichash_KEYBYTES_MAX", + "crypto_generichash_SALTBYTES", + "crypto_generichash_PERSONALBYTES", + "crypto_generichash_STATEBYTES", + "crypto_generichash_blake2b_salt_personal", + "crypto_generichash_blake2b_init", + "crypto_generichash_blake2b_update", + "crypto_generichash_blake2b_final", + "crypto_kx_keypair", + "crypto_kx_seed_keypair", + "crypto_kx_client_session_keys", + "crypto_kx_server_session_keys", + "crypto_kx_PUBLIC_KEY_BYTES", + "crypto_kx_SECRET_KEY_BYTES", + "crypto_kx_SEED_BYTES", + "crypto_kx_SESSION_KEY_BYTES", + "has_crypto_scalarmult_ed25519", + "crypto_scalarmult_BYTES", + "crypto_scalarmult_SCALARBYTES", + "crypto_scalarmult", + "crypto_scalarmult_base", + "crypto_scalarmult_ed25519_BYTES", + "crypto_scalarmult_ed25519_SCALARBYTES", + "crypto_scalarmult_ed25519", + "crypto_scalarmult_ed25519_base", + "crypto_scalarmult_ed25519_noclamp", + "crypto_scalarmult_ed25519_base_noclamp", + "crypto_secretbox_KEYBYTES", + "crypto_secretbox_NONCEBYTES", + "crypto_secretbox_ZEROBYTES", + "crypto_secretbox_BOXZEROBYTES", + "crypto_secretbox_MACBYTES", + "crypto_secretbox_MESSAGEBYTES_MAX", + "crypto_secretbox", + "crypto_secretbox_easy", + "crypto_secretbox_open", + "crypto_secretbox_open_easy", + "crypto_secretstream_xchacha20poly1305_ABYTES", + "crypto_secretstream_xchacha20poly1305_HEADERBYTES", + "crypto_secretstream_xchacha20poly1305_KEYBYTES", + "crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX", + "crypto_secretstream_xchacha20poly1305_STATEBYTES", + "crypto_secretstream_xchacha20poly1305_TAG_FINAL", + "crypto_secretstream_xchacha20poly1305_TAG_MESSAGE", + "crypto_secretstream_xchacha20poly1305_TAG_PUSH", + "crypto_secretstream_xchacha20poly1305_TAG_REKEY", + "crypto_secretstream_xchacha20poly1305_init_pull", + "crypto_secretstream_xchacha20poly1305_init_push", + "crypto_secretstream_xchacha20poly1305_keygen", + "crypto_secretstream_xchacha20poly1305_pull", + "crypto_secretstream_xchacha20poly1305_push", + "crypto_secretstream_xchacha20poly1305_rekey", + "crypto_secretstream_xchacha20poly1305_state", + "has_crypto_shorthash_siphashx24", + "crypto_shorthash_siphash24_BYTES", + "crypto_shorthash_siphash24_KEYBYTES", + "crypto_shorthash_siphash24", + "crypto_shorthash_siphashx24_BYTES", + "crypto_shorthash_siphashx24_KEYBYTES", + "crypto_shorthash_siphashx24", + "crypto_sign_BYTES", + "crypto_sign_SEEDBYTES", + "crypto_sign_PUBLICKEYBYTES", + "crypto_sign_SECRETKEYBYTES", + "crypto_sign_keypair", + "crypto_sign_seed_keypair", + "crypto_sign", + "crypto_sign_open", + "crypto_sign_ed25519_pk_to_curve25519", + "crypto_sign_ed25519_sk_to_curve25519", + "crypto_sign_ed25519_sk_to_pk", + "crypto_sign_ed25519_sk_to_seed", + "crypto_sign_ed25519ph_STATEBYTES", + "crypto_sign_ed25519ph_final_create", + "crypto_sign_ed25519ph_final_verify", + "crypto_sign_ed25519ph_state", + "crypto_sign_ed25519ph_update", + "crypto_pwhash_ALG_ARGON2I13", + "crypto_pwhash_ALG_ARGON2ID13", + "crypto_pwhash_ALG_DEFAULT", + "crypto_pwhash_BYTES_MAX", + "crypto_pwhash_BYTES_MIN", + "crypto_pwhash_PASSWD_MAX", + "crypto_pwhash_PASSWD_MIN", + "crypto_pwhash_SALTBYTES", + "crypto_pwhash_STRBYTES", + "crypto_pwhash_alg", + "crypto_pwhash_argon2i_MEMLIMIT_MIN", + "crypto_pwhash_argon2i_MEMLIMIT_MAX", + "crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE", + "crypto_pwhash_argon2i_MEMLIMIT_MODERATE", + "crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE", + "crypto_pwhash_argon2i_OPSLIMIT_MIN", + "crypto_pwhash_argon2i_OPSLIMIT_MAX", + "crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE", + "crypto_pwhash_argon2i_OPSLIMIT_MODERATE", + "crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE", + "crypto_pwhash_argon2i_STRPREFIX", + "crypto_pwhash_argon2id_MEMLIMIT_MIN", + "crypto_pwhash_argon2id_MEMLIMIT_MAX", + "crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE", + "crypto_pwhash_argon2id_MEMLIMIT_MODERATE", + "crypto_pwhash_argon2id_OPSLIMIT_MIN", + "crypto_pwhash_argon2id_OPSLIMIT_MAX", + "crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE", + "crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE", + "crypto_pwhash_argon2id_OPSLIMIT_MODERATE", + "crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE", + "crypto_pwhash_argon2id_STRPREFIX", + "crypto_pwhash_str_alg", + "crypto_pwhash_str_verify", + "has_crypto_pwhash_scryptsalsa208sha256", + "crypto_pwhash_scryptsalsa208sha256_BYTES_MAX", + "crypto_pwhash_scryptsalsa208sha256_BYTES_MIN", + "crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE", + "crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX", + "crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN", + "crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE", + "crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE", + "crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX", + "crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN", + "crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE", + "crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX", + "crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN", + "crypto_pwhash_scryptsalsa208sha256_SALTBYTES", + "crypto_pwhash_scryptsalsa208sha256_STRBYTES", + "crypto_pwhash_scryptsalsa208sha256_STRPREFIX", + "crypto_pwhash_scryptsalsa208sha256_ll", + "crypto_pwhash_scryptsalsa208sha256_str", + "crypto_pwhash_scryptsalsa208sha256_str_verify", + "nacl_bindings_pick_scrypt_params", + "randombytes", + "randombytes_buf_deterministic", + "sodium_init", + "sodium_add", + "sodium_increment", + "sodium_memcmp", + "sodium_pad", + "sodium_unpad", +] + + +# Initialize Sodium +sodium_init() diff --git a/lib/nacl/bindings/__pycache__/__init__.cpython-314.pyc b/lib/nacl/bindings/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..8a3a7ac Binary files /dev/null and b/lib/nacl/bindings/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_aead.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_aead.cpython-314.pyc new file mode 100644 index 0000000..c50a466 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_aead.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_box.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_box.cpython-314.pyc new file mode 100644 index 0000000..0b2d923 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_box.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_core.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_core.cpython-314.pyc new file mode 100644 index 0000000..e7db98c Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_core.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_generichash.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_generichash.cpython-314.pyc new file mode 100644 index 0000000..6de624a Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_generichash.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_hash.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_hash.cpython-314.pyc new file mode 100644 index 0000000..2ffe088 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_hash.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_kx.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_kx.cpython-314.pyc new file mode 100644 index 0000000..8f5c70e Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_kx.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_pwhash.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_pwhash.cpython-314.pyc new file mode 100644 index 0000000..9b3c545 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_pwhash.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_scalarmult.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_scalarmult.cpython-314.pyc new file mode 100644 index 0000000..bc729de Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_scalarmult.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_secretbox.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_secretbox.cpython-314.pyc new file mode 100644 index 0000000..16e5353 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_secretbox.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_secretstream.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_secretstream.cpython-314.pyc new file mode 100644 index 0000000..616e681 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_secretstream.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_shorthash.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_shorthash.cpython-314.pyc new file mode 100644 index 0000000..557819f Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_shorthash.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/crypto_sign.cpython-314.pyc b/lib/nacl/bindings/__pycache__/crypto_sign.cpython-314.pyc new file mode 100644 index 0000000..ce11b07 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/crypto_sign.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/randombytes.cpython-314.pyc b/lib/nacl/bindings/__pycache__/randombytes.cpython-314.pyc new file mode 100644 index 0000000..7b5f524 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/randombytes.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/sodium_core.cpython-314.pyc b/lib/nacl/bindings/__pycache__/sodium_core.cpython-314.pyc new file mode 100644 index 0000000..315e647 Binary files /dev/null and b/lib/nacl/bindings/__pycache__/sodium_core.cpython-314.pyc differ diff --git a/lib/nacl/bindings/__pycache__/utils.cpython-314.pyc b/lib/nacl/bindings/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..07a160a Binary files /dev/null and b/lib/nacl/bindings/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/nacl/bindings/crypto_aead.py b/lib/nacl/bindings/crypto_aead.py new file mode 100644 index 0000000..2f7da78 --- /dev/null +++ b/lib/nacl/bindings/crypto_aead.py @@ -0,0 +1,1069 @@ +# Copyright 2017 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + +""" +Implementations of authenticated encription with associated data (*AEAD*) +constructions building on the chacha20 stream cipher and the poly1305 +authenticator +""" + +crypto_aead_chacha20poly1305_ietf_KEYBYTES: int = ( + lib.crypto_aead_chacha20poly1305_ietf_keybytes() +) +crypto_aead_chacha20poly1305_ietf_NSECBYTES: int = ( + lib.crypto_aead_chacha20poly1305_ietf_nsecbytes() +) +crypto_aead_chacha20poly1305_ietf_NPUBBYTES: int = ( + lib.crypto_aead_chacha20poly1305_ietf_npubbytes() +) +crypto_aead_chacha20poly1305_ietf_ABYTES: int = ( + lib.crypto_aead_chacha20poly1305_ietf_abytes() +) +crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX: int = ( + lib.crypto_aead_chacha20poly1305_ietf_messagebytes_max() +) +_aead_chacha20poly1305_ietf_CRYPTBYTES_MAX = ( + crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX + + crypto_aead_chacha20poly1305_ietf_ABYTES +) + +crypto_aead_chacha20poly1305_KEYBYTES: int = ( + lib.crypto_aead_chacha20poly1305_keybytes() +) +crypto_aead_chacha20poly1305_NSECBYTES: int = ( + lib.crypto_aead_chacha20poly1305_nsecbytes() +) +crypto_aead_chacha20poly1305_NPUBBYTES: int = ( + lib.crypto_aead_chacha20poly1305_npubbytes() +) +crypto_aead_chacha20poly1305_ABYTES: int = ( + lib.crypto_aead_chacha20poly1305_abytes() +) +crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX: int = ( + lib.crypto_aead_chacha20poly1305_messagebytes_max() +) +_aead_chacha20poly1305_CRYPTBYTES_MAX = ( + crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX + + crypto_aead_chacha20poly1305_ABYTES +) + +crypto_aead_xchacha20poly1305_ietf_KEYBYTES: int = ( + lib.crypto_aead_xchacha20poly1305_ietf_keybytes() +) +crypto_aead_xchacha20poly1305_ietf_NSECBYTES: int = ( + lib.crypto_aead_xchacha20poly1305_ietf_nsecbytes() +) +crypto_aead_xchacha20poly1305_ietf_NPUBBYTES: int = ( + lib.crypto_aead_xchacha20poly1305_ietf_npubbytes() +) +crypto_aead_xchacha20poly1305_ietf_ABYTES: int = ( + lib.crypto_aead_xchacha20poly1305_ietf_abytes() +) +crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX: int = ( + lib.crypto_aead_xchacha20poly1305_ietf_messagebytes_max() +) +_aead_xchacha20poly1305_ietf_CRYPTBYTES_MAX = ( + crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + + crypto_aead_xchacha20poly1305_ietf_ABYTES +) + +crypto_aead_aegis256_KEYBYTES: int = lib.crypto_aead_aegis256_keybytes() +crypto_aead_aegis256_NSECBYTES: int = lib.crypto_aead_aegis256_nsecbytes() +crypto_aead_aegis256_NPUBBYTES: int = lib.crypto_aead_aegis256_npubbytes() +crypto_aead_aegis256_ABYTES: int = lib.crypto_aead_aegis256_abytes() +crypto_aead_aegis256_MESSAGEBYTES_MAX: int = ( + lib.crypto_aead_aegis256_messagebytes_max() +) +_aead_aegis256_CRYPTBYTES_MAX = ( + crypto_aead_aegis256_MESSAGEBYTES_MAX + crypto_aead_aegis256_ABYTES +) + +crypto_aead_aegis128l_KEYBYTES: int = lib.crypto_aead_aegis128l_keybytes() +crypto_aead_aegis128l_NSECBYTES: int = lib.crypto_aead_aegis128l_nsecbytes() +crypto_aead_aegis128l_NPUBBYTES: int = lib.crypto_aead_aegis128l_npubbytes() +crypto_aead_aegis128l_ABYTES: int = lib.crypto_aead_aegis128l_abytes() +crypto_aead_aegis128l_MESSAGEBYTES_MAX: int = ( + lib.crypto_aead_aegis128l_messagebytes_max() +) +_aead_aegis256_CRYPTBYTES_MAX = ( + crypto_aead_aegis128l_MESSAGEBYTES_MAX + crypto_aead_aegis128l_ABYTES +) + +crypto_aead_aes256gcm_KEYBYTES: int = lib.crypto_aead_aes256gcm_keybytes() +crypto_aead_aes256gcm_NSECBYTES: int = lib.crypto_aead_aes256gcm_nsecbytes() +crypto_aead_aes256gcm_NPUBBYTES: int = lib.crypto_aead_aes256gcm_npubbytes() +crypto_aead_aes256gcm_ABYTES: int = lib.crypto_aead_aes256gcm_abytes() +crypto_aead_aes256gcm_MESSAGEBYTES_MAX: int = ( + lib.crypto_aead_aes256gcm_messagebytes_max() +) +_aead_aegis256_CRYPTBYTES_MAX = ( + crypto_aead_aes256gcm_MESSAGEBYTES_MAX + crypto_aead_aes256gcm_ABYTES +) + + +def crypto_aead_chacha20poly1305_ietf_encrypt( + message: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Encrypt the given ``message`` using the IETF ratified chacha20poly1305 + construction described in RFC7539. + + :param message: + :type message: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: authenticated ciphertext + :rtype: bytes + """ + ensure( + isinstance(message, bytes), + "Input message type must be bytes", + raising=exc.TypeError, + ) + + mlen = len(message) + + ensure( + mlen <= crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX, + "Message must be at most {} bytes long".format( + crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_chacha20poly1305_ietf_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_ietf_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) + and len(key) == crypto_aead_chacha20poly1305_ietf_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_ietf_KEYBYTES + ), + raising=exc.TypeError, + ) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + mxout = mlen + crypto_aead_chacha20poly1305_ietf_ABYTES + + clen = ffi.new("unsigned long long *") + + ciphertext = ffi.new("unsigned char[]", mxout) + + res = lib.crypto_aead_chacha20poly1305_ietf_encrypt( + ciphertext, clen, message, mlen, _aad, aalen, ffi.NULL, nonce, key + ) + + ensure(res == 0, "Encryption failed.", raising=exc.CryptoError) + return ffi.buffer(ciphertext, clen[0])[:] + + +def crypto_aead_chacha20poly1305_ietf_decrypt( + ciphertext: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt the given ``ciphertext`` using the IETF ratified chacha20poly1305 + construction described in RFC7539. + + :param ciphertext: + :type ciphertext: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: message + :rtype: bytes + """ + ensure( + isinstance(ciphertext, bytes), + "Input ciphertext type must be bytes", + raising=exc.TypeError, + ) + + clen = len(ciphertext) + + ensure( + clen <= _aead_chacha20poly1305_ietf_CRYPTBYTES_MAX, + "Ciphertext must be at most {} bytes long".format( + _aead_chacha20poly1305_ietf_CRYPTBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_chacha20poly1305_ietf_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_ietf_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) + and len(key) == crypto_aead_chacha20poly1305_ietf_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_ietf_KEYBYTES + ), + raising=exc.TypeError, + ) + + mxout = clen - crypto_aead_chacha20poly1305_ietf_ABYTES + + mlen = ffi.new("unsigned long long *") + message = ffi.new("unsigned char[]", mxout) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + res = lib.crypto_aead_chacha20poly1305_ietf_decrypt( + message, mlen, ffi.NULL, ciphertext, clen, _aad, aalen, nonce, key + ) + + ensure(res == 0, "Decryption failed.", raising=exc.CryptoError) + + return ffi.buffer(message, mlen[0])[:] + + +def crypto_aead_chacha20poly1305_encrypt( + message: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Encrypt the given ``message`` using the "legacy" construction + described in draft-agl-tls-chacha20poly1305. + + :param message: + :type message: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: authenticated ciphertext + :rtype: bytes + """ + ensure( + isinstance(message, bytes), + "Input message type must be bytes", + raising=exc.TypeError, + ) + + mlen = len(message) + + ensure( + mlen <= crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX, + "Message must be at most {} bytes long".format( + crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_chacha20poly1305_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) + and len(key) == crypto_aead_chacha20poly1305_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_KEYBYTES + ), + raising=exc.TypeError, + ) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + mxout = mlen + crypto_aead_chacha20poly1305_ietf_ABYTES + + clen = ffi.new("unsigned long long *") + + ciphertext = ffi.new("unsigned char[]", mxout) + + res = lib.crypto_aead_chacha20poly1305_encrypt( + ciphertext, clen, message, mlen, _aad, aalen, ffi.NULL, nonce, key + ) + + ensure(res == 0, "Encryption failed.", raising=exc.CryptoError) + return ffi.buffer(ciphertext, clen[0])[:] + + +def crypto_aead_chacha20poly1305_decrypt( + ciphertext: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt the given ``ciphertext`` using the "legacy" construction + described in draft-agl-tls-chacha20poly1305. + + :param ciphertext: authenticated ciphertext + :type ciphertext: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: message + :rtype: bytes + """ + ensure( + isinstance(ciphertext, bytes), + "Input ciphertext type must be bytes", + raising=exc.TypeError, + ) + + clen = len(ciphertext) + + ensure( + clen <= _aead_chacha20poly1305_CRYPTBYTES_MAX, + "Ciphertext must be at most {} bytes long".format( + _aead_chacha20poly1305_CRYPTBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_chacha20poly1305_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) + and len(key) == crypto_aead_chacha20poly1305_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_chacha20poly1305_KEYBYTES + ), + raising=exc.TypeError, + ) + + mxout = clen - crypto_aead_chacha20poly1305_ABYTES + + mlen = ffi.new("unsigned long long *") + message = ffi.new("unsigned char[]", mxout) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + res = lib.crypto_aead_chacha20poly1305_decrypt( + message, mlen, ffi.NULL, ciphertext, clen, _aad, aalen, nonce, key + ) + + ensure(res == 0, "Decryption failed.", raising=exc.CryptoError) + + return ffi.buffer(message, mlen[0])[:] + + +def crypto_aead_xchacha20poly1305_ietf_encrypt( + message: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Encrypt the given ``message`` using the long-nonces xchacha20poly1305 + construction. + + :param message: + :type message: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: authenticated ciphertext + :rtype: bytes + """ + ensure( + isinstance(message, bytes), + "Input message type must be bytes", + raising=exc.TypeError, + ) + + mlen = len(message) + + ensure( + mlen <= crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX, + "Message must be at most {} bytes long".format( + crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) + and len(key) == crypto_aead_xchacha20poly1305_ietf_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_xchacha20poly1305_ietf_KEYBYTES + ), + raising=exc.TypeError, + ) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + mxout = mlen + crypto_aead_xchacha20poly1305_ietf_ABYTES + + clen = ffi.new("unsigned long long *") + + ciphertext = ffi.new("unsigned char[]", mxout) + + res = lib.crypto_aead_xchacha20poly1305_ietf_encrypt( + ciphertext, clen, message, mlen, _aad, aalen, ffi.NULL, nonce, key + ) + + ensure(res == 0, "Encryption failed.", raising=exc.CryptoError) + return ffi.buffer(ciphertext, clen[0])[:] + + +def crypto_aead_xchacha20poly1305_ietf_decrypt( + ciphertext: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt the given ``ciphertext`` using the long-nonces xchacha20poly1305 + construction. + + :param ciphertext: authenticated ciphertext + :type ciphertext: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: message + :rtype: bytes + """ + ensure( + isinstance(ciphertext, bytes), + "Input ciphertext type must be bytes", + raising=exc.TypeError, + ) + + clen = len(ciphertext) + + ensure( + clen <= _aead_xchacha20poly1305_ietf_CRYPTBYTES_MAX, + "Ciphertext must be at most {} bytes long".format( + _aead_xchacha20poly1305_ietf_CRYPTBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) + and len(key) == crypto_aead_xchacha20poly1305_ietf_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_xchacha20poly1305_ietf_KEYBYTES + ), + raising=exc.TypeError, + ) + + mxout = clen - crypto_aead_xchacha20poly1305_ietf_ABYTES + mlen = ffi.new("unsigned long long *") + message = ffi.new("unsigned char[]", mxout) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + res = lib.crypto_aead_xchacha20poly1305_ietf_decrypt( + message, mlen, ffi.NULL, ciphertext, clen, _aad, aalen, nonce, key + ) + + ensure(res == 0, "Decryption failed.", raising=exc.CryptoError) + + return ffi.buffer(message, mlen[0])[:] + + +def crypto_aead_aegis256_encrypt( + message: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Encrypt the given ``message`` using the AEGIS-256 + construction. + + :param message: + :type message: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: authenticated ciphertext + :rtype: bytes + """ + ensure( + isinstance(message, bytes), + "Input message type must be bytes", + raising=exc.TypeError, + ) + + mlen = len(message) + + ensure( + mlen <= crypto_aead_aegis256_MESSAGEBYTES_MAX, + "Message must be at most {} bytes long".format( + crypto_aead_aegis256_MESSAGEBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_aegis256_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_aegis256_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) and len(key) == crypto_aead_aegis256_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_aegis256_KEYBYTES + ), + raising=exc.TypeError, + ) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + mxout = mlen + crypto_aead_aegis256_ABYTES + + clen = ffi.new("unsigned long long *") + + ciphertext = ffi.new("unsigned char[]", mxout) + + res = lib.crypto_aead_aegis256_encrypt( + ciphertext, clen, message, mlen, _aad, aalen, ffi.NULL, nonce, key + ) + + ensure(res == 0, "Encryption failed.", raising=exc.CryptoError) + return ffi.buffer(ciphertext, clen[0])[:] + + +def crypto_aead_aegis256_decrypt( + ciphertext: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt the given ``ciphertext`` using the AEGIS-256 + construction. + + :param ciphertext: authenticated ciphertext + :type ciphertext: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: message + :rtype: bytes + """ + ensure( + isinstance(ciphertext, bytes), + "Input ciphertext type must be bytes", + raising=exc.TypeError, + ) + + clen = len(ciphertext) + + ensure( + clen <= _aead_aegis256_CRYPTBYTES_MAX, + "Ciphertext must be at most {} bytes long".format( + _aead_aegis256_CRYPTBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_aegis256_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_aegis256_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) and len(key) == crypto_aead_aegis256_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_aegis256_KEYBYTES + ), + raising=exc.TypeError, + ) + + mxout = clen - crypto_aead_aegis256_ABYTES + mlen = ffi.new("unsigned long long *") + message = ffi.new("unsigned char[]", mxout) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + res = lib.crypto_aead_aegis256_decrypt( + message, mlen, ffi.NULL, ciphertext, clen, _aad, aalen, nonce, key + ) + + ensure(res == 0, "Decryption failed.", raising=exc.CryptoError) + + return ffi.buffer(message, mlen[0])[:] + + +def crypto_aead_aegis128l_encrypt( + message: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Encrypt the given ``message`` using the AEGIS-128L + construction. + + :param message: + :type message: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: authenticated ciphertext + :rtype: bytes + """ + ensure( + isinstance(message, bytes), + "Input message type must be bytes", + raising=exc.TypeError, + ) + + mlen = len(message) + + ensure( + mlen <= crypto_aead_aegis128l_MESSAGEBYTES_MAX, + "Message must be at most {} bytes long".format( + crypto_aead_aegis128l_MESSAGEBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_aegis128l_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_aegis128l_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) and len(key) == crypto_aead_aegis128l_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_aegis128l_KEYBYTES + ), + raising=exc.TypeError, + ) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + mxout = mlen + crypto_aead_aegis128l_ABYTES + + clen = ffi.new("unsigned long long *") + + ciphertext = ffi.new("unsigned char[]", mxout) + + res = lib.crypto_aead_aegis128l_encrypt( + ciphertext, clen, message, mlen, _aad, aalen, ffi.NULL, nonce, key + ) + + ensure(res == 0, "Encryption failed.", raising=exc.CryptoError) + return ffi.buffer(ciphertext, clen[0])[:] + + +def crypto_aead_aegis128l_decrypt( + ciphertext: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt the given ``ciphertext`` using the AEGIS-128L + construction. + + :param ciphertext: authenticated ciphertext + :type ciphertext: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: message + :rtype: bytes + """ + ensure( + isinstance(ciphertext, bytes), + "Input ciphertext type must be bytes", + raising=exc.TypeError, + ) + + clen = len(ciphertext) + + ensure( + clen <= _aead_aegis256_CRYPTBYTES_MAX, + "Ciphertext must be at most {} bytes long".format( + _aead_aegis256_CRYPTBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_aegis128l_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_aegis128l_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) and len(key) == crypto_aead_aegis128l_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_aegis128l_KEYBYTES + ), + raising=exc.TypeError, + ) + + mxout = clen - crypto_aead_aegis128l_ABYTES + mlen = ffi.new("unsigned long long *") + message = ffi.new("unsigned char[]", mxout) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + res = lib.crypto_aead_aegis128l_decrypt( + message, mlen, ffi.NULL, ciphertext, clen, _aad, aalen, nonce, key + ) + + ensure(res == 0, "Decryption failed.", raising=exc.CryptoError) + + return ffi.buffer(message, mlen[0])[:] + + +def crypto_aead_aes256gcm_encrypt( + message: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Encrypt the given ``message`` using the AES-256-GCM + construction. Requires the Intel AES-NI extensions, + or the ARM Crypto extensions. + + :param message: + :type message: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: authenticated ciphertext + :rtype: bytes + """ + ensure( + lib.crypto_aead_aes256gcm_is_available() == 1, + "Construction requires hardware acceleration", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(message, bytes), + "Input message type must be bytes", + raising=exc.TypeError, + ) + + mlen = len(message) + + ensure( + mlen <= crypto_aead_aes256gcm_MESSAGEBYTES_MAX, + "Message must be at most {} bytes long".format( + crypto_aead_aes256gcm_MESSAGEBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_aes256gcm_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_aes256gcm_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) and len(key) == crypto_aead_aes256gcm_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_aes256gcm_KEYBYTES + ), + raising=exc.TypeError, + ) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + mxout = mlen + crypto_aead_aes256gcm_ABYTES + + clen = ffi.new("unsigned long long *") + + ciphertext = ffi.new("unsigned char[]", mxout) + + res = lib.crypto_aead_aes256gcm_encrypt( + ciphertext, clen, message, mlen, _aad, aalen, ffi.NULL, nonce, key + ) + + ensure(res == 0, "Encryption failed.", raising=exc.CryptoError) + return ffi.buffer(ciphertext, clen[0])[:] + + +def crypto_aead_aes256gcm_decrypt( + ciphertext: bytes, aad: Optional[bytes], nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt the given ``ciphertext`` using the AES-256-GCM + construction. Requires the Intel AES-NI extensions, + or the ARM Crypto extensions. + + :param ciphertext: authenticated ciphertext + :type ciphertext: bytes + :param aad: + :type aad: Optional[bytes] + :param nonce: + :type nonce: bytes + :param key: + :type key: bytes + :return: message + :rtype: bytes + """ + ensure( + lib.crypto_aead_aes256gcm_is_available() == 1, + "Construction requires hardware acceleration", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(ciphertext, bytes), + "Input ciphertext type must be bytes", + raising=exc.TypeError, + ) + + clen = len(ciphertext) + + ensure( + clen <= _aead_aegis256_CRYPTBYTES_MAX, + "Ciphertext must be at most {} bytes long".format( + _aead_aegis256_CRYPTBYTES_MAX + ), + raising=exc.ValueError, + ) + + ensure( + isinstance(aad, bytes) or (aad is None), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + ensure( + isinstance(nonce, bytes) + and len(nonce) == crypto_aead_aes256gcm_NPUBBYTES, + "Nonce must be a {} bytes long bytes sequence".format( + crypto_aead_aes256gcm_NPUBBYTES + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(key, bytes) and len(key) == crypto_aead_aes256gcm_KEYBYTES, + "Key must be a {} bytes long bytes sequence".format( + crypto_aead_aes256gcm_KEYBYTES + ), + raising=exc.TypeError, + ) + + mxout = clen - crypto_aead_aes256gcm_ABYTES + mlen = ffi.new("unsigned long long *") + message = ffi.new("unsigned char[]", mxout) + + if aad: + _aad = aad + aalen = len(aad) + else: + _aad = ffi.NULL + aalen = 0 + + res = lib.crypto_aead_aes256gcm_decrypt( + message, mlen, ffi.NULL, ciphertext, clen, _aad, aalen, nonce, key + ) + + ensure(res == 0, "Decryption failed.", raising=exc.CryptoError) + + return ffi.buffer(message, mlen[0])[:] diff --git a/lib/nacl/bindings/crypto_box.py b/lib/nacl/bindings/crypto_box.py new file mode 100644 index 0000000..da6e4cb --- /dev/null +++ b/lib/nacl/bindings/crypto_box.py @@ -0,0 +1,475 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Tuple + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +__all__ = ["crypto_box_keypair", "crypto_box"] + + +crypto_box_SECRETKEYBYTES: int = lib.crypto_box_secretkeybytes() +crypto_box_PUBLICKEYBYTES: int = lib.crypto_box_publickeybytes() +crypto_box_SEEDBYTES: int = lib.crypto_box_seedbytes() +crypto_box_NONCEBYTES: int = lib.crypto_box_noncebytes() +crypto_box_ZEROBYTES: int = lib.crypto_box_zerobytes() +crypto_box_BOXZEROBYTES: int = lib.crypto_box_boxzerobytes() +crypto_box_BEFORENMBYTES: int = lib.crypto_box_beforenmbytes() +crypto_box_SEALBYTES: int = lib.crypto_box_sealbytes() +crypto_box_MACBYTES: int = lib.crypto_box_macbytes() + + +def crypto_box_keypair() -> Tuple[bytes, bytes]: + """ + Returns a randomly generated public and secret key. + + :rtype: (bytes(public_key), bytes(secret_key)) + """ + pk = ffi.new("unsigned char[]", crypto_box_PUBLICKEYBYTES) + sk = ffi.new("unsigned char[]", crypto_box_SECRETKEYBYTES) + + rc = lib.crypto_box_keypair(pk, sk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ( + ffi.buffer(pk, crypto_box_PUBLICKEYBYTES)[:], + ffi.buffer(sk, crypto_box_SECRETKEYBYTES)[:], + ) + + +def crypto_box_seed_keypair(seed: bytes) -> Tuple[bytes, bytes]: + """ + Returns a (public, secret) key pair deterministically generated + from an input ``seed``. + + .. warning:: The seed **must** be high-entropy; therefore, + its generator **must** be a cryptographic quality + random function like, for example, :func:`~nacl.utils.random`. + + .. warning:: The seed **must** be protected and remain secret. + Anyone who knows the seed is really in possession of + the corresponding PrivateKey. + + + :param seed: bytes + :rtype: (bytes(public_key), bytes(secret_key)) + """ + ensure(isinstance(seed, bytes), "seed must be bytes", raising=TypeError) + + if len(seed) != crypto_box_SEEDBYTES: + raise exc.ValueError("Invalid seed") + + pk = ffi.new("unsigned char[]", crypto_box_PUBLICKEYBYTES) + sk = ffi.new("unsigned char[]", crypto_box_SECRETKEYBYTES) + + rc = lib.crypto_box_seed_keypair(pk, sk, seed) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ( + ffi.buffer(pk, crypto_box_PUBLICKEYBYTES)[:], + ffi.buffer(sk, crypto_box_SECRETKEYBYTES)[:], + ) + + +def crypto_box(message: bytes, nonce: bytes, pk: bytes, sk: bytes) -> bytes: + """ + Encrypts and returns a message ``message`` using the secret key ``sk``, + public key ``pk``, and the nonce ``nonce``. + + :param message: bytes + :param nonce: bytes + :param pk: bytes + :param sk: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce size") + + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + if len(sk) != crypto_box_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + padded = (b"\x00" * crypto_box_ZEROBYTES) + message + ciphertext = ffi.new("unsigned char[]", len(padded)) + + rc = lib.crypto_box(ciphertext, padded, len(padded), nonce, pk, sk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(ciphertext, len(padded))[crypto_box_BOXZEROBYTES:] + + +def crypto_box_open( + ciphertext: bytes, nonce: bytes, pk: bytes, sk: bytes +) -> bytes: + """ + Decrypts and returns an encrypted message ``ciphertext``, using the secret + key ``sk``, public key ``pk``, and the nonce ``nonce``. + + :param ciphertext: bytes + :param nonce: bytes + :param pk: bytes + :param sk: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce size") + + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + if len(sk) != crypto_box_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + padded = (b"\x00" * crypto_box_BOXZEROBYTES) + ciphertext + plaintext = ffi.new("unsigned char[]", len(padded)) + + res = lib.crypto_box_open(plaintext, padded, len(padded), nonce, pk, sk) + ensure( + res == 0, + "An error occurred trying to decrypt the message", + raising=exc.CryptoError, + ) + + return ffi.buffer(plaintext, len(padded))[crypto_box_ZEROBYTES:] + + +def crypto_box_beforenm(pk: bytes, sk: bytes) -> bytes: + """ + Computes and returns the shared key for the public key ``pk`` and the + secret key ``sk``. This can be used to speed up operations where the same + set of keys is going to be used multiple times. + + :param pk: bytes + :param sk: bytes + :rtype: bytes + """ + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + if len(sk) != crypto_box_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + k = ffi.new("unsigned char[]", crypto_box_BEFORENMBYTES) + + rc = lib.crypto_box_beforenm(k, pk, sk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(k, crypto_box_BEFORENMBYTES)[:] + + +def crypto_box_afternm(message: bytes, nonce: bytes, k: bytes) -> bytes: + """ + Encrypts and returns the message ``message`` using the shared key ``k`` and + the nonce ``nonce``. + + :param message: bytes + :param nonce: bytes + :param k: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + if len(k) != crypto_box_BEFORENMBYTES: + raise exc.ValueError("Invalid shared key") + + padded = b"\x00" * crypto_box_ZEROBYTES + message + ciphertext = ffi.new("unsigned char[]", len(padded)) + + rc = lib.crypto_box_afternm(ciphertext, padded, len(padded), nonce, k) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(ciphertext, len(padded))[crypto_box_BOXZEROBYTES:] + + +def crypto_box_open_afternm( + ciphertext: bytes, nonce: bytes, k: bytes +) -> bytes: + """ + Decrypts and returns the encrypted message ``ciphertext``, using the shared + key ``k`` and the nonce ``nonce``. + + :param ciphertext: bytes + :param nonce: bytes + :param k: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + if len(k) != crypto_box_BEFORENMBYTES: + raise exc.ValueError("Invalid shared key") + + padded = (b"\x00" * crypto_box_BOXZEROBYTES) + ciphertext + plaintext = ffi.new("unsigned char[]", len(padded)) + + res = lib.crypto_box_open_afternm(plaintext, padded, len(padded), nonce, k) + ensure( + res == 0, + "An error occurred trying to decrypt the message", + raising=exc.CryptoError, + ) + + return ffi.buffer(plaintext, len(padded))[crypto_box_ZEROBYTES:] + + +def crypto_box_easy( + message: bytes, nonce: bytes, pk: bytes, sk: bytes +) -> bytes: + """ + Encrypts and returns a message ``message`` using the secret key ``sk``, + public key ``pk``, and the nonce ``nonce``. + + :param message: bytes + :param nonce: bytes + :param pk: bytes + :param sk: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce size") + + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + if len(sk) != crypto_box_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + _mlen = len(message) + _clen = crypto_box_MACBYTES + _mlen + + ciphertext = ffi.new("unsigned char[]", _clen) + + rc = lib.crypto_box_easy(ciphertext, message, _mlen, nonce, pk, sk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(ciphertext, _clen)[:] + + +def crypto_box_open_easy( + ciphertext: bytes, nonce: bytes, pk: bytes, sk: bytes +) -> bytes: + """ + Decrypts and returns an encrypted message ``ciphertext``, using the secret + key ``sk``, public key ``pk``, and the nonce ``nonce``. + + :param ciphertext: bytes + :param nonce: bytes + :param pk: bytes + :param sk: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce size") + + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + if len(sk) != crypto_box_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + _clen = len(ciphertext) + + ensure( + _clen >= crypto_box_MACBYTES, + "Input ciphertext must be at least {} long".format( + crypto_box_MACBYTES + ), + raising=exc.TypeError, + ) + + _mlen = _clen - crypto_box_MACBYTES + + plaintext = ffi.new("unsigned char[]", max(1, _mlen)) + + res = lib.crypto_box_open_easy(plaintext, ciphertext, _clen, nonce, pk, sk) + ensure( + res == 0, + "An error occurred trying to decrypt the message", + raising=exc.CryptoError, + ) + + return ffi.buffer(plaintext, _mlen)[:] + + +def crypto_box_easy_afternm(message: bytes, nonce: bytes, k: bytes) -> bytes: + """ + Encrypts and returns the message ``message`` using the shared key ``k`` and + the nonce ``nonce``. + + :param message: bytes + :param nonce: bytes + :param k: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + if len(k) != crypto_box_BEFORENMBYTES: + raise exc.ValueError("Invalid shared key") + + _mlen = len(message) + _clen = crypto_box_MACBYTES + _mlen + + ciphertext = ffi.new("unsigned char[]", _clen) + + rc = lib.crypto_box_easy_afternm(ciphertext, message, _mlen, nonce, k) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(ciphertext, _clen)[:] + + +def crypto_box_open_easy_afternm( + ciphertext: bytes, nonce: bytes, k: bytes +) -> bytes: + """ + Decrypts and returns the encrypted message ``ciphertext``, using the shared + key ``k`` and the nonce ``nonce``. + + :param ciphertext: bytes + :param nonce: bytes + :param k: bytes + :rtype: bytes + """ + if len(nonce) != crypto_box_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + if len(k) != crypto_box_BEFORENMBYTES: + raise exc.ValueError("Invalid shared key") + + _clen = len(ciphertext) + + ensure( + _clen >= crypto_box_MACBYTES, + "Input ciphertext must be at least {} long".format( + crypto_box_MACBYTES + ), + raising=exc.TypeError, + ) + + _mlen = _clen - crypto_box_MACBYTES + + plaintext = ffi.new("unsigned char[]", max(1, _mlen)) + + res = lib.crypto_box_open_easy_afternm( + plaintext, ciphertext, _clen, nonce, k + ) + ensure( + res == 0, + "An error occurred trying to decrypt the message", + raising=exc.CryptoError, + ) + + return ffi.buffer(plaintext, _mlen)[:] + + +def crypto_box_seal(message: bytes, pk: bytes) -> bytes: + """ + Encrypts and returns a message ``message`` using an ephemeral secret key + and the public key ``pk``. + The ephemeral public key, which is embedded in the sealed box, is also + used, in combination with ``pk``, to derive the nonce needed for the + underlying box construct. + + :param message: bytes + :param pk: bytes + :rtype: bytes + + .. versionadded:: 1.2 + """ + ensure( + isinstance(message, bytes), + "input message must be bytes", + raising=TypeError, + ) + + ensure( + isinstance(pk, bytes), "public key must be bytes", raising=TypeError + ) + + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + _mlen = len(message) + _clen = crypto_box_SEALBYTES + _mlen + + ciphertext = ffi.new("unsigned char[]", _clen) + + rc = lib.crypto_box_seal(ciphertext, message, _mlen, pk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(ciphertext, _clen)[:] + + +def crypto_box_seal_open(ciphertext: bytes, pk: bytes, sk: bytes) -> bytes: + """ + Decrypts and returns an encrypted message ``ciphertext``, using the + recipent's secret key ``sk`` and the sender's ephemeral public key + embedded in the sealed box. The box construct nonce is derived from + the recipient's public key ``pk`` and the sender's public key. + + :param ciphertext: bytes + :param pk: bytes + :param sk: bytes + :rtype: bytes + + .. versionadded:: 1.2 + """ + ensure( + isinstance(ciphertext, bytes), + "input ciphertext must be bytes", + raising=TypeError, + ) + + ensure( + isinstance(pk, bytes), "public key must be bytes", raising=TypeError + ) + + ensure( + isinstance(sk, bytes), "secret key must be bytes", raising=TypeError + ) + + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise exc.ValueError("Invalid public key") + + if len(sk) != crypto_box_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + _clen = len(ciphertext) + + ensure( + _clen >= crypto_box_SEALBYTES, + ("Input ciphertext must be at least {} long").format( + crypto_box_SEALBYTES + ), + raising=exc.TypeError, + ) + + _mlen = _clen - crypto_box_SEALBYTES + + # zero-length malloc results are implementation.dependent + plaintext = ffi.new("unsigned char[]", max(1, _mlen)) + + res = lib.crypto_box_seal_open(plaintext, ciphertext, _clen, pk, sk) + ensure( + res == 0, + "An error occurred trying to decrypt the message", + raising=exc.CryptoError, + ) + + return ffi.buffer(plaintext, _mlen)[:] diff --git a/lib/nacl/bindings/crypto_core.py b/lib/nacl/bindings/crypto_core.py new file mode 100644 index 0000000..e64a064 --- /dev/null +++ b/lib/nacl/bindings/crypto_core.py @@ -0,0 +1,449 @@ +# Copyright 2018 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +has_crypto_core_ed25519 = bool(lib.PYNACL_HAS_CRYPTO_CORE_ED25519) + +crypto_core_ed25519_BYTES = 0 +crypto_core_ed25519_SCALARBYTES = 0 +crypto_core_ed25519_NONREDUCEDSCALARBYTES = 0 + +if has_crypto_core_ed25519: + crypto_core_ed25519_BYTES = lib.crypto_core_ed25519_bytes() + crypto_core_ed25519_SCALARBYTES = lib.crypto_core_ed25519_scalarbytes() + crypto_core_ed25519_NONREDUCEDSCALARBYTES = ( + lib.crypto_core_ed25519_nonreducedscalarbytes() + ) + + +def crypto_core_ed25519_is_valid_point(p: bytes) -> bool: + """ + Check if ``p`` represents a point on the edwards25519 curve, in canonical + form, on the main subgroup, and that the point doesn't have a small order. + + :param p: a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type p: bytes + :return: point validity + :rtype: bool + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(p, bytes) and len(p) == crypto_core_ed25519_BYTES, + "Point must be a crypto_core_ed25519_BYTES long bytes sequence", + raising=exc.TypeError, + ) + + rc = lib.crypto_core_ed25519_is_valid_point(p) + return rc == 1 + + +def crypto_core_ed25519_from_uniform(r: bytes) -> bytes: + """ + Maps a 32 bytes vector ``r`` to a point. The point is guaranteed to be on the main subgroup. + This function directly exposes the Elligator 2 map, uses the high bit to set + the sign of the X coordinate, and the resulting point is multiplied by the cofactor. + + :param r: a :py:data:`.crypto_core_ed25519_BYTES` long bytes + sequence representing arbitrary data + :type r: bytes + :return: a point on the edwards25519 curve main order subgroup, represented as a + :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(r, bytes) and len(r) == crypto_core_ed25519_BYTES, + "Integer r must be a {} long bytes sequence".format( + "crypto_core_ed25519_BYTES" + ), + raising=exc.TypeError, + ) + + p = ffi.new("unsigned char[]", crypto_core_ed25519_BYTES) + + rc = lib.crypto_core_ed25519_from_uniform(p, r) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(p, crypto_core_ed25519_BYTES)[:] + + +def crypto_core_ed25519_add(p: bytes, q: bytes) -> bytes: + """ + Add two points on the edwards25519 curve. + + :param p: a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type p: bytes + :param q: a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type q: bytes + :return: a point on the edwards25519 curve represented as + a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(p, bytes) + and isinstance(q, bytes) + and len(p) == crypto_core_ed25519_BYTES + and len(q) == crypto_core_ed25519_BYTES, + "Each point must be a {} long bytes sequence".format( + "crypto_core_ed25519_BYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_BYTES) + + rc = lib.crypto_core_ed25519_add(r, p, q) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(r, crypto_core_ed25519_BYTES)[:] + + +def crypto_core_ed25519_sub(p: bytes, q: bytes) -> bytes: + """ + Subtract a point from another on the edwards25519 curve. + + :param p: a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type p: bytes + :param q: a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type q: bytes + :return: a point on the edwards25519 curve represented as + a :py:data:`.crypto_core_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(p, bytes) + and isinstance(q, bytes) + and len(p) == crypto_core_ed25519_BYTES + and len(q) == crypto_core_ed25519_BYTES, + "Each point must be a {} long bytes sequence".format( + "crypto_core_ed25519_BYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_BYTES) + + rc = lib.crypto_core_ed25519_sub(r, p, q) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(r, crypto_core_ed25519_BYTES)[:] + + +def crypto_core_ed25519_scalar_invert(s: bytes) -> bytes: + """ + Return the multiplicative inverse of integer ``s`` modulo ``L``, + i.e an integer ``i`` such that ``s * i = 1 (mod L)``, where ``L`` + is the order of the main subgroup. + + Raises a ``exc.RuntimeError`` if ``s`` is the integer zero. + + :param s: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type s: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(s, bytes) and len(s) == crypto_core_ed25519_SCALARBYTES, + "Integer s must be a {} long bytes sequence".format( + "crypto_core_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + rc = lib.crypto_core_ed25519_scalar_invert(r, s) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] + + +def crypto_core_ed25519_scalar_negate(s: bytes) -> bytes: + """ + Return the integer ``n`` such that ``s + n = 0 (mod L)``, where ``L`` + is the order of the main subgroup. + + :param s: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type s: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(s, bytes) and len(s) == crypto_core_ed25519_SCALARBYTES, + "Integer s must be a {} long bytes sequence".format( + "crypto_core_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + lib.crypto_core_ed25519_scalar_negate(r, s) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] + + +def crypto_core_ed25519_scalar_complement(s: bytes) -> bytes: + """ + Return the complement of integer ``s`` modulo ``L``, i.e. an integer + ``c`` such that ``s + c = 1 (mod L)``, where ``L`` is the order of + the main subgroup. + + :param s: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type s: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(s, bytes) and len(s) == crypto_core_ed25519_SCALARBYTES, + "Integer s must be a {} long bytes sequence".format( + "crypto_core_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + lib.crypto_core_ed25519_scalar_complement(r, s) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] + + +def crypto_core_ed25519_scalar_add(p: bytes, q: bytes) -> bytes: + """ + Add integers ``p`` and ``q`` modulo ``L``, where ``L`` is the order of + the main subgroup. + + :param p: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type p: bytes + :param q: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type q: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(p, bytes) + and isinstance(q, bytes) + and len(p) == crypto_core_ed25519_SCALARBYTES + and len(q) == crypto_core_ed25519_SCALARBYTES, + "Each integer must be a {} long bytes sequence".format( + "crypto_core_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + lib.crypto_core_ed25519_scalar_add(r, p, q) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] + + +def crypto_core_ed25519_scalar_sub(p: bytes, q: bytes) -> bytes: + """ + Subtract integers ``p`` and ``q`` modulo ``L``, where ``L`` is the + order of the main subgroup. + + :param p: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type p: bytes + :param q: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type q: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(p, bytes) + and isinstance(q, bytes) + and len(p) == crypto_core_ed25519_SCALARBYTES + and len(q) == crypto_core_ed25519_SCALARBYTES, + "Each integer must be a {} long bytes sequence".format( + "crypto_core_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + lib.crypto_core_ed25519_scalar_sub(r, p, q) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] + + +def crypto_core_ed25519_scalar_mul(p: bytes, q: bytes) -> bytes: + """ + Multiply integers ``p`` and ``q`` modulo ``L``, where ``L`` is the + order of the main subgroup. + + :param p: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type p: bytes + :param q: a :py:data:`.crypto_core_ed25519_SCALARBYTES` + long bytes sequence representing an integer + :type q: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(p, bytes) + and isinstance(q, bytes) + and len(p) == crypto_core_ed25519_SCALARBYTES + and len(q) == crypto_core_ed25519_SCALARBYTES, + "Each integer must be a {} long bytes sequence".format( + "crypto_core_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + lib.crypto_core_ed25519_scalar_mul(r, p, q) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] + + +def crypto_core_ed25519_scalar_reduce(s: bytes) -> bytes: + """ + Reduce integer ``s`` to ``s`` modulo ``L``, where ``L`` is the order + of the main subgroup. + + :param s: a :py:data:`.crypto_core_ed25519_NONREDUCEDSCALARBYTES` + long bytes sequence representing an integer + :type s: bytes + :return: an integer represented as a + :py:data:`.crypto_core_ed25519_SCALARBYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_core_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(s, bytes) + and len(s) == crypto_core_ed25519_NONREDUCEDSCALARBYTES, + "Integer s must be a {} long bytes sequence".format( + "crypto_core_ed25519_NONREDUCEDSCALARBYTES" + ), + raising=exc.TypeError, + ) + + r = ffi.new("unsigned char[]", crypto_core_ed25519_SCALARBYTES) + + lib.crypto_core_ed25519_scalar_reduce(r, s) + + return ffi.buffer(r, crypto_core_ed25519_SCALARBYTES)[:] diff --git a/lib/nacl/bindings/crypto_generichash.py b/lib/nacl/bindings/crypto_generichash.py new file mode 100644 index 0000000..6ab385a --- /dev/null +++ b/lib/nacl/bindings/crypto_generichash.py @@ -0,0 +1,281 @@ +# Copyright 2013-2019 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import NoReturn, TypeVar + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +crypto_generichash_BYTES: int = lib.crypto_generichash_blake2b_bytes() +crypto_generichash_BYTES_MIN: int = lib.crypto_generichash_blake2b_bytes_min() +crypto_generichash_BYTES_MAX: int = lib.crypto_generichash_blake2b_bytes_max() +crypto_generichash_KEYBYTES: int = lib.crypto_generichash_blake2b_keybytes() +crypto_generichash_KEYBYTES_MIN: int = ( + lib.crypto_generichash_blake2b_keybytes_min() +) +crypto_generichash_KEYBYTES_MAX: int = ( + lib.crypto_generichash_blake2b_keybytes_max() +) +crypto_generichash_SALTBYTES: int = lib.crypto_generichash_blake2b_saltbytes() +crypto_generichash_PERSONALBYTES: int = ( + lib.crypto_generichash_blake2b_personalbytes() +) +crypto_generichash_STATEBYTES: int = lib.crypto_generichash_statebytes() + +_OVERLONG = "{0} length greater than {1} bytes" +_TOOBIG = "{0} greater than {1}" + + +def _checkparams( + digest_size: int, key: bytes, salt: bytes, person: bytes +) -> None: + """Check hash parameters""" + ensure( + isinstance(key, bytes), + "Key must be a bytes sequence", + raising=exc.TypeError, + ) + + ensure( + isinstance(salt, bytes), + "Salt must be a bytes sequence", + raising=exc.TypeError, + ) + + ensure( + isinstance(person, bytes), + "Person must be a bytes sequence", + raising=exc.TypeError, + ) + + ensure( + isinstance(digest_size, int), + "Digest size must be an integer number", + raising=exc.TypeError, + ) + + ensure( + digest_size <= crypto_generichash_BYTES_MAX, + _TOOBIG.format("Digest_size", crypto_generichash_BYTES_MAX), + raising=exc.ValueError, + ) + + ensure( + len(key) <= crypto_generichash_KEYBYTES_MAX, + _OVERLONG.format("Key", crypto_generichash_KEYBYTES_MAX), + raising=exc.ValueError, + ) + + ensure( + len(salt) <= crypto_generichash_SALTBYTES, + _OVERLONG.format("Salt", crypto_generichash_SALTBYTES), + raising=exc.ValueError, + ) + + ensure( + len(person) <= crypto_generichash_PERSONALBYTES, + _OVERLONG.format("Person", crypto_generichash_PERSONALBYTES), + raising=exc.ValueError, + ) + + +def generichash_blake2b_salt_personal( + data: bytes, + digest_size: int = crypto_generichash_BYTES, + key: bytes = b"", + salt: bytes = b"", + person: bytes = b"", +) -> bytes: + """One shot hash interface + + :param data: the input data to the hash function + :type data: bytes + :param digest_size: must be at most + :py:data:`.crypto_generichash_BYTES_MAX`; + the default digest size is + :py:data:`.crypto_generichash_BYTES` + :type digest_size: int + :param key: must be at most + :py:data:`.crypto_generichash_KEYBYTES_MAX` long + :type key: bytes + :param salt: must be at most + :py:data:`.crypto_generichash_SALTBYTES` long; + will be zero-padded if needed + :type salt: bytes + :param person: must be at most + :py:data:`.crypto_generichash_PERSONALBYTES` long: + will be zero-padded if needed + :type person: bytes + :return: digest_size long digest + :rtype: bytes + """ + + _checkparams(digest_size, key, salt, person) + + ensure( + isinstance(data, bytes), + "Input data must be a bytes sequence", + raising=exc.TypeError, + ) + + digest = ffi.new("unsigned char[]", digest_size) + + # both _salt and _personal must be zero-padded to the correct length + _salt = ffi.new("unsigned char []", crypto_generichash_SALTBYTES) + _person = ffi.new("unsigned char []", crypto_generichash_PERSONALBYTES) + + ffi.memmove(_salt, salt, len(salt)) + ffi.memmove(_person, person, len(person)) + + rc = lib.crypto_generichash_blake2b_salt_personal( + digest, digest_size, data, len(data), key, len(key), _salt, _person + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + return ffi.buffer(digest, digest_size)[:] + + +_Blake2State = TypeVar("_Blake2State", bound="Blake2State") + + +class Blake2State: + """ + Python-level wrapper for the crypto_generichash_blake2b state buffer + """ + + __slots__ = ["_statebuf", "digest_size"] + + def __init__(self, digest_size: int): + self._statebuf = ffi.new( + "unsigned char[]", crypto_generichash_STATEBYTES + ) + self.digest_size = digest_size + + def __reduce__(self) -> NoReturn: + """ + Raise the same exception as hashlib's blake implementation + on copy.copy() + """ + raise TypeError( + "can't pickle {} objects".format(self.__class__.__name__) + ) + + def copy(self: _Blake2State) -> _Blake2State: + _st = self.__class__(self.digest_size) + ffi.memmove( + _st._statebuf, self._statebuf, crypto_generichash_STATEBYTES + ) + return _st + + +def generichash_blake2b_init( + key: bytes = b"", + salt: bytes = b"", + person: bytes = b"", + digest_size: int = crypto_generichash_BYTES, +) -> Blake2State: + """ + Create a new initialized blake2b hash state + + :param key: must be at most + :py:data:`.crypto_generichash_KEYBYTES_MAX` long + :type key: bytes + :param salt: must be at most + :py:data:`.crypto_generichash_SALTBYTES` long; + will be zero-padded if needed + :type salt: bytes + :param person: must be at most + :py:data:`.crypto_generichash_PERSONALBYTES` long: + will be zero-padded if needed + :type person: bytes + :param digest_size: must be at most + :py:data:`.crypto_generichash_BYTES_MAX`; + the default digest size is + :py:data:`.crypto_generichash_BYTES` + :type digest_size: int + :return: a initialized :py:class:`.Blake2State` + :rtype: object + """ + + _checkparams(digest_size, key, salt, person) + + state = Blake2State(digest_size) + + # both _salt and _personal must be zero-padded to the correct length + _salt = ffi.new("unsigned char []", crypto_generichash_SALTBYTES) + _person = ffi.new("unsigned char []", crypto_generichash_PERSONALBYTES) + + ffi.memmove(_salt, salt, len(salt)) + ffi.memmove(_person, person, len(person)) + + rc = lib.crypto_generichash_blake2b_init_salt_personal( + state._statebuf, key, len(key), digest_size, _salt, _person + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + return state + + +def generichash_blake2b_update(state: Blake2State, data: bytes) -> None: + """Update the blake2b hash state + + :param state: a initialized Blake2bState object as returned from + :py:func:`.crypto_generichash_blake2b_init` + :type state: :py:class:`.Blake2State` + :param data: + :type data: bytes + """ + + ensure( + isinstance(state, Blake2State), + "State must be a Blake2State object", + raising=exc.TypeError, + ) + + ensure( + isinstance(data, bytes), + "Input data must be a bytes sequence", + raising=exc.TypeError, + ) + + rc = lib.crypto_generichash_blake2b_update( + state._statebuf, data, len(data) + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + +def generichash_blake2b_final(state: Blake2State) -> bytes: + """Finalize the blake2b hash state and return the digest. + + :param state: a initialized Blake2bState object as returned from + :py:func:`.crypto_generichash_blake2b_init` + :type state: :py:class:`.Blake2State` + :return: the blake2 digest of the passed-in data stream + :rtype: bytes + """ + + ensure( + isinstance(state, Blake2State), + "State must be a Blake2State object", + raising=exc.TypeError, + ) + + _digest = ffi.new("unsigned char[]", crypto_generichash_BYTES_MAX) + rc = lib.crypto_generichash_blake2b_final( + state._statebuf, _digest, state.digest_size + ) + + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + return ffi.buffer(_digest, state.digest_size)[:] diff --git a/lib/nacl/bindings/crypto_hash.py b/lib/nacl/bindings/crypto_hash.py new file mode 100644 index 0000000..2bab399 --- /dev/null +++ b/lib/nacl/bindings/crypto_hash.py @@ -0,0 +1,63 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +# crypto_hash_BYTES = lib.crypto_hash_bytes() +crypto_hash_BYTES: int = lib.crypto_hash_sha512_bytes() +crypto_hash_sha256_BYTES: int = lib.crypto_hash_sha256_bytes() +crypto_hash_sha512_BYTES: int = lib.crypto_hash_sha512_bytes() + + +def crypto_hash(message: bytes) -> bytes: + """ + Hashes and returns the message ``message``. + + :param message: bytes + :rtype: bytes + """ + digest = ffi.new("unsigned char[]", crypto_hash_BYTES) + rc = lib.crypto_hash(digest, message, len(message)) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + return ffi.buffer(digest, crypto_hash_BYTES)[:] + + +def crypto_hash_sha256(message: bytes) -> bytes: + """ + Hashes and returns the message ``message``. + + :param message: bytes + :rtype: bytes + """ + digest = ffi.new("unsigned char[]", crypto_hash_sha256_BYTES) + rc = lib.crypto_hash_sha256(digest, message, len(message)) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + return ffi.buffer(digest, crypto_hash_sha256_BYTES)[:] + + +def crypto_hash_sha512(message: bytes) -> bytes: + """ + Hashes and returns the message ``message``. + + :param message: bytes + :rtype: bytes + """ + digest = ffi.new("unsigned char[]", crypto_hash_sha512_BYTES) + rc = lib.crypto_hash_sha512(digest, message, len(message)) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + return ffi.buffer(digest, crypto_hash_sha512_BYTES)[:] diff --git a/lib/nacl/bindings/crypto_kx.py b/lib/nacl/bindings/crypto_kx.py new file mode 100644 index 0000000..3c649e4 --- /dev/null +++ b/lib/nacl/bindings/crypto_kx.py @@ -0,0 +1,200 @@ +# Copyright 2018 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Tuple + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + +__all__ = [ + "crypto_kx_keypair", + "crypto_kx_client_session_keys", + "crypto_kx_server_session_keys", + "crypto_kx_PUBLIC_KEY_BYTES", + "crypto_kx_SECRET_KEY_BYTES", + "crypto_kx_SEED_BYTES", + "crypto_kx_SESSION_KEY_BYTES", +] + +""" +Implementations of client, server key exchange +""" +crypto_kx_PUBLIC_KEY_BYTES: int = lib.crypto_kx_publickeybytes() +crypto_kx_SECRET_KEY_BYTES: int = lib.crypto_kx_secretkeybytes() +crypto_kx_SEED_BYTES: int = lib.crypto_kx_seedbytes() +crypto_kx_SESSION_KEY_BYTES: int = lib.crypto_kx_sessionkeybytes() + + +def crypto_kx_keypair() -> Tuple[bytes, bytes]: + """ + Generate a key pair. + This is a duplicate crypto_box_keypair, but + is included for api consistency. + :return: (public_key, secret_key) + :rtype: (bytes, bytes) + """ + public_key = ffi.new("unsigned char[]", crypto_kx_PUBLIC_KEY_BYTES) + secret_key = ffi.new("unsigned char[]", crypto_kx_SECRET_KEY_BYTES) + res = lib.crypto_kx_keypair(public_key, secret_key) + ensure(res == 0, "Key generation failed.", raising=exc.CryptoError) + + return ( + ffi.buffer(public_key, crypto_kx_PUBLIC_KEY_BYTES)[:], + ffi.buffer(secret_key, crypto_kx_SECRET_KEY_BYTES)[:], + ) + + +def crypto_kx_seed_keypair(seed: bytes) -> Tuple[bytes, bytes]: + """ + Generate a key pair with a given seed. + This is functionally the same as crypto_box_seed_keypair, however + it uses the blake2b hash primitive instead of sha512. + It is included mainly for api consistency when using crypto_kx. + :param seed: random seed + :type seed: bytes + :return: (public_key, secret_key) + :rtype: (bytes, bytes) + """ + public_key = ffi.new("unsigned char[]", crypto_kx_PUBLIC_KEY_BYTES) + secret_key = ffi.new("unsigned char[]", crypto_kx_SECRET_KEY_BYTES) + ensure( + isinstance(seed, bytes) and len(seed) == crypto_kx_SEED_BYTES, + "Seed must be a {} byte long bytes sequence".format( + crypto_kx_SEED_BYTES + ), + raising=exc.TypeError, + ) + res = lib.crypto_kx_seed_keypair(public_key, secret_key, seed) + ensure(res == 0, "Key generation failed.", raising=exc.CryptoError) + + return ( + ffi.buffer(public_key, crypto_kx_PUBLIC_KEY_BYTES)[:], + ffi.buffer(secret_key, crypto_kx_SECRET_KEY_BYTES)[:], + ) + + +def crypto_kx_client_session_keys( + client_public_key: bytes, + client_secret_key: bytes, + server_public_key: bytes, +) -> Tuple[bytes, bytes]: + """ + Generate session keys for the client. + :param client_public_key: + :type client_public_key: bytes + :param client_secret_key: + :type client_secret_key: bytes + :param server_public_key: + :type server_public_key: bytes + :return: (rx_key, tx_key) + :rtype: (bytes, bytes) + """ + ensure( + isinstance(client_public_key, bytes) + and len(client_public_key) == crypto_kx_PUBLIC_KEY_BYTES, + "Client public key must be a {} bytes long bytes sequence".format( + crypto_kx_PUBLIC_KEY_BYTES + ), + raising=exc.TypeError, + ) + ensure( + isinstance(client_secret_key, bytes) + and len(client_secret_key) == crypto_kx_SECRET_KEY_BYTES, + "Client secret key must be a {} bytes long bytes sequence".format( + crypto_kx_PUBLIC_KEY_BYTES + ), + raising=exc.TypeError, + ) + ensure( + isinstance(server_public_key, bytes) + and len(server_public_key) == crypto_kx_PUBLIC_KEY_BYTES, + "Server public key must be a {} bytes long bytes sequence".format( + crypto_kx_PUBLIC_KEY_BYTES + ), + raising=exc.TypeError, + ) + + rx_key = ffi.new("unsigned char[]", crypto_kx_SESSION_KEY_BYTES) + tx_key = ffi.new("unsigned char[]", crypto_kx_SESSION_KEY_BYTES) + res = lib.crypto_kx_client_session_keys( + rx_key, tx_key, client_public_key, client_secret_key, server_public_key + ) + ensure( + res == 0, + "Client session key generation failed.", + raising=exc.CryptoError, + ) + + return ( + ffi.buffer(rx_key, crypto_kx_SESSION_KEY_BYTES)[:], + ffi.buffer(tx_key, crypto_kx_SESSION_KEY_BYTES)[:], + ) + + +def crypto_kx_server_session_keys( + server_public_key: bytes, + server_secret_key: bytes, + client_public_key: bytes, +) -> Tuple[bytes, bytes]: + """ + Generate session keys for the server. + :param server_public_key: + :type server_public_key: bytes + :param server_secret_key: + :type server_secret_key: bytes + :param client_public_key: + :type client_public_key: bytes + :return: (rx_key, tx_key) + :rtype: (bytes, bytes) + """ + ensure( + isinstance(server_public_key, bytes) + and len(server_public_key) == crypto_kx_PUBLIC_KEY_BYTES, + "Server public key must be a {} bytes long bytes sequence".format( + crypto_kx_PUBLIC_KEY_BYTES + ), + raising=exc.TypeError, + ) + ensure( + isinstance(server_secret_key, bytes) + and len(server_secret_key) == crypto_kx_SECRET_KEY_BYTES, + "Server secret key must be a {} bytes long bytes sequence".format( + crypto_kx_PUBLIC_KEY_BYTES + ), + raising=exc.TypeError, + ) + ensure( + isinstance(client_public_key, bytes) + and len(client_public_key) == crypto_kx_PUBLIC_KEY_BYTES, + "Client public key must be a {} bytes long bytes sequence".format( + crypto_kx_PUBLIC_KEY_BYTES + ), + raising=exc.TypeError, + ) + + rx_key = ffi.new("unsigned char[]", crypto_kx_SESSION_KEY_BYTES) + tx_key = ffi.new("unsigned char[]", crypto_kx_SESSION_KEY_BYTES) + res = lib.crypto_kx_server_session_keys( + rx_key, tx_key, server_public_key, server_secret_key, client_public_key + ) + ensure( + res == 0, + "Server session key generation failed.", + raising=exc.CryptoError, + ) + + return ( + ffi.buffer(rx_key, crypto_kx_SESSION_KEY_BYTES)[:], + ffi.buffer(tx_key, crypto_kx_SESSION_KEY_BYTES)[:], + ) diff --git a/lib/nacl/bindings/crypto_pwhash.py b/lib/nacl/bindings/crypto_pwhash.py new file mode 100644 index 0000000..7f62360 --- /dev/null +++ b/lib/nacl/bindings/crypto_pwhash.py @@ -0,0 +1,599 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from typing import Tuple + +import nacl.exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +has_crypto_pwhash_scryptsalsa208sha256 = bool( + lib.PYNACL_HAS_CRYPTO_PWHASH_SCRYPTSALSA208SHA256 +) + +crypto_pwhash_scryptsalsa208sha256_STRPREFIX = b"" +crypto_pwhash_scryptsalsa208sha256_SALTBYTES = 0 +crypto_pwhash_scryptsalsa208sha256_STRBYTES = 0 +crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN = 0 +crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX = 0 +crypto_pwhash_scryptsalsa208sha256_BYTES_MIN = 0 +crypto_pwhash_scryptsalsa208sha256_BYTES_MAX = 0 +crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN = 0 +crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX = 0 +crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN = 0 +crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX = 0 +crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE = 0 +crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE = 0 +crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE = 0 +crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE = 0 + +if has_crypto_pwhash_scryptsalsa208sha256: + crypto_pwhash_scryptsalsa208sha256_STRPREFIX = ffi.string( + ffi.cast("char *", lib.crypto_pwhash_scryptsalsa208sha256_strprefix()) + )[:] + crypto_pwhash_scryptsalsa208sha256_SALTBYTES = ( + lib.crypto_pwhash_scryptsalsa208sha256_saltbytes() + ) + crypto_pwhash_scryptsalsa208sha256_STRBYTES = ( + lib.crypto_pwhash_scryptsalsa208sha256_strbytes() + ) + crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN = ( + lib.crypto_pwhash_scryptsalsa208sha256_passwd_min() + ) + crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX = ( + lib.crypto_pwhash_scryptsalsa208sha256_passwd_max() + ) + crypto_pwhash_scryptsalsa208sha256_BYTES_MIN = ( + lib.crypto_pwhash_scryptsalsa208sha256_bytes_min() + ) + crypto_pwhash_scryptsalsa208sha256_BYTES_MAX = ( + lib.crypto_pwhash_scryptsalsa208sha256_bytes_max() + ) + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN = ( + lib.crypto_pwhash_scryptsalsa208sha256_memlimit_min() + ) + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX = ( + lib.crypto_pwhash_scryptsalsa208sha256_memlimit_max() + ) + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN = ( + lib.crypto_pwhash_scryptsalsa208sha256_opslimit_min() + ) + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX = ( + lib.crypto_pwhash_scryptsalsa208sha256_opslimit_max() + ) + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE = ( + lib.crypto_pwhash_scryptsalsa208sha256_opslimit_interactive() + ) + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE = ( + lib.crypto_pwhash_scryptsalsa208sha256_memlimit_interactive() + ) + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE = ( + lib.crypto_pwhash_scryptsalsa208sha256_opslimit_sensitive() + ) + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE = ( + lib.crypto_pwhash_scryptsalsa208sha256_memlimit_sensitive() + ) + +crypto_pwhash_ALG_ARGON2I13: int = lib.crypto_pwhash_alg_argon2i13() +crypto_pwhash_ALG_ARGON2ID13: int = lib.crypto_pwhash_alg_argon2id13() +crypto_pwhash_ALG_DEFAULT: int = lib.crypto_pwhash_alg_default() + +crypto_pwhash_SALTBYTES: int = lib.crypto_pwhash_saltbytes() +crypto_pwhash_STRBYTES: int = lib.crypto_pwhash_strbytes() + +crypto_pwhash_PASSWD_MIN: int = lib.crypto_pwhash_passwd_min() +crypto_pwhash_PASSWD_MAX: int = lib.crypto_pwhash_passwd_max() +crypto_pwhash_BYTES_MIN: int = lib.crypto_pwhash_bytes_min() +crypto_pwhash_BYTES_MAX: int = lib.crypto_pwhash_bytes_max() + +crypto_pwhash_argon2i_STRPREFIX: bytes = ffi.string( + ffi.cast("char *", lib.crypto_pwhash_argon2i_strprefix()) +)[:] +crypto_pwhash_argon2i_MEMLIMIT_MIN: int = ( + lib.crypto_pwhash_argon2i_memlimit_min() +) +crypto_pwhash_argon2i_MEMLIMIT_MAX: int = ( + lib.crypto_pwhash_argon2i_memlimit_max() +) +crypto_pwhash_argon2i_OPSLIMIT_MIN: int = ( + lib.crypto_pwhash_argon2i_opslimit_min() +) +crypto_pwhash_argon2i_OPSLIMIT_MAX: int = ( + lib.crypto_pwhash_argon2i_opslimit_max() +) +crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE: int = ( + lib.crypto_pwhash_argon2i_opslimit_interactive() +) +crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE: int = ( + lib.crypto_pwhash_argon2i_memlimit_interactive() +) +crypto_pwhash_argon2i_OPSLIMIT_MODERATE: int = ( + lib.crypto_pwhash_argon2i_opslimit_moderate() +) +crypto_pwhash_argon2i_MEMLIMIT_MODERATE: int = ( + lib.crypto_pwhash_argon2i_memlimit_moderate() +) +crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE: int = ( + lib.crypto_pwhash_argon2i_opslimit_sensitive() +) +crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE: int = ( + lib.crypto_pwhash_argon2i_memlimit_sensitive() +) + +crypto_pwhash_argon2id_STRPREFIX: bytes = ffi.string( + ffi.cast("char *", lib.crypto_pwhash_argon2id_strprefix()) +)[:] +crypto_pwhash_argon2id_MEMLIMIT_MIN: int = ( + lib.crypto_pwhash_argon2id_memlimit_min() +) +crypto_pwhash_argon2id_MEMLIMIT_MAX: int = ( + lib.crypto_pwhash_argon2id_memlimit_max() +) +crypto_pwhash_argon2id_OPSLIMIT_MIN: int = ( + lib.crypto_pwhash_argon2id_opslimit_min() +) +crypto_pwhash_argon2id_OPSLIMIT_MAX: int = ( + lib.crypto_pwhash_argon2id_opslimit_max() +) +crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE: int = ( + lib.crypto_pwhash_argon2id_opslimit_interactive() +) +crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE: int = ( + lib.crypto_pwhash_argon2id_memlimit_interactive() +) +crypto_pwhash_argon2id_OPSLIMIT_MODERATE: int = ( + lib.crypto_pwhash_argon2id_opslimit_moderate() +) +crypto_pwhash_argon2id_MEMLIMIT_MODERATE: int = ( + lib.crypto_pwhash_argon2id_memlimit_moderate() +) +crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE: int = ( + lib.crypto_pwhash_argon2id_opslimit_sensitive() +) +crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE: int = ( + lib.crypto_pwhash_argon2id_memlimit_sensitive() +) + +SCRYPT_OPSLIMIT_INTERACTIVE = ( + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE +) +SCRYPT_MEMLIMIT_INTERACTIVE = ( + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE +) +SCRYPT_OPSLIMIT_SENSITIVE = ( + crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE +) +SCRYPT_MEMLIMIT_SENSITIVE = ( + crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE +) +SCRYPT_SALTBYTES = crypto_pwhash_scryptsalsa208sha256_SALTBYTES +SCRYPT_STRBYTES = crypto_pwhash_scryptsalsa208sha256_STRBYTES + +SCRYPT_PR_MAX = (1 << 30) - 1 +LOG2_UINT64_MAX = 63 +UINT64_MAX = (1 << 64) - 1 +SCRYPT_MAX_MEM = 32 * (1024 * 1024) + + +def _check_memory_occupation( + n: int, r: int, p: int, maxmem: int = SCRYPT_MAX_MEM +) -> None: + ensure(r != 0, "Invalid block size", raising=exc.ValueError) + + ensure(p != 0, "Invalid parallelization factor", raising=exc.ValueError) + + ensure( + (n & (n - 1)) == 0, + "Cost factor must be a power of 2", + raising=exc.ValueError, + ) + + ensure(n > 1, "Cost factor must be at least 2", raising=exc.ValueError) + + ensure( + p <= SCRYPT_PR_MAX / r, + "p*r is greater than {}".format(SCRYPT_PR_MAX), + raising=exc.ValueError, + ) + + ensure(n < (1 << (16 * r)), raising=exc.ValueError) + + Blen = p * 128 * r + + i = UINT64_MAX / 128 + + ensure(n + 2 <= i / r, raising=exc.ValueError) + + Vlen = 32 * r * (n + 2) * 4 + + ensure(Blen <= UINT64_MAX - Vlen, raising=exc.ValueError) + + ensure(Blen <= sys.maxsize - Vlen, raising=exc.ValueError) + + ensure( + Blen + Vlen <= maxmem, + "Memory limit would be exceeded with the chosen n, r, p", + raising=exc.ValueError, + ) + + +def nacl_bindings_pick_scrypt_params( + opslimit: int, memlimit: int +) -> Tuple[int, int, int]: + """Python implementation of libsodium's pickparams""" + + if opslimit < 32768: + opslimit = 32768 + + r = 8 + + if opslimit < (memlimit // 32): + p = 1 + maxn = opslimit // (4 * r) + for n_log2 in range(1, 63): # pragma: no branch + if (2**n_log2) > (maxn // 2): + break + else: + maxn = memlimit // (r * 128) + for n_log2 in range(1, 63): # pragma: no branch + if (2**n_log2) > maxn // 2: + break + + maxrp = (opslimit // 4) // (2**n_log2) + + if maxrp > 0x3FFFFFFF: # pragma: no cover + maxrp = 0x3FFFFFFF + + p = maxrp // r + + return n_log2, r, p + + +def crypto_pwhash_scryptsalsa208sha256_ll( + passwd: bytes, + salt: bytes, + n: int, + r: int, + p: int, + dklen: int = 64, + maxmem: int = SCRYPT_MAX_MEM, +) -> bytes: + """ + Derive a cryptographic key using the ``passwd`` and ``salt`` + given as input. + + The work factor can be tuned by by picking different + values for the parameters + + :param bytes passwd: + :param bytes salt: + :param bytes salt: *must* be *exactly* :py:const:`.SALTBYTES` long + :param int dklen: + :param int opslimit: + :param int n: + :param int r: block size, + :param int p: the parallelism factor + :param int maxmem: the maximum available memory available for scrypt's + operations + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_pwhash_scryptsalsa208sha256, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure(isinstance(n, int), raising=TypeError) + ensure(isinstance(r, int), raising=TypeError) + ensure(isinstance(p, int), raising=TypeError) + + ensure(isinstance(passwd, bytes), raising=TypeError) + ensure(isinstance(salt, bytes), raising=TypeError) + + _check_memory_occupation(n, r, p, maxmem) + + buf = ffi.new("uint8_t[]", dklen) + + ret = lib.crypto_pwhash_scryptsalsa208sha256_ll( + passwd, len(passwd), salt, len(salt), n, r, p, buf, dklen + ) + + ensure( + ret == 0, + "Unexpected failure in key derivation", + raising=exc.RuntimeError, + ) + + return ffi.buffer(ffi.cast("char *", buf), dklen)[:] + + +def crypto_pwhash_scryptsalsa208sha256_str( + passwd: bytes, + opslimit: int = SCRYPT_OPSLIMIT_INTERACTIVE, + memlimit: int = SCRYPT_MEMLIMIT_INTERACTIVE, +) -> bytes: + """ + Derive a cryptographic key using the ``passwd`` and ``salt`` + given as input, returning a string representation which includes + the salt and the tuning parameters. + + The returned string can be directly stored as a password hash. + + See :py:func:`.crypto_pwhash_scryptsalsa208sha256` for a short + discussion about ``opslimit`` and ``memlimit`` values. + + :param bytes passwd: + :param int opslimit: + :param int memlimit: + :return: serialized key hash, including salt and tuning parameters + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_pwhash_scryptsalsa208sha256, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + buf = ffi.new("char[]", SCRYPT_STRBYTES) + + ret = lib.crypto_pwhash_scryptsalsa208sha256_str( + buf, passwd, len(passwd), opslimit, memlimit + ) + + ensure( + ret == 0, + "Unexpected failure in password hashing", + raising=exc.RuntimeError, + ) + + return ffi.string(buf) + + +def crypto_pwhash_scryptsalsa208sha256_str_verify( + passwd_hash: bytes, passwd: bytes +) -> bool: + """ + Verifies the ``passwd`` against the ``passwd_hash`` that was generated. + Returns True or False depending on the success + + :param passwd_hash: bytes + :param passwd: bytes + :rtype: boolean + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_pwhash_scryptsalsa208sha256, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + len(passwd_hash) == SCRYPT_STRBYTES - 1, + "Invalid password hash", + raising=exc.ValueError, + ) + + ret = lib.crypto_pwhash_scryptsalsa208sha256_str_verify( + passwd_hash, passwd, len(passwd) + ) + ensure(ret == 0, "Wrong password", raising=exc.InvalidkeyError) + # all went well, therefore: + return True + + +def _check_argon2_limits_alg(opslimit: int, memlimit: int, alg: int) -> None: + if alg == crypto_pwhash_ALG_ARGON2I13: + if memlimit < crypto_pwhash_argon2i_MEMLIMIT_MIN: + raise exc.ValueError( + "memlimit must be at least {} bytes".format( + crypto_pwhash_argon2i_MEMLIMIT_MIN + ) + ) + elif memlimit > crypto_pwhash_argon2i_MEMLIMIT_MAX: + raise exc.ValueError( + "memlimit must be at most {} bytes".format( + crypto_pwhash_argon2i_MEMLIMIT_MAX + ) + ) + if opslimit < crypto_pwhash_argon2i_OPSLIMIT_MIN: + raise exc.ValueError( + "opslimit must be at least {}".format( + crypto_pwhash_argon2i_OPSLIMIT_MIN + ) + ) + elif opslimit > crypto_pwhash_argon2i_OPSLIMIT_MAX: + raise exc.ValueError( + "opslimit must be at most {}".format( + crypto_pwhash_argon2i_OPSLIMIT_MAX + ) + ) + + elif alg == crypto_pwhash_ALG_ARGON2ID13: + if memlimit < crypto_pwhash_argon2id_MEMLIMIT_MIN: + raise exc.ValueError( + "memlimit must be at least {} bytes".format( + crypto_pwhash_argon2id_MEMLIMIT_MIN + ) + ) + elif memlimit > crypto_pwhash_argon2id_MEMLIMIT_MAX: + raise exc.ValueError( + "memlimit must be at most {} bytes".format( + crypto_pwhash_argon2id_MEMLIMIT_MAX + ) + ) + if opslimit < crypto_pwhash_argon2id_OPSLIMIT_MIN: + raise exc.ValueError( + "opslimit must be at least {}".format( + crypto_pwhash_argon2id_OPSLIMIT_MIN + ) + ) + elif opslimit > crypto_pwhash_argon2id_OPSLIMIT_MAX: + raise exc.ValueError( + "opslimit must be at most {}".format( + crypto_pwhash_argon2id_OPSLIMIT_MAX + ) + ) + else: + raise exc.TypeError("Unsupported algorithm") + + +def crypto_pwhash_alg( + outlen: int, + passwd: bytes, + salt: bytes, + opslimit: int, + memlimit: int, + alg: int, +) -> bytes: + """ + Derive a raw cryptographic key using the ``passwd`` and the ``salt`` + given as input to the ``alg`` algorithm. + + :param outlen: the length of the derived key + :type outlen: int + :param passwd: The input password + :type passwd: bytes + :param salt: + :type salt: bytes + :param opslimit: computational cost + :type opslimit: int + :param memlimit: memory cost + :type memlimit: int + :param alg: algorithm identifier + :type alg: int + :return: derived key + :rtype: bytes + """ + ensure(isinstance(outlen, int), raising=exc.TypeError) + ensure(isinstance(opslimit, int), raising=exc.TypeError) + ensure(isinstance(memlimit, int), raising=exc.TypeError) + ensure(isinstance(alg, int), raising=exc.TypeError) + ensure(isinstance(passwd, bytes), raising=exc.TypeError) + + if len(salt) != crypto_pwhash_SALTBYTES: + raise exc.ValueError( + "salt must be exactly {} bytes long".format( + crypto_pwhash_SALTBYTES + ) + ) + + if outlen < crypto_pwhash_BYTES_MIN: + raise exc.ValueError( + "derived key must be at least {} bytes long".format( + crypto_pwhash_BYTES_MIN + ) + ) + + elif outlen > crypto_pwhash_BYTES_MAX: + raise exc.ValueError( + "derived key must be at most {} bytes long".format( + crypto_pwhash_BYTES_MAX + ) + ) + + _check_argon2_limits_alg(opslimit, memlimit, alg) + + outbuf = ffi.new("unsigned char[]", outlen) + + ret = lib.crypto_pwhash( + outbuf, outlen, passwd, len(passwd), salt, opslimit, memlimit, alg + ) + + ensure( + ret == 0, + "Unexpected failure in key derivation", + raising=exc.RuntimeError, + ) + + return ffi.buffer(outbuf, outlen)[:] + + +def crypto_pwhash_str_alg( + passwd: bytes, + opslimit: int, + memlimit: int, + alg: int, +) -> bytes: + """ + Derive a cryptographic key using the ``passwd`` given as input + and a random salt, returning a string representation which + includes the salt, the tuning parameters and the used algorithm. + + :param passwd: The input password + :type passwd: bytes + :param opslimit: computational cost + :type opslimit: int + :param memlimit: memory cost + :type memlimit: int + :param alg: The algorithm to use + :type alg: int + :return: serialized derived key and parameters + :rtype: bytes + """ + ensure(isinstance(opslimit, int), raising=TypeError) + ensure(isinstance(memlimit, int), raising=TypeError) + ensure(isinstance(passwd, bytes), raising=TypeError) + + _check_argon2_limits_alg(opslimit, memlimit, alg) + + outbuf = ffi.new("char[]", 128) + + ret = lib.crypto_pwhash_str_alg( + outbuf, passwd, len(passwd), opslimit, memlimit, alg + ) + + ensure( + ret == 0, + "Unexpected failure in key derivation", + raising=exc.RuntimeError, + ) + + return ffi.string(outbuf) + + +def crypto_pwhash_str_verify(passwd_hash: bytes, passwd: bytes) -> bool: + """ + Verifies the ``passwd`` against a given password hash. + + Returns True on success, raises InvalidkeyError on failure + :param passwd_hash: saved password hash + :type passwd_hash: bytes + :param passwd: password to be checked + :type passwd: bytes + :return: success + :rtype: boolean + """ + ensure(isinstance(passwd_hash, bytes), raising=TypeError) + ensure(isinstance(passwd, bytes), raising=TypeError) + ensure( + len(passwd_hash) <= 127, + "Hash must be at most 127 bytes long", + raising=exc.ValueError, + ) + + ret = lib.crypto_pwhash_str_verify(passwd_hash, passwd, len(passwd)) + + ensure(ret == 0, "Wrong password", raising=exc.InvalidkeyError) + # all went well, therefore: + return True + + +crypto_pwhash_argon2i_str_verify = crypto_pwhash_str_verify diff --git a/lib/nacl/bindings/crypto_scalarmult.py b/lib/nacl/bindings/crypto_scalarmult.py new file mode 100644 index 0000000..ca4a281 --- /dev/null +++ b/lib/nacl/bindings/crypto_scalarmult.py @@ -0,0 +1,240 @@ +# Copyright 2013-2018 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +has_crypto_scalarmult_ed25519 = bool(lib.PYNACL_HAS_CRYPTO_SCALARMULT_ED25519) + +crypto_scalarmult_BYTES: int = lib.crypto_scalarmult_bytes() +crypto_scalarmult_SCALARBYTES: int = lib.crypto_scalarmult_scalarbytes() + +crypto_scalarmult_ed25519_BYTES = 0 +crypto_scalarmult_ed25519_SCALARBYTES = 0 + +if has_crypto_scalarmult_ed25519: + crypto_scalarmult_ed25519_BYTES = lib.crypto_scalarmult_ed25519_bytes() + crypto_scalarmult_ed25519_SCALARBYTES = ( + lib.crypto_scalarmult_ed25519_scalarbytes() + ) + + +def crypto_scalarmult_base(n: bytes) -> bytes: + """ + Computes and returns the scalar product of a standard group element and an + integer ``n``. + + :param n: bytes + :rtype: bytes + """ + q = ffi.new("unsigned char[]", crypto_scalarmult_BYTES) + + rc = lib.crypto_scalarmult_base(q, n) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(q, crypto_scalarmult_SCALARBYTES)[:] + + +def crypto_scalarmult(n: bytes, p: bytes) -> bytes: + """ + Computes and returns the scalar product of the given group element and an + integer ``n``. + + :param p: bytes + :param n: bytes + :rtype: bytes + """ + q = ffi.new("unsigned char[]", crypto_scalarmult_BYTES) + + rc = lib.crypto_scalarmult(q, n, p) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(q, crypto_scalarmult_SCALARBYTES)[:] + + +def crypto_scalarmult_ed25519_base(n: bytes) -> bytes: + """ + Computes and returns the scalar product of a standard group element and an + integer ``n`` on the edwards25519 curve. + + :param n: a :py:data:`.crypto_scalarmult_ed25519_SCALARBYTES` long bytes + sequence representing a scalar + :type n: bytes + :return: a point on the edwards25519 curve, represented as a + :py:data:`.crypto_scalarmult_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_scalarmult_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(n, bytes) + and len(n) == crypto_scalarmult_ed25519_SCALARBYTES, + "Input must be a {} long bytes sequence".format( + "crypto_scalarmult_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + q = ffi.new("unsigned char[]", crypto_scalarmult_ed25519_BYTES) + + rc = lib.crypto_scalarmult_ed25519_base(q, n) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(q, crypto_scalarmult_ed25519_BYTES)[:] + + +def crypto_scalarmult_ed25519_base_noclamp(n: bytes) -> bytes: + """ + Computes and returns the scalar product of a standard group element and an + integer ``n`` on the edwards25519 curve. The integer ``n`` is not clamped. + + :param n: a :py:data:`.crypto_scalarmult_ed25519_SCALARBYTES` long bytes + sequence representing a scalar + :type n: bytes + :return: a point on the edwards25519 curve, represented as a + :py:data:`.crypto_scalarmult_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_scalarmult_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(n, bytes) + and len(n) == crypto_scalarmult_ed25519_SCALARBYTES, + "Input must be a {} long bytes sequence".format( + "crypto_scalarmult_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + q = ffi.new("unsigned char[]", crypto_scalarmult_ed25519_BYTES) + + rc = lib.crypto_scalarmult_ed25519_base_noclamp(q, n) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(q, crypto_scalarmult_ed25519_BYTES)[:] + + +def crypto_scalarmult_ed25519(n: bytes, p: bytes) -> bytes: + """ + Computes and returns the scalar product of a *clamped* integer ``n`` + and the given group element on the edwards25519 curve. + The scalar is clamped, as done in the public key generation case, + by setting to zero the bits in position [0, 1, 2, 255] and setting + to one the bit in position 254. + + :param n: a :py:data:`.crypto_scalarmult_ed25519_SCALARBYTES` long bytes + sequence representing a scalar + :type n: bytes + :param p: a :py:data:`.crypto_scalarmult_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type p: bytes + :return: a point on the edwards25519 curve, represented as a + :py:data:`.crypto_scalarmult_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_scalarmult_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(n, bytes) + and len(n) == crypto_scalarmult_ed25519_SCALARBYTES, + "Input must be a {} long bytes sequence".format( + "crypto_scalarmult_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(p, bytes) and len(p) == crypto_scalarmult_ed25519_BYTES, + "Input must be a {} long bytes sequence".format( + "crypto_scalarmult_ed25519_BYTES" + ), + raising=exc.TypeError, + ) + + q = ffi.new("unsigned char[]", crypto_scalarmult_ed25519_BYTES) + + rc = lib.crypto_scalarmult_ed25519(q, n, p) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(q, crypto_scalarmult_ed25519_BYTES)[:] + + +def crypto_scalarmult_ed25519_noclamp(n: bytes, p: bytes) -> bytes: + """ + Computes and returns the scalar product of an integer ``n`` + and the given group element on the edwards25519 curve. The integer + ``n`` is not clamped. + + :param n: a :py:data:`.crypto_scalarmult_ed25519_SCALARBYTES` long bytes + sequence representing a scalar + :type n: bytes + :param p: a :py:data:`.crypto_scalarmult_ed25519_BYTES` long bytes sequence + representing a point on the edwards25519 curve + :type p: bytes + :return: a point on the edwards25519 curve, represented as a + :py:data:`.crypto_scalarmult_ed25519_BYTES` long bytes sequence + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_scalarmult_ed25519, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + isinstance(n, bytes) + and len(n) == crypto_scalarmult_ed25519_SCALARBYTES, + "Input must be a {} long bytes sequence".format( + "crypto_scalarmult_ed25519_SCALARBYTES" + ), + raising=exc.TypeError, + ) + + ensure( + isinstance(p, bytes) and len(p) == crypto_scalarmult_ed25519_BYTES, + "Input must be a {} long bytes sequence".format( + "crypto_scalarmult_ed25519_BYTES" + ), + raising=exc.TypeError, + ) + + q = ffi.new("unsigned char[]", crypto_scalarmult_ed25519_BYTES) + + rc = lib.crypto_scalarmult_ed25519_noclamp(q, n, p) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(q, crypto_scalarmult_ed25519_BYTES)[:] diff --git a/lib/nacl/bindings/crypto_secretbox.py b/lib/nacl/bindings/crypto_secretbox.py new file mode 100644 index 0000000..d1ad113 --- /dev/null +++ b/lib/nacl/bindings/crypto_secretbox.py @@ -0,0 +1,159 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +crypto_secretbox_KEYBYTES: int = lib.crypto_secretbox_keybytes() +crypto_secretbox_NONCEBYTES: int = lib.crypto_secretbox_noncebytes() +crypto_secretbox_ZEROBYTES: int = lib.crypto_secretbox_zerobytes() +crypto_secretbox_BOXZEROBYTES: int = lib.crypto_secretbox_boxzerobytes() +crypto_secretbox_MACBYTES: int = lib.crypto_secretbox_macbytes() +crypto_secretbox_MESSAGEBYTES_MAX: int = ( + lib.crypto_secretbox_messagebytes_max() +) + + +def crypto_secretbox(message: bytes, nonce: bytes, key: bytes) -> bytes: + """ + Encrypts and returns the message ``message`` with the secret ``key`` and + the nonce ``nonce``. + + :param message: bytes + :param nonce: bytes + :param key: bytes + :rtype: bytes + """ + if len(key) != crypto_secretbox_KEYBYTES: + raise exc.ValueError("Invalid key") + + if len(nonce) != crypto_secretbox_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + padded = b"\x00" * crypto_secretbox_ZEROBYTES + message + ciphertext = ffi.new("unsigned char[]", len(padded)) + + res = lib.crypto_secretbox(ciphertext, padded, len(padded), nonce, key) + ensure(res == 0, "Encryption failed", raising=exc.CryptoError) + + ciphertext = ffi.buffer(ciphertext, len(padded)) + return ciphertext[crypto_secretbox_BOXZEROBYTES:] + + +def crypto_secretbox_open( + ciphertext: bytes, nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt and returns the encrypted message ``ciphertext`` with the secret + ``key`` and the nonce ``nonce``. + + :param ciphertext: bytes + :param nonce: bytes + :param key: bytes + :rtype: bytes + """ + if len(key) != crypto_secretbox_KEYBYTES: + raise exc.ValueError("Invalid key") + + if len(nonce) != crypto_secretbox_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + padded = b"\x00" * crypto_secretbox_BOXZEROBYTES + ciphertext + plaintext = ffi.new("unsigned char[]", len(padded)) + + res = lib.crypto_secretbox_open(plaintext, padded, len(padded), nonce, key) + ensure( + res == 0, + "Decryption failed. Ciphertext failed verification", + raising=exc.CryptoError, + ) + + plaintext = ffi.buffer(plaintext, len(padded)) + return plaintext[crypto_secretbox_ZEROBYTES:] + + +def crypto_secretbox_easy(message: bytes, nonce: bytes, key: bytes) -> bytes: + """ + Encrypts and returns the message ``message`` with the secret ``key`` and + the nonce ``nonce``. + + :param message: bytes + :param nonce: bytes + :param key: bytes + :rtype: bytes + """ + if len(key) != crypto_secretbox_KEYBYTES: + raise exc.ValueError("Invalid key") + + if len(nonce) != crypto_secretbox_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + _mlen = len(message) + _clen = crypto_secretbox_MACBYTES + _mlen + + ciphertext = ffi.new("unsigned char[]", _clen) + + res = lib.crypto_secretbox_easy(ciphertext, message, _mlen, nonce, key) + ensure(res == 0, "Encryption failed", raising=exc.CryptoError) + + ciphertext = ffi.buffer(ciphertext, _clen) + return ciphertext[:] + + +def crypto_secretbox_open_easy( + ciphertext: bytes, nonce: bytes, key: bytes +) -> bytes: + """ + Decrypt and returns the encrypted message ``ciphertext`` with the secret + ``key`` and the nonce ``nonce``. + + :param ciphertext: bytes + :param nonce: bytes + :param key: bytes + :rtype: bytes + """ + if len(key) != crypto_secretbox_KEYBYTES: + raise exc.ValueError("Invalid key") + + if len(nonce) != crypto_secretbox_NONCEBYTES: + raise exc.ValueError("Invalid nonce") + + _clen = len(ciphertext) + + ensure( + _clen >= crypto_secretbox_MACBYTES, + "Input ciphertext must be at least {} long".format( + crypto_secretbox_MACBYTES + ), + raising=exc.TypeError, + ) + + _mlen = _clen - crypto_secretbox_MACBYTES + + plaintext = ffi.new("unsigned char[]", max(1, _mlen)) + + res = lib.crypto_secretbox_open_easy( + plaintext, ciphertext, _clen, nonce, key + ) + ensure( + res == 0, + "Decryption failed. Ciphertext failed verification", + raising=exc.CryptoError, + ) + + plaintext = ffi.buffer(plaintext, _mlen) + return plaintext[:] diff --git a/lib/nacl/bindings/crypto_secretstream.py b/lib/nacl/bindings/crypto_secretstream.py new file mode 100644 index 0000000..59b074c --- /dev/null +++ b/lib/nacl/bindings/crypto_secretstream.py @@ -0,0 +1,358 @@ +# Copyright 2013-2018 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union, cast + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +crypto_secretstream_xchacha20poly1305_ABYTES: int = ( + lib.crypto_secretstream_xchacha20poly1305_abytes() +) +crypto_secretstream_xchacha20poly1305_HEADERBYTES: int = ( + lib.crypto_secretstream_xchacha20poly1305_headerbytes() +) +crypto_secretstream_xchacha20poly1305_KEYBYTES: int = ( + lib.crypto_secretstream_xchacha20poly1305_keybytes() +) +crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX: int = ( + lib.crypto_secretstream_xchacha20poly1305_messagebytes_max() +) +crypto_secretstream_xchacha20poly1305_STATEBYTES: int = ( + lib.crypto_secretstream_xchacha20poly1305_statebytes() +) + + +crypto_secretstream_xchacha20poly1305_TAG_MESSAGE: int = ( + lib.crypto_secretstream_xchacha20poly1305_tag_message() +) +crypto_secretstream_xchacha20poly1305_TAG_PUSH: int = ( + lib.crypto_secretstream_xchacha20poly1305_tag_push() +) +crypto_secretstream_xchacha20poly1305_TAG_REKEY: int = ( + lib.crypto_secretstream_xchacha20poly1305_tag_rekey() +) +crypto_secretstream_xchacha20poly1305_TAG_FINAL: int = ( + lib.crypto_secretstream_xchacha20poly1305_tag_final() +) + + +def crypto_secretstream_xchacha20poly1305_keygen() -> bytes: + """ + Generate a key for use with + :func:`.crypto_secretstream_xchacha20poly1305_init_push`. + + """ + keybuf = ffi.new( + "unsigned char[]", + crypto_secretstream_xchacha20poly1305_KEYBYTES, + ) + lib.crypto_secretstream_xchacha20poly1305_keygen(keybuf) + return ffi.buffer(keybuf)[:] + + +class crypto_secretstream_xchacha20poly1305_state: + """ + An object wrapping the crypto_secretstream_xchacha20poly1305 state. + + """ + + __slots__ = ["statebuf", "rawbuf", "tagbuf"] + + def __init__(self) -> None: + """Initialize a clean state object.""" + ByteString = Union[bytes, bytearray, memoryview] + self.statebuf: ByteString = ffi.new( + "unsigned char[]", + crypto_secretstream_xchacha20poly1305_STATEBYTES, + ) + + self.rawbuf: Optional[ByteString] = None + self.tagbuf: Optional[ByteString] = None + + +def crypto_secretstream_xchacha20poly1305_init_push( + state: crypto_secretstream_xchacha20poly1305_state, key: bytes +) -> bytes: + """ + Initialize a crypto_secretstream_xchacha20poly1305 encryption buffer. + + :param state: a secretstream state object + :type state: crypto_secretstream_xchacha20poly1305_state + :param key: must be + :data:`.crypto_secretstream_xchacha20poly1305_KEYBYTES` long + :type key: bytes + :return: header + :rtype: bytes + + """ + ensure( + isinstance(state, crypto_secretstream_xchacha20poly1305_state), + "State must be a crypto_secretstream_xchacha20poly1305_state object", + raising=exc.TypeError, + ) + ensure( + isinstance(key, bytes), + "Key must be a bytes sequence", + raising=exc.TypeError, + ) + ensure( + len(key) == crypto_secretstream_xchacha20poly1305_KEYBYTES, + "Invalid key length", + raising=exc.ValueError, + ) + + headerbuf = ffi.new( + "unsigned char []", + crypto_secretstream_xchacha20poly1305_HEADERBYTES, + ) + + rc = lib.crypto_secretstream_xchacha20poly1305_init_push( + state.statebuf, headerbuf, key + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + return ffi.buffer(headerbuf)[:] + + +def crypto_secretstream_xchacha20poly1305_push( + state: crypto_secretstream_xchacha20poly1305_state, + m: bytes, + ad: Optional[bytes] = None, + tag: int = crypto_secretstream_xchacha20poly1305_TAG_MESSAGE, +) -> bytes: + """ + Add an encrypted message to the secret stream. + + :param state: a secretstream state object + :type state: crypto_secretstream_xchacha20poly1305_state + :param m: the message to encrypt, the maximum length of an individual + message is + :data:`.crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX`. + :type m: bytes + :param ad: additional data to include in the authentication tag + :type ad: bytes or None + :param tag: the message tag, usually + :data:`.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE` or + :data:`.crypto_secretstream_xchacha20poly1305_TAG_FINAL`. + :type tag: int + :return: ciphertext + :rtype: bytes + + """ + ensure( + isinstance(state, crypto_secretstream_xchacha20poly1305_state), + "State must be a crypto_secretstream_xchacha20poly1305_state object", + raising=exc.TypeError, + ) + ensure(isinstance(m, bytes), "Message is not bytes", raising=exc.TypeError) + ensure( + len(m) <= crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX, + "Message is too long", + raising=exc.ValueError, + ) + ensure( + ad is None or isinstance(ad, bytes), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + clen = len(m) + crypto_secretstream_xchacha20poly1305_ABYTES + if state.rawbuf is None or len(state.rawbuf) < clen: + state.rawbuf = ffi.new("unsigned char[]", clen) + + if ad is None: + ad = ffi.NULL + adlen = 0 + else: + adlen = len(ad) + + rc = lib.crypto_secretstream_xchacha20poly1305_push( + state.statebuf, + state.rawbuf, + ffi.NULL, + m, + len(m), + ad, + adlen, + tag, + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + return ffi.buffer(state.rawbuf, clen)[:] + + +def crypto_secretstream_xchacha20poly1305_init_pull( + state: crypto_secretstream_xchacha20poly1305_state, + header: bytes, + key: bytes, +) -> None: + """ + Initialize a crypto_secretstream_xchacha20poly1305 decryption buffer. + + :param state: a secretstream state object + :type state: crypto_secretstream_xchacha20poly1305_state + :param header: must be + :data:`.crypto_secretstream_xchacha20poly1305_HEADERBYTES` long + :type header: bytes + :param key: must be + :data:`.crypto_secretstream_xchacha20poly1305_KEYBYTES` long + :type key: bytes + + """ + ensure( + isinstance(state, crypto_secretstream_xchacha20poly1305_state), + "State must be a crypto_secretstream_xchacha20poly1305_state object", + raising=exc.TypeError, + ) + ensure( + isinstance(header, bytes), + "Header must be a bytes sequence", + raising=exc.TypeError, + ) + ensure( + len(header) == crypto_secretstream_xchacha20poly1305_HEADERBYTES, + "Invalid header length", + raising=exc.ValueError, + ) + ensure( + isinstance(key, bytes), + "Key must be a bytes sequence", + raising=exc.TypeError, + ) + ensure( + len(key) == crypto_secretstream_xchacha20poly1305_KEYBYTES, + "Invalid key length", + raising=exc.ValueError, + ) + + if state.tagbuf is None: + state.tagbuf = ffi.new("unsigned char *") + + rc = lib.crypto_secretstream_xchacha20poly1305_init_pull( + state.statebuf, header, key + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + +def crypto_secretstream_xchacha20poly1305_pull( + state: crypto_secretstream_xchacha20poly1305_state, + c: bytes, + ad: Optional[bytes] = None, +) -> Tuple[bytes, int]: + """ + Read a decrypted message from the secret stream. + + :param state: a secretstream state object + :type state: crypto_secretstream_xchacha20poly1305_state + :param c: the ciphertext to decrypt, the maximum length of an individual + ciphertext is + :data:`.crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX` + + :data:`.crypto_secretstream_xchacha20poly1305_ABYTES`. + :type c: bytes + :param ad: additional data to include in the authentication tag + :type ad: bytes or None + :return: (message, tag) + :rtype: (bytes, int) + + """ + ensure( + isinstance(state, crypto_secretstream_xchacha20poly1305_state), + "State must be a crypto_secretstream_xchacha20poly1305_state object", + raising=exc.TypeError, + ) + ensure( + state.tagbuf is not None, + ( + "State must be initialized using " + "crypto_secretstream_xchacha20poly1305_init_pull" + ), + raising=exc.ValueError, + ) + ensure( + isinstance(c, bytes), + "Ciphertext is not bytes", + raising=exc.TypeError, + ) + ensure( + len(c) >= crypto_secretstream_xchacha20poly1305_ABYTES, + "Ciphertext is too short", + raising=exc.ValueError, + ) + ensure( + len(c) + <= ( + crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX + + crypto_secretstream_xchacha20poly1305_ABYTES + ), + "Ciphertext is too long", + raising=exc.ValueError, + ) + ensure( + ad is None or isinstance(ad, bytes), + "Additional data must be bytes or None", + raising=exc.TypeError, + ) + + mlen = len(c) - crypto_secretstream_xchacha20poly1305_ABYTES + if state.rawbuf is None or len(state.rawbuf) < mlen: + state.rawbuf = ffi.new("unsigned char[]", mlen) + + if ad is None: + ad = ffi.NULL + adlen = 0 + else: + adlen = len(ad) + + rc = lib.crypto_secretstream_xchacha20poly1305_pull( + state.statebuf, + state.rawbuf, + ffi.NULL, + state.tagbuf, + c, + len(c), + ad, + adlen, + ) + ensure(rc == 0, "Unexpected failure", raising=exc.RuntimeError) + + # Cast safety: we `ensure` above that `state.tagbuf is not None`. + return ( + ffi.buffer(state.rawbuf, mlen)[:], + int(cast(bytes, state.tagbuf)[0]), + ) + + +def crypto_secretstream_xchacha20poly1305_rekey( + state: crypto_secretstream_xchacha20poly1305_state, +) -> None: + """ + Explicitly change the encryption key in the stream. + + Normally the stream is re-keyed as needed or an explicit ``tag`` of + :data:`.crypto_secretstream_xchacha20poly1305_TAG_REKEY` is added to a + message to ensure forward secrecy, but this method can be used instead + if the re-keying is controlled without adding the tag. + + :param state: a secretstream state object + :type state: crypto_secretstream_xchacha20poly1305_state + + """ + ensure( + isinstance(state, crypto_secretstream_xchacha20poly1305_state), + "State must be a crypto_secretstream_xchacha20poly1305_state object", + raising=exc.TypeError, + ) + lib.crypto_secretstream_xchacha20poly1305_rekey(state.statebuf) diff --git a/lib/nacl/bindings/crypto_shorthash.py b/lib/nacl/bindings/crypto_shorthash.py new file mode 100644 index 0000000..8f7d209 --- /dev/null +++ b/lib/nacl/bindings/crypto_shorthash.py @@ -0,0 +1,81 @@ +# Copyright 2016 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import nacl.exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +has_crypto_shorthash_siphashx24 = bool( + lib.PYNACL_HAS_CRYPTO_SHORTHASH_SIPHASHX24 +) + +BYTES: int = lib.crypto_shorthash_siphash24_bytes() +KEYBYTES: int = lib.crypto_shorthash_siphash24_keybytes() + +XBYTES = 0 +XKEYBYTES = 0 + +if has_crypto_shorthash_siphashx24: + XBYTES = lib.crypto_shorthash_siphashx24_bytes() + XKEYBYTES = lib.crypto_shorthash_siphashx24_keybytes() + + +def crypto_shorthash_siphash24(data: bytes, key: bytes) -> bytes: + """Compute a fast, cryptographic quality, keyed hash of the input data + + :param data: + :type data: bytes + :param key: len(key) must be equal to + :py:data:`.KEYBYTES` (16) + :type key: bytes + """ + if len(key) != KEYBYTES: + raise exc.ValueError( + "Key length must be exactly {} bytes".format(KEYBYTES) + ) + digest = ffi.new("unsigned char[]", BYTES) + rc = lib.crypto_shorthash_siphash24(digest, data, len(data), key) + + ensure(rc == 0, raising=exc.RuntimeError) + return ffi.buffer(digest, BYTES)[:] + + +def crypto_shorthash_siphashx24(data: bytes, key: bytes) -> bytes: + """Compute a fast, cryptographic quality, keyed hash of the input data + + :param data: + :type data: bytes + :param key: len(key) must be equal to + :py:data:`.XKEYBYTES` (16) + :type key: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + """ + ensure( + has_crypto_shorthash_siphashx24, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + if len(key) != XKEYBYTES: + raise exc.ValueError( + "Key length must be exactly {} bytes".format(XKEYBYTES) + ) + digest = ffi.new("unsigned char[]", XBYTES) + rc = lib.crypto_shorthash_siphashx24(digest, data, len(data), key) + + ensure(rc == 0, raising=exc.RuntimeError) + return ffi.buffer(digest, XBYTES)[:] diff --git a/lib/nacl/bindings/crypto_sign.py b/lib/nacl/bindings/crypto_sign.py new file mode 100644 index 0000000..f459f6a --- /dev/null +++ b/lib/nacl/bindings/crypto_sign.py @@ -0,0 +1,327 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Tuple + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +crypto_sign_BYTES: int = lib.crypto_sign_bytes() +# crypto_sign_SEEDBYTES = lib.crypto_sign_seedbytes() +crypto_sign_SEEDBYTES: int = lib.crypto_sign_secretkeybytes() // 2 +crypto_sign_PUBLICKEYBYTES: int = lib.crypto_sign_publickeybytes() +crypto_sign_SECRETKEYBYTES: int = lib.crypto_sign_secretkeybytes() + +crypto_sign_curve25519_BYTES: int = lib.crypto_box_secretkeybytes() + +crypto_sign_ed25519ph_STATEBYTES: int = lib.crypto_sign_ed25519ph_statebytes() + + +def crypto_sign_keypair() -> Tuple[bytes, bytes]: + """ + Returns a randomly generated public key and secret key. + + :rtype: (bytes(public_key), bytes(secret_key)) + """ + pk = ffi.new("unsigned char[]", crypto_sign_PUBLICKEYBYTES) + sk = ffi.new("unsigned char[]", crypto_sign_SECRETKEYBYTES) + + rc = lib.crypto_sign_keypair(pk, sk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ( + ffi.buffer(pk, crypto_sign_PUBLICKEYBYTES)[:], + ffi.buffer(sk, crypto_sign_SECRETKEYBYTES)[:], + ) + + +def crypto_sign_seed_keypair(seed: bytes) -> Tuple[bytes, bytes]: + """ + Computes and returns the public key and secret key using the seed ``seed``. + + :param seed: bytes + :rtype: (bytes(public_key), bytes(secret_key)) + """ + if len(seed) != crypto_sign_SEEDBYTES: + raise exc.ValueError("Invalid seed") + + pk = ffi.new("unsigned char[]", crypto_sign_PUBLICKEYBYTES) + sk = ffi.new("unsigned char[]", crypto_sign_SECRETKEYBYTES) + + rc = lib.crypto_sign_seed_keypair(pk, sk, seed) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ( + ffi.buffer(pk, crypto_sign_PUBLICKEYBYTES)[:], + ffi.buffer(sk, crypto_sign_SECRETKEYBYTES)[:], + ) + + +def crypto_sign(message: bytes, sk: bytes) -> bytes: + """ + Signs the message ``message`` using the secret key ``sk`` and returns the + signed message. + + :param message: bytes + :param sk: bytes + :rtype: bytes + """ + signed = ffi.new("unsigned char[]", len(message) + crypto_sign_BYTES) + signed_len = ffi.new("unsigned long long *") + + rc = lib.crypto_sign(signed, signed_len, message, len(message), sk) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(signed, signed_len[0])[:] + + +def crypto_sign_open(signed: bytes, pk: bytes) -> bytes: + """ + Verifies the signature of the signed message ``signed`` using the public + key ``pk`` and returns the unsigned message. + + :param signed: bytes + :param pk: bytes + :rtype: bytes + """ + message = ffi.new("unsigned char[]", len(signed)) + message_len = ffi.new("unsigned long long *") + + if ( + lib.crypto_sign_open(message, message_len, signed, len(signed), pk) + != 0 + ): + raise exc.BadSignatureError("Signature was forged or corrupt") + + return ffi.buffer(message, message_len[0])[:] + + +def crypto_sign_ed25519_pk_to_curve25519(public_key_bytes: bytes) -> bytes: + """ + Converts a public Ed25519 key (encoded as bytes ``public_key_bytes``) to + a public Curve25519 key as bytes. + + Raises a ValueError if ``public_key_bytes`` is not of length + ``crypto_sign_PUBLICKEYBYTES`` + + :param public_key_bytes: bytes + :rtype: bytes + """ + if len(public_key_bytes) != crypto_sign_PUBLICKEYBYTES: + raise exc.ValueError("Invalid curve public key") + + curve_public_key_len = crypto_sign_curve25519_BYTES + curve_public_key = ffi.new("unsigned char[]", curve_public_key_len) + + rc = lib.crypto_sign_ed25519_pk_to_curve25519( + curve_public_key, public_key_bytes + ) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(curve_public_key, curve_public_key_len)[:] + + +def crypto_sign_ed25519_sk_to_curve25519(secret_key_bytes: bytes) -> bytes: + """ + Converts a secret Ed25519 key (encoded as bytes ``secret_key_bytes``) to + a secret Curve25519 key as bytes. + + Raises a ValueError if ``secret_key_bytes``is not of length + ``crypto_sign_SECRETKEYBYTES`` + + :param secret_key_bytes: bytes + :rtype: bytes + """ + if len(secret_key_bytes) != crypto_sign_SECRETKEYBYTES: + raise exc.ValueError("Invalid curve secret key") + + curve_secret_key_len = crypto_sign_curve25519_BYTES + curve_secret_key = ffi.new("unsigned char[]", curve_secret_key_len) + + rc = lib.crypto_sign_ed25519_sk_to_curve25519( + curve_secret_key, secret_key_bytes + ) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(curve_secret_key, curve_secret_key_len)[:] + + +def crypto_sign_ed25519_sk_to_pk(secret_key_bytes: bytes) -> bytes: + """ + Extract the public Ed25519 key from a secret Ed25519 key (encoded + as bytes ``secret_key_bytes``). + + Raises a ValueError if ``secret_key_bytes``is not of length + ``crypto_sign_SECRETKEYBYTES`` + + :param secret_key_bytes: bytes + :rtype: bytes + """ + if len(secret_key_bytes) != crypto_sign_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + return secret_key_bytes[crypto_sign_SEEDBYTES:] + + +def crypto_sign_ed25519_sk_to_seed(secret_key_bytes: bytes) -> bytes: + """ + Extract the seed from a secret Ed25519 key (encoded + as bytes ``secret_key_bytes``). + + Raises a ValueError if ``secret_key_bytes``is not of length + ``crypto_sign_SECRETKEYBYTES`` + + :param secret_key_bytes: bytes + :rtype: bytes + """ + if len(secret_key_bytes) != crypto_sign_SECRETKEYBYTES: + raise exc.ValueError("Invalid secret key") + + return secret_key_bytes[:crypto_sign_SEEDBYTES] + + +class crypto_sign_ed25519ph_state: + """ + State object wrapping the sha-512 state used in ed25519ph computation + """ + + __slots__ = ["state"] + + def __init__(self) -> None: + self.state: bytes = ffi.new( + "unsigned char[]", crypto_sign_ed25519ph_STATEBYTES + ) + + rc = lib.crypto_sign_ed25519ph_init(self.state) + + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + +def crypto_sign_ed25519ph_update( + edph: crypto_sign_ed25519ph_state, pmsg: bytes +) -> None: + """ + Update the hash state wrapped in edph + + :param edph: the ed25519ph state being updated + :type edph: crypto_sign_ed25519ph_state + :param pmsg: the partial message + :type pmsg: bytes + :rtype: None + """ + ensure( + isinstance(edph, crypto_sign_ed25519ph_state), + "edph parameter must be a ed25519ph_state object", + raising=exc.TypeError, + ) + ensure( + isinstance(pmsg, bytes), + "pmsg parameter must be a bytes object", + raising=exc.TypeError, + ) + rc = lib.crypto_sign_ed25519ph_update(edph.state, pmsg, len(pmsg)) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + +def crypto_sign_ed25519ph_final_create( + edph: crypto_sign_ed25519ph_state, sk: bytes +) -> bytes: + """ + Create a signature for the data hashed in edph + using the secret key sk + + :param edph: the ed25519ph state for the data + being signed + :type edph: crypto_sign_ed25519ph_state + :param sk: the ed25519 secret key (secret and public part) + :type sk: bytes + :return: ed25519ph signature + :rtype: bytes + """ + ensure( + isinstance(edph, crypto_sign_ed25519ph_state), + "edph parameter must be a ed25519ph_state object", + raising=exc.TypeError, + ) + ensure( + isinstance(sk, bytes), + "secret key parameter must be a bytes object", + raising=exc.TypeError, + ) + ensure( + len(sk) == crypto_sign_SECRETKEYBYTES, + ("secret key must be {} bytes long").format( + crypto_sign_SECRETKEYBYTES + ), + raising=exc.TypeError, + ) + signature = ffi.new("unsigned char[]", crypto_sign_BYTES) + rc = lib.crypto_sign_ed25519ph_final_create( + edph.state, signature, ffi.NULL, sk + ) + ensure(rc == 0, "Unexpected library error", raising=exc.RuntimeError) + + return ffi.buffer(signature, crypto_sign_BYTES)[:] + + +def crypto_sign_ed25519ph_final_verify( + edph: crypto_sign_ed25519ph_state, signature: bytes, pk: bytes +) -> bool: + """ + Verify a prehashed signature using the public key pk + + :param edph: the ed25519ph state for the data + being verified + :type edph: crypto_sign_ed25519ph_state + :param signature: the signature being verified + :type signature: bytes + :param pk: the ed25519 public part of the signing key + :type pk: bytes + :return: True if the signature is valid + :rtype: boolean + :raises exc.BadSignatureError: if the signature is not valid + """ + ensure( + isinstance(edph, crypto_sign_ed25519ph_state), + "edph parameter must be a ed25519ph_state object", + raising=exc.TypeError, + ) + ensure( + isinstance(signature, bytes), + "signature parameter must be a bytes object", + raising=exc.TypeError, + ) + ensure( + len(signature) == crypto_sign_BYTES, + ("signature must be {} bytes long").format(crypto_sign_BYTES), + raising=exc.TypeError, + ) + ensure( + isinstance(pk, bytes), + "public key parameter must be a bytes object", + raising=exc.TypeError, + ) + ensure( + len(pk) == crypto_sign_PUBLICKEYBYTES, + ("public key must be {} bytes long").format( + crypto_sign_PUBLICKEYBYTES + ), + raising=exc.TypeError, + ) + rc = lib.crypto_sign_ed25519ph_final_verify(edph.state, signature, pk) + if rc != 0: + raise exc.BadSignatureError("Signature was forged or corrupt") + + return True diff --git a/lib/nacl/bindings/randombytes.py b/lib/nacl/bindings/randombytes.py new file mode 100644 index 0000000..ed76deb --- /dev/null +++ b/lib/nacl/bindings/randombytes.py @@ -0,0 +1,51 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib + +randombytes_SEEDBYTES: int = lib.randombytes_seedbytes() + + +def randombytes(size: int) -> bytes: + """ + Returns ``size`` number of random bytes from a cryptographically secure + random source. + + :param size: int + :rtype: bytes + """ + buf = ffi.new("unsigned char[]", size) + lib.randombytes(buf, size) + return ffi.buffer(buf, size)[:] + + +def randombytes_buf_deterministic(size: int, seed: bytes) -> bytes: + """ + Returns ``size`` number of deterministically generated pseudorandom bytes + from a seed + + :param size: int + :param seed: bytes + :rtype: bytes + """ + if len(seed) != randombytes_SEEDBYTES: + raise exc.TypeError( + "Deterministic random bytes must be generated from 32 bytes" + ) + + buf = ffi.new("unsigned char[]", size) + lib.randombytes_buf_deterministic(buf, size, seed) + return ffi.buffer(buf, size)[:] diff --git a/lib/nacl/bindings/sodium_core.py b/lib/nacl/bindings/sodium_core.py new file mode 100644 index 0000000..7ebb84c --- /dev/null +++ b/lib/nacl/bindings/sodium_core.py @@ -0,0 +1,33 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nacl import exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +def _sodium_init() -> None: + ensure( + lib.sodium_init() != -1, + "Could not initialize sodium", + raising=exc.RuntimeError, + ) + + +def sodium_init() -> None: + """ + Initializes sodium, picking the best implementations available for this + machine. + """ + ffi.init_once(_sodium_init, "libsodium") diff --git a/lib/nacl/bindings/utils.py b/lib/nacl/bindings/utils.py new file mode 100644 index 0000000..0ff22e3 --- /dev/null +++ b/lib/nacl/bindings/utils.py @@ -0,0 +1,141 @@ +# Copyright 2013-2017 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nacl.exceptions as exc +from nacl._sodium import ffi, lib +from nacl.exceptions import ensure + + +def sodium_memcmp(inp1: bytes, inp2: bytes) -> bool: + """ + Compare contents of two memory regions in constant time + """ + ensure(isinstance(inp1, bytes), raising=exc.TypeError) + ensure(isinstance(inp2, bytes), raising=exc.TypeError) + + ln = max(len(inp1), len(inp2)) + + buf1 = ffi.new("char []", ln) + buf2 = ffi.new("char []", ln) + + ffi.memmove(buf1, inp1, len(inp1)) + ffi.memmove(buf2, inp2, len(inp2)) + + eqL = len(inp1) == len(inp2) + eqC = lib.sodium_memcmp(buf1, buf2, ln) == 0 + + return eqL and eqC + + +def sodium_pad(s: bytes, blocksize: int) -> bytes: + """ + Pad the input bytearray ``s`` to a multiple of ``blocksize`` + using the ISO/IEC 7816-4 algorithm + + :param s: input bytes string + :type s: bytes + :param blocksize: + :type blocksize: int + :return: padded string + :rtype: bytes + """ + ensure(isinstance(s, bytes), raising=exc.TypeError) + ensure(isinstance(blocksize, int), raising=exc.TypeError) + if blocksize <= 0: + raise exc.ValueError + s_len = len(s) + m_len = s_len + blocksize + buf = ffi.new("unsigned char []", m_len) + p_len = ffi.new("size_t []", 1) + ffi.memmove(buf, s, s_len) + rc = lib.sodium_pad(p_len, buf, s_len, blocksize, m_len) + ensure(rc == 0, "Padding failure", raising=exc.CryptoError) + return ffi.buffer(buf, p_len[0])[:] + + +def sodium_unpad(s: bytes, blocksize: int) -> bytes: + """ + Remove ISO/IEC 7816-4 padding from the input byte array ``s`` + + :param s: input bytes string + :type s: bytes + :param blocksize: + :type blocksize: int + :return: unpadded string + :rtype: bytes + """ + ensure(isinstance(s, bytes), raising=exc.TypeError) + ensure(isinstance(blocksize, int), raising=exc.TypeError) + s_len = len(s) + u_len = ffi.new("size_t []", 1) + rc = lib.sodium_unpad(u_len, s, s_len, blocksize) + if rc != 0: + raise exc.CryptoError("Unpadding failure") + return s[: u_len[0]] + + +def sodium_increment(inp: bytes) -> bytes: + """ + Increment the value of a byte-sequence interpreted + as the little-endian representation of a unsigned big integer. + + :param inp: input bytes buffer + :type inp: bytes + :return: a byte-sequence representing, as a little-endian + unsigned big integer, the value ``to_int(inp)`` + incremented by one. + :rtype: bytes + + """ + ensure(isinstance(inp, bytes), raising=exc.TypeError) + + ln = len(inp) + buf = ffi.new("unsigned char []", ln) + + ffi.memmove(buf, inp, ln) + + lib.sodium_increment(buf, ln) + + return ffi.buffer(buf, ln)[:] + + +def sodium_add(a: bytes, b: bytes) -> bytes: + """ + Given a couple of *same-sized* byte sequences, interpreted as the + little-endian representation of two unsigned integers, compute + the modular addition of the represented values, in constant time for + a given common length of the byte sequences. + + :param a: input bytes buffer + :type a: bytes + :param b: input bytes buffer + :type b: bytes + :return: a byte-sequence representing, as a little-endian big integer, + the integer value of ``(to_int(a) + to_int(b)) mod 2^(8*len(a))`` + :rtype: bytes + """ + ensure(isinstance(a, bytes), raising=exc.TypeError) + ensure(isinstance(b, bytes), raising=exc.TypeError) + ln = len(a) + ensure(len(b) == ln, raising=exc.TypeError) + + buf_a = ffi.new("unsigned char []", ln) + buf_b = ffi.new("unsigned char []", ln) + + ffi.memmove(buf_a, a, ln) + ffi.memmove(buf_b, b, ln) + + lib.sodium_add(buf_a, buf_b, ln) + + return ffi.buffer(buf_a, ln)[:] diff --git a/lib/nacl/encoding.py b/lib/nacl/encoding.py new file mode 100644 index 0000000..6740cfb --- /dev/null +++ b/lib/nacl/encoding.py @@ -0,0 +1,105 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import binascii +from abc import ABCMeta, abstractmethod +from typing import SupportsBytes, Type + + +# TODO: when the minimum supported version of Python is 3.8, we can import +# Protocol from typing, and replace Encoder with a Protocol instead. +class _Encoder(metaclass=ABCMeta): + @staticmethod + @abstractmethod + def encode(data: bytes) -> bytes: + """Transform raw data to encoded data.""" + + @staticmethod + @abstractmethod + def decode(data: bytes) -> bytes: + """Transform encoded data back to raw data. + + Decoding after encoding should be a no-op, i.e. `decode(encode(x)) == x`. + """ + + +# Functions that use encoders are passed a subclass of _Encoder, not an instance +# (because the methods are all static). Let's gloss over that detail by defining +# an alias for Type[_Encoder]. +Encoder = Type[_Encoder] + + +class RawEncoder(_Encoder): + @staticmethod + def encode(data: bytes) -> bytes: + return data + + @staticmethod + def decode(data: bytes) -> bytes: + return data + + +class HexEncoder(_Encoder): + @staticmethod + def encode(data: bytes) -> bytes: + return binascii.hexlify(data) + + @staticmethod + def decode(data: bytes) -> bytes: + return binascii.unhexlify(data) + + +class Base16Encoder(_Encoder): + @staticmethod + def encode(data: bytes) -> bytes: + return base64.b16encode(data) + + @staticmethod + def decode(data: bytes) -> bytes: + return base64.b16decode(data) + + +class Base32Encoder(_Encoder): + @staticmethod + def encode(data: bytes) -> bytes: + return base64.b32encode(data) + + @staticmethod + def decode(data: bytes) -> bytes: + return base64.b32decode(data) + + +class Base64Encoder(_Encoder): + @staticmethod + def encode(data: bytes) -> bytes: + return base64.b64encode(data) + + @staticmethod + def decode(data: bytes) -> bytes: + return base64.b64decode(data) + + +class URLSafeBase64Encoder(_Encoder): + @staticmethod + def encode(data: bytes) -> bytes: + return base64.urlsafe_b64encode(data) + + @staticmethod + def decode(data: bytes) -> bytes: + return base64.urlsafe_b64decode(data) + + +class Encodable: + def encode(self: SupportsBytes, encoder: Encoder = RawEncoder) -> bytes: + return encoder.encode(bytes(self)) diff --git a/lib/nacl/exceptions.py b/lib/nacl/exceptions.py new file mode 100644 index 0000000..40b1635 --- /dev/null +++ b/lib/nacl/exceptions.py @@ -0,0 +1,88 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# We create a clone of various builtin Exception types which additionally +# inherit from CryptoError. Below, we refer to the parent types via the +# `builtins` namespace, so mypy can distinguish between (e.g.) +# `nacl.exceptions.RuntimeError` and `builtins.RuntimeError`. +import builtins +from typing import Type + + +class CryptoError(Exception): + """ + Base exception for all nacl related errors + """ + + +class BadSignatureError(CryptoError): + """ + Raised when the signature was forged or otherwise corrupt. + """ + + +class RuntimeError(builtins.RuntimeError, CryptoError): + pass + + +class AssertionError(builtins.AssertionError, CryptoError): + pass + + +class TypeError(builtins.TypeError, CryptoError): + pass + + +class ValueError(builtins.ValueError, CryptoError): + pass + + +class InvalidkeyError(CryptoError): + pass + + +class CryptPrefixError(InvalidkeyError): + pass + + +class UnavailableError(RuntimeError): + """ + is a subclass of :class:`~nacl.exceptions.RuntimeError`, raised when + trying to call functions not available in a minimal build of + libsodium or due to hardware limitations. + """ + + pass + + +def ensure(cond: bool, *args: object, **kwds: Type[Exception]) -> None: + """ + Return if a condition is true, otherwise raise a caller-configurable + :py:class:`Exception` + :param bool cond: the condition to be checked + :param sequence args: the arguments to be passed to the exception's + constructor + The only accepted named parameter is `raising` used to configure the + exception to be raised if `cond` is not `True` + """ + _CHK_UNEXP = "check_condition() got an unexpected keyword argument {0}" + + raising = kwds.pop("raising", AssertionError) + if kwds: + raise TypeError(_CHK_UNEXP.format(repr(kwds.popitem()[0]))) + + if cond is True: + return + raise raising(*args) diff --git a/lib/nacl/hash.py b/lib/nacl/hash.py new file mode 100644 index 0000000..9f81590 --- /dev/null +++ b/lib/nacl/hash.py @@ -0,0 +1,181 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The :mod:`nacl.hash` module exposes one-shot interfaces +for libsodium selected hash primitives and the constants needed +for their usage. +""" + +import nacl.bindings +import nacl.encoding + + +BLAKE2B_BYTES = nacl.bindings.crypto_generichash_BYTES +"""Default digest size for :func:`blake2b` hash""" +BLAKE2B_BYTES_MIN = nacl.bindings.crypto_generichash_BYTES_MIN +"""Minimum allowed digest size for :func:`blake2b` hash""" +BLAKE2B_BYTES_MAX = nacl.bindings.crypto_generichash_BYTES_MAX +"""Maximum allowed digest size for :func:`blake2b` hash""" +BLAKE2B_KEYBYTES = nacl.bindings.crypto_generichash_KEYBYTES +"""Default size of the ``key`` byte array for :func:`blake2b` hash""" +BLAKE2B_KEYBYTES_MIN = nacl.bindings.crypto_generichash_KEYBYTES_MIN +"""Minimum allowed size of the ``key`` byte array for :func:`blake2b` hash""" +BLAKE2B_KEYBYTES_MAX = nacl.bindings.crypto_generichash_KEYBYTES_MAX +"""Maximum allowed size of the ``key`` byte array for :func:`blake2b` hash""" +BLAKE2B_SALTBYTES = nacl.bindings.crypto_generichash_SALTBYTES +"""Maximum allowed length of the ``salt`` byte array for +:func:`blake2b` hash""" +BLAKE2B_PERSONALBYTES = nacl.bindings.crypto_generichash_PERSONALBYTES +"""Maximum allowed length of the ``personalization`` +byte array for :func:`blake2b` hash""" + +SIPHASH_BYTES = nacl.bindings.crypto_shorthash_siphash24_BYTES +"""Size of the :func:`siphash24` digest""" +SIPHASH_KEYBYTES = nacl.bindings.crypto_shorthash_siphash24_KEYBYTES +"""Size of the secret ``key`` used by the :func:`siphash24` MAC""" + +SIPHASHX_AVAILABLE = nacl.bindings.has_crypto_shorthash_siphashx24 +"""``True`` if :func:`siphashx24` is available to be called""" + +SIPHASHX_BYTES = nacl.bindings.crypto_shorthash_siphashx24_BYTES +"""Size of the :func:`siphashx24` digest""" +SIPHASHX_KEYBYTES = nacl.bindings.crypto_shorthash_siphashx24_KEYBYTES +"""Size of the secret ``key`` used by the :func:`siphashx24` MAC""" + +_b2b_hash = nacl.bindings.crypto_generichash_blake2b_salt_personal +_sip_hash = nacl.bindings.crypto_shorthash_siphash24 +_sip_hashx = nacl.bindings.crypto_shorthash_siphashx24 + + +def sha256( + message: bytes, encoder: nacl.encoding.Encoder = nacl.encoding.HexEncoder +) -> bytes: + """ + Hashes ``message`` with SHA256. + + :param message: The message to hash. + :type message: bytes + :param encoder: A class that is able to encode the hashed message. + :returns: The hashed message. + :rtype: bytes + """ + return encoder.encode(nacl.bindings.crypto_hash_sha256(message)) + + +def sha512( + message: bytes, encoder: nacl.encoding.Encoder = nacl.encoding.HexEncoder +) -> bytes: + """ + Hashes ``message`` with SHA512. + + :param message: The message to hash. + :type message: bytes + :param encoder: A class that is able to encode the hashed message. + :returns: The hashed message. + :rtype: bytes + """ + return encoder.encode(nacl.bindings.crypto_hash_sha512(message)) + + +def blake2b( + data: bytes, + digest_size: int = BLAKE2B_BYTES, + key: bytes = b"", + salt: bytes = b"", + person: bytes = b"", + encoder: nacl.encoding.Encoder = nacl.encoding.HexEncoder, +) -> bytes: + """ + Hashes ``data`` with blake2b. + + :param data: the digest input byte sequence + :type data: bytes + :param digest_size: the requested digest size; must be at most + :const:`BLAKE2B_BYTES_MAX`; + the default digest size is + :const:`BLAKE2B_BYTES` + :type digest_size: int + :param key: the key to be set for keyed MAC/PRF usage; if set, the key + must be at most :data:`~nacl.hash.BLAKE2B_KEYBYTES_MAX` long + :type key: bytes + :param salt: an initialization salt at most + :const:`BLAKE2B_SALTBYTES` long; + it will be zero-padded if needed + :type salt: bytes + :param person: a personalization string at most + :const:`BLAKE2B_PERSONALBYTES` long; + it will be zero-padded if needed + :type person: bytes + :param encoder: the encoder to use on returned digest + :type encoder: class + :returns: The hashed message. + :rtype: bytes + """ + + digest = _b2b_hash( + data, digest_size=digest_size, key=key, salt=salt, person=person + ) + return encoder.encode(digest) + + +generichash = blake2b + + +def siphash24( + message: bytes, + key: bytes = b"", + encoder: nacl.encoding.Encoder = nacl.encoding.HexEncoder, +) -> bytes: + """ + Computes a keyed MAC of ``message`` using the short-input-optimized + siphash-2-4 construction. + + :param message: The message to hash. + :type message: bytes + :param key: the message authentication key for the siphash MAC construct + :type key: bytes(:const:`SIPHASH_KEYBYTES`) + :param encoder: A class that is able to encode the hashed message. + :returns: The hashed message. + :rtype: bytes(:const:`SIPHASH_BYTES`) + """ + digest = _sip_hash(message, key) + return encoder.encode(digest) + + +shorthash = siphash24 + + +def siphashx24( + message: bytes, + key: bytes = b"", + encoder: nacl.encoding.Encoder = nacl.encoding.HexEncoder, +) -> bytes: + """ + Computes a keyed MAC of ``message`` using the 128 bit variant of the + siphash-2-4 construction. + + :param message: The message to hash. + :type message: bytes + :param key: the message authentication key for the siphash MAC construct + :type key: bytes(:const:`SIPHASHX_KEYBYTES`) + :param encoder: A class that is able to encode the hashed message. + :returns: The hashed message. + :rtype: bytes(:const:`SIPHASHX_BYTES`) + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + + .. versionadded:: 1.2 + """ + digest = _sip_hashx(message, key) + return encoder.encode(digest) diff --git a/lib/nacl/hashlib.py b/lib/nacl/hashlib.py new file mode 100644 index 0000000..9d289da --- /dev/null +++ b/lib/nacl/hashlib.py @@ -0,0 +1,143 @@ +# Copyright 2016-2019 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import binascii +from typing import NoReturn + +import nacl.bindings +from nacl.utils import bytes_as_string + +BYTES = nacl.bindings.crypto_generichash_BYTES +BYTES_MIN = nacl.bindings.crypto_generichash_BYTES_MIN +BYTES_MAX = nacl.bindings.crypto_generichash_BYTES_MAX +KEYBYTES = nacl.bindings.crypto_generichash_KEYBYTES +KEYBYTES_MIN = nacl.bindings.crypto_generichash_KEYBYTES_MIN +KEYBYTES_MAX = nacl.bindings.crypto_generichash_KEYBYTES_MAX +SALTBYTES = nacl.bindings.crypto_generichash_SALTBYTES +PERSONALBYTES = nacl.bindings.crypto_generichash_PERSONALBYTES + +SCRYPT_AVAILABLE = nacl.bindings.has_crypto_pwhash_scryptsalsa208sha256 + +_b2b_init = nacl.bindings.crypto_generichash_blake2b_init +_b2b_final = nacl.bindings.crypto_generichash_blake2b_final +_b2b_update = nacl.bindings.crypto_generichash_blake2b_update + + +class blake2b: + """ + :py:mod:`hashlib` API compatible blake2b algorithm implementation + """ + + MAX_DIGEST_SIZE = BYTES + MAX_KEY_SIZE = KEYBYTES_MAX + PERSON_SIZE = PERSONALBYTES + SALT_SIZE = SALTBYTES + + def __init__( + self, + data: bytes = b"", + digest_size: int = BYTES, + key: bytes = b"", + salt: bytes = b"", + person: bytes = b"", + ): + """ + :py:class:`.blake2b` algorithm initializer + + :param data: + :type data: bytes + :param int digest_size: the requested digest size; must be + at most :py:attr:`.MAX_DIGEST_SIZE`; + the default digest size is :py:data:`.BYTES` + :param key: the key to be set for keyed MAC/PRF usage; if set, + the key must be at most :py:data:`.KEYBYTES_MAX` long + :type key: bytes + :param salt: a initialization salt at most + :py:attr:`.SALT_SIZE` long; it will be zero-padded + if needed + :type salt: bytes + :param person: a personalization string at most + :py:attr:`.PERSONAL_SIZE` long; it will be zero-padded + if needed + :type person: bytes + """ + + self._state = _b2b_init( + key=key, salt=salt, person=person, digest_size=digest_size + ) + self._digest_size = digest_size + + if data: + self.update(data) + + @property + def digest_size(self) -> int: + return self._digest_size + + @property + def block_size(self) -> int: + return 128 + + @property + def name(self) -> str: + return "blake2b" + + def update(self, data: bytes) -> None: + _b2b_update(self._state, data) + + def digest(self) -> bytes: + _st = self._state.copy() + return _b2b_final(_st) + + def hexdigest(self) -> str: + return bytes_as_string(binascii.hexlify(self.digest())) + + def copy(self) -> "blake2b": + _cp = type(self)(digest_size=self.digest_size) + _st = self._state.copy() + _cp._state = _st + return _cp + + def __reduce__(self) -> NoReturn: + """ + Raise the same exception as hashlib's blake implementation + on copy.copy() + """ + raise TypeError( + "can't pickle {} objects".format(self.__class__.__name__) + ) + + +def scrypt( + password: bytes, + salt: bytes = b"", + n: int = 2**20, + r: int = 8, + p: int = 1, + maxmem: int = 2**25, + dklen: int = 64, +) -> bytes: + """ + Derive a cryptographic key using the scrypt KDF. + + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + + Implements the same signature as the ``hashlib.scrypt`` implemented + in cpython version 3.6 + """ + return nacl.bindings.crypto_pwhash_scryptsalsa208sha256_ll( + password, salt, n, r, p, maxmem=maxmem, dklen=dklen + ) diff --git a/lib/nacl/public.py b/lib/nacl/public.py new file mode 100644 index 0000000..a6fc958 --- /dev/null +++ b/lib/nacl/public.py @@ -0,0 +1,421 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import ClassVar, Generic, Optional, Type, TypeVar + +import nacl.bindings +from nacl import encoding +from nacl import exceptions as exc +from nacl.encoding import Encoder +from nacl.utils import EncryptedMessage, StringFixer, random + + +class PublicKey(encoding.Encodable, StringFixer): + """ + The public key counterpart to an Curve25519 :class:`nacl.public.PrivateKey` + for encrypting messages. + + :param public_key: [:class:`bytes`] Encoded Curve25519 public key + :param encoder: A class that is able to decode the `public_key` + + :cvar SIZE: The size that the public key is required to be + """ + + SIZE: ClassVar[int] = nacl.bindings.crypto_box_PUBLICKEYBYTES + + def __init__( + self, + public_key: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ): + self._public_key = encoder.decode(public_key) + if not isinstance(self._public_key, bytes): + raise exc.TypeError("PublicKey must be created from 32 bytes") + + if len(self._public_key) != self.SIZE: + raise exc.ValueError( + "The public key must be exactly {} bytes long".format( + self.SIZE + ) + ) + + def __bytes__(self) -> bytes: + return self._public_key + + def __hash__(self) -> int: + return hash(bytes(self)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, self.__class__): + return False + return nacl.bindings.sodium_memcmp(bytes(self), bytes(other)) + + def __ne__(self, other: object) -> bool: + return not (self == other) + + +class PrivateKey(encoding.Encodable, StringFixer): + """ + Private key for decrypting messages using the Curve25519 algorithm. + + .. warning:: This **must** be protected and remain secret. Anyone who + knows the value of your :class:`~nacl.public.PrivateKey` can decrypt + any message encrypted by the corresponding + :class:`~nacl.public.PublicKey` + + :param private_key: The private key used to decrypt messages + :param encoder: The encoder class used to decode the given keys + + :cvar SIZE: The size that the private key is required to be + :cvar SEED_SIZE: The size that the seed used to generate the + private key is required to be + """ + + SIZE: ClassVar[int] = nacl.bindings.crypto_box_SECRETKEYBYTES + SEED_SIZE: ClassVar[int] = nacl.bindings.crypto_box_SEEDBYTES + + def __init__( + self, + private_key: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ): + # Decode the secret_key + private_key = encoder.decode(private_key) + # verify the given secret key type and size are correct + if not ( + isinstance(private_key, bytes) and len(private_key) == self.SIZE + ): + raise exc.TypeError( + ( + "PrivateKey must be created from a {} bytes long raw secret key" + ).format(self.SIZE) + ) + + raw_public_key = nacl.bindings.crypto_scalarmult_base(private_key) + + self._private_key = private_key + self.public_key = PublicKey(raw_public_key) + + @classmethod + def from_seed( + cls, + seed: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> "PrivateKey": + """ + Generate a PrivateKey using a deterministic construction + starting from a caller-provided seed + + .. warning:: The seed **must** be high-entropy; therefore, + its generator **must** be a cryptographic quality + random function like, for example, :func:`~nacl.utils.random`. + + .. warning:: The seed **must** be protected and remain secret. + Anyone who knows the seed is really in possession of + the corresponding PrivateKey. + + :param seed: The seed used to generate the private key + :rtype: :class:`~nacl.public.PrivateKey` + """ + # decode the seed + seed = encoder.decode(seed) + # Verify the given seed type and size are correct + if not (isinstance(seed, bytes) and len(seed) == cls.SEED_SIZE): + raise exc.TypeError( + ( + "PrivateKey seed must be a {} bytes long binary sequence" + ).format(cls.SEED_SIZE) + ) + # generate a raw key pair from the given seed + raw_pk, raw_sk = nacl.bindings.crypto_box_seed_keypair(seed) + # construct a instance from the raw secret key + return cls(raw_sk) + + def __bytes__(self) -> bytes: + return self._private_key + + def __hash__(self) -> int: + return hash((type(self), bytes(self.public_key))) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, self.__class__): + return False + return self.public_key == other.public_key + + def __ne__(self, other: object) -> bool: + return not (self == other) + + @classmethod + def generate(cls) -> "PrivateKey": + """ + Generates a random :class:`~nacl.public.PrivateKey` object + + :rtype: :class:`~nacl.public.PrivateKey` + """ + return cls(random(PrivateKey.SIZE), encoder=encoding.RawEncoder) + + +_Box = TypeVar("_Box", bound="Box") + + +class Box(encoding.Encodable, StringFixer): + """ + The Box class boxes and unboxes messages between a pair of keys + + The ciphertexts generated by :class:`~nacl.public.Box` include a 16 + byte authenticator which is checked as part of the decryption. An invalid + authenticator will cause the decrypt function to raise an exception. The + authenticator is not a signature. Once you've decrypted the message you've + demonstrated the ability to create arbitrary valid message, so messages you + send are repudiable. For non-repudiable messages, sign them after + encryption. + + :param private_key: :class:`~nacl.public.PrivateKey` used to encrypt and + decrypt messages + :param public_key: :class:`~nacl.public.PublicKey` used to encrypt and + decrypt messages + + :cvar NONCE_SIZE: The size that the nonce is required to be. + """ + + NONCE_SIZE: ClassVar[int] = nacl.bindings.crypto_box_NONCEBYTES + _shared_key: bytes + + def __init__(self, private_key: PrivateKey, public_key: PublicKey): + if not isinstance(private_key, PrivateKey) or not isinstance( + public_key, PublicKey + ): + raise exc.TypeError( + "Box must be created from a PrivateKey and a PublicKey" + ) + self._shared_key = nacl.bindings.crypto_box_beforenm( + public_key.encode(encoder=encoding.RawEncoder), + private_key.encode(encoder=encoding.RawEncoder), + ) + + def __bytes__(self) -> bytes: + return self._shared_key + + @classmethod + def decode( + cls: Type[_Box], encoded: bytes, encoder: Encoder = encoding.RawEncoder + ) -> _Box: + """ + Alternative constructor. Creates a Box from an existing Box's shared key. + """ + # Create an empty box + box: _Box = cls.__new__(cls) + + # Assign our decoded value to the shared key of the box + box._shared_key = encoder.decode(encoded) + + return box + + def encrypt( + self, + plaintext: bytes, + nonce: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> EncryptedMessage: + """ + Encrypts the plaintext message using the given `nonce` (or generates + one randomly if omitted) and returns the ciphertext encoded with the + encoder. + + .. warning:: It is **VITALLY** important that the nonce is a nonce, + i.e. it is a number used only once for any given key. If you fail + to do this, you compromise the privacy of the messages encrypted. + + :param plaintext: [:class:`bytes`] The plaintext message to encrypt + :param nonce: [:class:`bytes`] The nonce to use in the encryption + :param encoder: The encoder to use to encode the ciphertext + :rtype: [:class:`nacl.utils.EncryptedMessage`] + """ + if nonce is None: + nonce = random(self.NONCE_SIZE) + + if len(nonce) != self.NONCE_SIZE: + raise exc.ValueError( + "The nonce must be exactly %s bytes long" % self.NONCE_SIZE + ) + + ciphertext = nacl.bindings.crypto_box_easy_afternm( + plaintext, + nonce, + self._shared_key, + ) + + encoded_nonce = encoder.encode(nonce) + encoded_ciphertext = encoder.encode(ciphertext) + + return EncryptedMessage._from_parts( + encoded_nonce, + encoded_ciphertext, + encoder.encode(nonce + ciphertext), + ) + + def decrypt( + self, + ciphertext: bytes, + nonce: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> bytes: + """ + Decrypts the ciphertext using the `nonce` (explicitly, when passed as a + parameter or implicitly, when omitted, as part of the ciphertext) and + returns the plaintext message. + + :param ciphertext: [:class:`bytes`] The encrypted message to decrypt + :param nonce: [:class:`bytes`] The nonce used when encrypting the + ciphertext + :param encoder: The encoder used to decode the ciphertext. + :rtype: [:class:`bytes`] + """ + # Decode our ciphertext + ciphertext = encoder.decode(ciphertext) + + if nonce is None: + # If we were given the nonce and ciphertext combined, split them. + nonce = ciphertext[: self.NONCE_SIZE] + ciphertext = ciphertext[self.NONCE_SIZE :] + + if len(nonce) != self.NONCE_SIZE: + raise exc.ValueError( + "The nonce must be exactly %s bytes long" % self.NONCE_SIZE + ) + + plaintext = nacl.bindings.crypto_box_open_easy_afternm( + ciphertext, + nonce, + self._shared_key, + ) + + return plaintext + + def shared_key(self) -> bytes: + """ + Returns the Curve25519 shared secret, that can then be used as a key in + other symmetric ciphers. + + .. warning:: It is **VITALLY** important that you use a nonce with your + symmetric cipher. If you fail to do this, you compromise the + privacy of the messages encrypted. Ensure that the key length of + your cipher is 32 bytes. + :rtype: [:class:`bytes`] + """ + + return self._shared_key + + +_Key = TypeVar("_Key", PublicKey, PrivateKey) + + +class SealedBox(Generic[_Key], encoding.Encodable, StringFixer): + """ + The SealedBox class boxes and unboxes messages addressed to + a specified key-pair by using ephemeral sender's key pairs, + whose private part will be discarded just after encrypting + a single plaintext message. + + The ciphertexts generated by :class:`~nacl.public.SecretBox` include + the public part of the ephemeral key before the :class:`~nacl.public.Box` + ciphertext. + + :param recipient_key: a :class:`~nacl.public.PublicKey` used to encrypt + messages and derive nonces, or a :class:`~nacl.public.PrivateKey` used + to decrypt messages. + + .. versionadded:: 1.2 + """ + + _public_key: bytes + _private_key: Optional[bytes] + + def __init__(self, recipient_key: _Key): + if isinstance(recipient_key, PublicKey): + self._public_key = recipient_key.encode( + encoder=encoding.RawEncoder + ) + self._private_key = None + elif isinstance(recipient_key, PrivateKey): + self._private_key = recipient_key.encode( + encoder=encoding.RawEncoder + ) + self._public_key = recipient_key.public_key.encode( + encoder=encoding.RawEncoder + ) + else: + raise exc.TypeError( + "SealedBox must be created from a PublicKey or a PrivateKey" + ) + + def __bytes__(self) -> bytes: + return self._public_key + + def encrypt( + self, + plaintext: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> bytes: + """ + Encrypts the plaintext message using a random-generated ephemeral + key pair and returns a "composed ciphertext", containing both + the public part of the key pair and the ciphertext proper, + encoded with the encoder. + + The private part of the ephemeral key-pair will be scrubbed before + returning the ciphertext, therefore, the sender will not be able to + decrypt the generated ciphertext. + + :param plaintext: [:class:`bytes`] The plaintext message to encrypt + :param encoder: The encoder to use to encode the ciphertext + :return bytes: encoded ciphertext + """ + + ciphertext = nacl.bindings.crypto_box_seal(plaintext, self._public_key) + + encoded_ciphertext = encoder.encode(ciphertext) + + return encoded_ciphertext + + def decrypt( + self: "SealedBox[PrivateKey]", + ciphertext: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> bytes: + """ + Decrypts the ciphertext using the ephemeral public key enclosed + in the ciphertext and the SealedBox private key, returning + the plaintext message. + + :param ciphertext: [:class:`bytes`] The encrypted message to decrypt + :param encoder: The encoder used to decode the ciphertext. + :return bytes: The original plaintext + :raises TypeError: if this SealedBox was created with a + :class:`~nacl.public.PublicKey` rather than a + :class:`~nacl.public.PrivateKey`. + """ + # Decode our ciphertext + ciphertext = encoder.decode(ciphertext) + + if self._private_key is None: + raise TypeError( + "SealedBoxes created with a public key cannot decrypt" + ) + plaintext = nacl.bindings.crypto_box_seal_open( + ciphertext, + self._public_key, + self._private_key, + ) + + return plaintext diff --git a/lib/nacl/pwhash/__init__.py b/lib/nacl/pwhash/__init__.py new file mode 100644 index 0000000..ffd76a6 --- /dev/null +++ b/lib/nacl/pwhash/__init__.py @@ -0,0 +1,75 @@ +# Copyright 2017 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nacl.exceptions import CryptPrefixError + +from . import _argon2, argon2i, argon2id, scrypt + +STRPREFIX = argon2id.STRPREFIX + +PWHASH_SIZE = argon2id.PWHASH_SIZE + +assert _argon2.ALG_ARGON2_DEFAULT == _argon2.ALG_ARGON2ID13 +# since version 1.0.15 of libsodium + +PASSWD_MIN = argon2id.PASSWD_MIN +PASSWD_MAX = argon2id.PASSWD_MAX +MEMLIMIT_MAX = argon2id.MEMLIMIT_MAX +MEMLIMIT_MIN = argon2id.MEMLIMIT_MIN +OPSLIMIT_MAX = argon2id.OPSLIMIT_MAX +OPSLIMIT_MIN = argon2id.OPSLIMIT_MIN +OPSLIMIT_INTERACTIVE = argon2id.OPSLIMIT_INTERACTIVE +MEMLIMIT_INTERACTIVE = argon2id.MEMLIMIT_INTERACTIVE +OPSLIMIT_MODERATE = argon2id.OPSLIMIT_MODERATE +MEMLIMIT_MODERATE = argon2id.MEMLIMIT_MODERATE +OPSLIMIT_SENSITIVE = argon2id.OPSLIMIT_SENSITIVE +MEMLIMIT_SENSITIVE = argon2id.MEMLIMIT_SENSITIVE + +str = argon2id.str + +assert argon2i.ALG != argon2id.ALG + +SCRYPT_SALTBYTES = scrypt.SALTBYTES +SCRYPT_PWHASH_SIZE = scrypt.PWHASH_SIZE +SCRYPT_OPSLIMIT_INTERACTIVE = scrypt.OPSLIMIT_INTERACTIVE +SCRYPT_MEMLIMIT_INTERACTIVE = scrypt.MEMLIMIT_INTERACTIVE +SCRYPT_OPSLIMIT_SENSITIVE = scrypt.OPSLIMIT_SENSITIVE +SCRYPT_MEMLIMIT_SENSITIVE = scrypt.MEMLIMIT_SENSITIVE + + +kdf_scryptsalsa208sha256 = scrypt.kdf +scryptsalsa208sha256_str = scrypt.str +verify_scryptsalsa208sha256 = scrypt.verify + + +def verify(password_hash: bytes, password: bytes) -> bool: + """ + Takes a modular crypt encoded stored password hash derived using one + of the algorithms supported by `libsodium` and checks if the user provided + password will hash to the same string when using the parameters saved + in the stored hash + """ + if password_hash.startswith(argon2id.STRPREFIX): + return argon2id.verify(password_hash, password) + elif password_hash.startswith(argon2i.STRPREFIX): + return argon2id.verify(password_hash, password) + elif scrypt.AVAILABLE and password_hash.startswith(scrypt.STRPREFIX): + return scrypt.verify(password_hash, password) + else: + raise ( + CryptPrefixError( + "given password_hash is not in a supported format" + ) + ) diff --git a/lib/nacl/pwhash/__pycache__/__init__.cpython-314.pyc b/lib/nacl/pwhash/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..027c23f Binary files /dev/null and b/lib/nacl/pwhash/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/nacl/pwhash/__pycache__/_argon2.cpython-314.pyc b/lib/nacl/pwhash/__pycache__/_argon2.cpython-314.pyc new file mode 100644 index 0000000..e6ccc92 Binary files /dev/null and b/lib/nacl/pwhash/__pycache__/_argon2.cpython-314.pyc differ diff --git a/lib/nacl/pwhash/__pycache__/argon2i.cpython-314.pyc b/lib/nacl/pwhash/__pycache__/argon2i.cpython-314.pyc new file mode 100644 index 0000000..f7ac47e Binary files /dev/null and b/lib/nacl/pwhash/__pycache__/argon2i.cpython-314.pyc differ diff --git a/lib/nacl/pwhash/__pycache__/argon2id.cpython-314.pyc b/lib/nacl/pwhash/__pycache__/argon2id.cpython-314.pyc new file mode 100644 index 0000000..f554fc3 Binary files /dev/null and b/lib/nacl/pwhash/__pycache__/argon2id.cpython-314.pyc differ diff --git a/lib/nacl/pwhash/__pycache__/scrypt.cpython-314.pyc b/lib/nacl/pwhash/__pycache__/scrypt.cpython-314.pyc new file mode 100644 index 0000000..253b253 Binary files /dev/null and b/lib/nacl/pwhash/__pycache__/scrypt.cpython-314.pyc differ diff --git a/lib/nacl/pwhash/_argon2.py b/lib/nacl/pwhash/_argon2.py new file mode 100644 index 0000000..856eda0 --- /dev/null +++ b/lib/nacl/pwhash/_argon2.py @@ -0,0 +1,49 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nacl.bindings + +_argon2_strbytes_plus_one = nacl.bindings.crypto_pwhash_STRBYTES + +PWHASH_SIZE = _argon2_strbytes_plus_one - 1 +SALTBYTES = nacl.bindings.crypto_pwhash_SALTBYTES + +PASSWD_MIN = nacl.bindings.crypto_pwhash_PASSWD_MIN +PASSWD_MAX = nacl.bindings.crypto_pwhash_PASSWD_MAX + +PWHASH_SIZE = _argon2_strbytes_plus_one - 1 + +BYTES_MAX = nacl.bindings.crypto_pwhash_BYTES_MAX +BYTES_MIN = nacl.bindings.crypto_pwhash_BYTES_MIN + +ALG_ARGON2I13 = nacl.bindings.crypto_pwhash_ALG_ARGON2I13 +ALG_ARGON2ID13 = nacl.bindings.crypto_pwhash_ALG_ARGON2ID13 +ALG_ARGON2_DEFAULT = nacl.bindings.crypto_pwhash_ALG_DEFAULT + + +def verify(password_hash: bytes, password: bytes) -> bool: + """ + Takes a modular crypt encoded argon2i or argon2id stored password hash + and checks if the user provided password will hash to the same string + when using the stored parameters + + :param password_hash: password hash serialized in modular crypt() format + :type password_hash: bytes + :param password: user provided password + :type password: bytes + :rtype: boolean + + .. versionadded:: 1.2 + """ + return nacl.bindings.crypto_pwhash_str_verify(password_hash, password) diff --git a/lib/nacl/pwhash/argon2i.py b/lib/nacl/pwhash/argon2i.py new file mode 100644 index 0000000..f9b3af7 --- /dev/null +++ b/lib/nacl/pwhash/argon2i.py @@ -0,0 +1,132 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nacl.bindings +import nacl.encoding + +from . import _argon2 + +ALG = _argon2.ALG_ARGON2I13 +STRPREFIX = nacl.bindings.crypto_pwhash_argon2i_STRPREFIX + +SALTBYTES = _argon2.SALTBYTES + +PASSWD_MIN = _argon2.PASSWD_MIN +PASSWD_MAX = _argon2.PASSWD_MAX + +PWHASH_SIZE = _argon2.PWHASH_SIZE + +BYTES_MIN = _argon2.BYTES_MIN +BYTES_MAX = _argon2.BYTES_MAX + +verify = _argon2.verify + +MEMLIMIT_MAX = nacl.bindings.crypto_pwhash_argon2i_MEMLIMIT_MAX +MEMLIMIT_MIN = nacl.bindings.crypto_pwhash_argon2i_MEMLIMIT_MIN +OPSLIMIT_MAX = nacl.bindings.crypto_pwhash_argon2i_OPSLIMIT_MAX +OPSLIMIT_MIN = nacl.bindings.crypto_pwhash_argon2i_OPSLIMIT_MIN + +OPSLIMIT_INTERACTIVE = nacl.bindings.crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE +MEMLIMIT_INTERACTIVE = nacl.bindings.crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE +OPSLIMIT_SENSITIVE = nacl.bindings.crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE +MEMLIMIT_SENSITIVE = nacl.bindings.crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE + +OPSLIMIT_MODERATE = nacl.bindings.crypto_pwhash_argon2i_OPSLIMIT_MODERATE +MEMLIMIT_MODERATE = nacl.bindings.crypto_pwhash_argon2i_MEMLIMIT_MODERATE + + +def kdf( + size: int, + password: bytes, + salt: bytes, + opslimit: int = OPSLIMIT_SENSITIVE, + memlimit: int = MEMLIMIT_SENSITIVE, + encoder: nacl.encoding.Encoder = nacl.encoding.RawEncoder, +) -> bytes: + """ + Derive a ``size`` bytes long key from a caller-supplied + ``password`` and ``salt`` pair using the argon2i + memory-hard construct. + + the enclosing module provides the constants + + - :py:const:`.OPSLIMIT_INTERACTIVE` + - :py:const:`.MEMLIMIT_INTERACTIVE` + - :py:const:`.OPSLIMIT_MODERATE` + - :py:const:`.MEMLIMIT_MODERATE` + - :py:const:`.OPSLIMIT_SENSITIVE` + - :py:const:`.MEMLIMIT_SENSITIVE` + + as a guidance for correct settings. + + :param size: derived key size, must be between + :py:const:`.BYTES_MIN` and + :py:const:`.BYTES_MAX` + :type size: int + :param password: password used to seed the key derivation procedure; + it length must be between + :py:const:`.PASSWD_MIN` and + :py:const:`.PASSWD_MAX` + :type password: bytes + :param salt: **RANDOM** salt used in the key derivation procedure; + its length must be exactly :py:const:`.SALTBYTES` + :type salt: bytes + :param opslimit: the time component (operation count) + of the key derivation procedure's computational cost; + it must be between + :py:const:`.OPSLIMIT_MIN` and + :py:const:`.OPSLIMIT_MAX` + :type opslimit: int + :param memlimit: the memory occupation component + of the key derivation procedure's computational cost; + it must be between + :py:const:`.MEMLIMIT_MIN` and + :py:const:`.MEMLIMIT_MAX` + :type memlimit: int + :rtype: bytes + + .. versionadded:: 1.2 + """ + + return encoder.encode( + nacl.bindings.crypto_pwhash_alg( + size, password, salt, opslimit, memlimit, ALG + ) + ) + + +def str( + password: bytes, + opslimit: int = OPSLIMIT_INTERACTIVE, + memlimit: int = MEMLIMIT_INTERACTIVE, +) -> bytes: + """ + Hashes a password with a random salt, using the memory-hard + argon2i construct and returning an ascii string that has all + the needed info to check against a future password + + + The default settings for opslimit and memlimit are those deemed + correct for the interactive user login case. + + :param bytes password: + :param int opslimit: + :param int memlimit: + :rtype: bytes + + .. versionadded:: 1.2 + """ + return nacl.bindings.crypto_pwhash_str_alg( + password, opslimit, memlimit, ALG + ) diff --git a/lib/nacl/pwhash/argon2id.py b/lib/nacl/pwhash/argon2id.py new file mode 100644 index 0000000..f3aa3f7 --- /dev/null +++ b/lib/nacl/pwhash/argon2id.py @@ -0,0 +1,135 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nacl.bindings +import nacl.encoding + +from . import _argon2 + +ALG = _argon2.ALG_ARGON2ID13 +STRPREFIX = nacl.bindings.crypto_pwhash_argon2id_STRPREFIX + +SALTBYTES = _argon2.SALTBYTES + +PASSWD_MIN = _argon2.PASSWD_MIN +PASSWD_MAX = _argon2.PASSWD_MAX + +PWHASH_SIZE = _argon2.PWHASH_SIZE + +BYTES_MIN = _argon2.BYTES_MIN +BYTES_MAX = _argon2.BYTES_MAX + +verify = _argon2.verify + +MEMLIMIT_MIN = nacl.bindings.crypto_pwhash_argon2id_MEMLIMIT_MIN +MEMLIMIT_MAX = nacl.bindings.crypto_pwhash_argon2id_MEMLIMIT_MAX +OPSLIMIT_MIN = nacl.bindings.crypto_pwhash_argon2id_OPSLIMIT_MIN +OPSLIMIT_MAX = nacl.bindings.crypto_pwhash_argon2id_OPSLIMIT_MAX + +OPSLIMIT_INTERACTIVE = ( + nacl.bindings.crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE +) +MEMLIMIT_INTERACTIVE = ( + nacl.bindings.crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE +) +OPSLIMIT_SENSITIVE = nacl.bindings.crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE +MEMLIMIT_SENSITIVE = nacl.bindings.crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE + +OPSLIMIT_MODERATE = nacl.bindings.crypto_pwhash_argon2id_OPSLIMIT_MODERATE +MEMLIMIT_MODERATE = nacl.bindings.crypto_pwhash_argon2id_MEMLIMIT_MODERATE + + +def kdf( + size: int, + password: bytes, + salt: bytes, + opslimit: int = OPSLIMIT_SENSITIVE, + memlimit: int = MEMLIMIT_SENSITIVE, + encoder: nacl.encoding.Encoder = nacl.encoding.RawEncoder, +) -> bytes: + """ + Derive a ``size`` bytes long key from a caller-supplied + ``password`` and ``salt`` pair using the argon2id + memory-hard construct. + + the enclosing module provides the constants + + - :py:const:`.OPSLIMIT_INTERACTIVE` + - :py:const:`.MEMLIMIT_INTERACTIVE` + - :py:const:`.OPSLIMIT_MODERATE` + - :py:const:`.MEMLIMIT_MODERATE` + - :py:const:`.OPSLIMIT_SENSITIVE` + - :py:const:`.MEMLIMIT_SENSITIVE` + + as a guidance for correct settings. + + :param size: derived key size, must be between + :py:const:`.BYTES_MIN` and + :py:const:`.BYTES_MAX` + :type size: int + :param password: password used to seed the key derivation procedure; + it length must be between + :py:const:`.PASSWD_MIN` and + :py:const:`.PASSWD_MAX` + :type password: bytes + :param salt: **RANDOM** salt used in the key derivation procedure; + its length must be exactly :py:const:`.SALTBYTES` + :type salt: bytes + :param opslimit: the time component (operation count) + of the key derivation procedure's computational cost; + it must be between + :py:const:`.OPSLIMIT_MIN` and + :py:const:`.OPSLIMIT_MAX` + :type opslimit: int + :param memlimit: the memory occupation component + of the key derivation procedure's computational cost; + it must be between + :py:const:`.MEMLIMIT_MIN` and + :py:const:`.MEMLIMIT_MAX` + :type memlimit: int + :rtype: bytes + + .. versionadded:: 1.2 + """ + + return encoder.encode( + nacl.bindings.crypto_pwhash_alg( + size, password, salt, opslimit, memlimit, ALG + ) + ) + + +def str( + password: bytes, + opslimit: int = OPSLIMIT_INTERACTIVE, + memlimit: int = MEMLIMIT_INTERACTIVE, +) -> bytes: + """ + Hashes a password with a random salt, using the memory-hard + argon2id construct and returning an ascii string that has all + the needed info to check against a future password + + The default settings for opslimit and memlimit are those deemed + correct for the interactive user login case. + + :param bytes password: + :param int opslimit: + :param int memlimit: + :rtype: bytes + + .. versionadded:: 1.2 + """ + return nacl.bindings.crypto_pwhash_str_alg( + password, opslimit, memlimit, ALG + ) diff --git a/lib/nacl/pwhash/scrypt.py b/lib/nacl/pwhash/scrypt.py new file mode 100644 index 0000000..b9fc9d8 --- /dev/null +++ b/lib/nacl/pwhash/scrypt.py @@ -0,0 +1,211 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import cast + +import nacl.bindings +import nacl.encoding +from nacl import exceptions as exc +from nacl.exceptions import ensure + +_strbytes_plus_one = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_STRBYTES + +AVAILABLE = nacl.bindings.has_crypto_pwhash_scryptsalsa208sha256 + +STRPREFIX = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_STRPREFIX + +SALTBYTES = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_SALTBYTES + +PASSWD_MIN = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN +PASSWD_MAX = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX + +PWHASH_SIZE = _strbytes_plus_one - 1 + +BYTES_MIN = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_BYTES_MIN +BYTES_MAX = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_BYTES_MAX + +MEMLIMIT_MIN = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN +MEMLIMIT_MAX = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX +OPSLIMIT_MIN = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN +OPSLIMIT_MAX = nacl.bindings.crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX + +OPSLIMIT_INTERACTIVE = ( + nacl.bindings.crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE +) +MEMLIMIT_INTERACTIVE = ( + nacl.bindings.crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE +) +OPSLIMIT_SENSITIVE = ( + nacl.bindings.crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE +) +MEMLIMIT_SENSITIVE = ( + nacl.bindings.crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE +) + +OPSLIMIT_MODERATE = 8 * OPSLIMIT_INTERACTIVE +MEMLIMIT_MODERATE = 8 * MEMLIMIT_INTERACTIVE + + +def kdf( + size: int, + password: bytes, + salt: bytes, + opslimit: int = OPSLIMIT_SENSITIVE, + memlimit: int = MEMLIMIT_SENSITIVE, + encoder: nacl.encoding.Encoder = nacl.encoding.RawEncoder, +) -> bytes: + """ + Derive a ``size`` bytes long key from a caller-supplied + ``password`` and ``salt`` pair using the scryptsalsa208sha256 + memory-hard construct. + + + the enclosing module provides the constants + + - :py:const:`.OPSLIMIT_INTERACTIVE` + - :py:const:`.MEMLIMIT_INTERACTIVE` + - :py:const:`.OPSLIMIT_SENSITIVE` + - :py:const:`.MEMLIMIT_SENSITIVE` + - :py:const:`.OPSLIMIT_MODERATE` + - :py:const:`.MEMLIMIT_MODERATE` + + as a guidance for correct settings respectively for the + interactive login and the long term key protecting sensitive data + use cases. + + :param size: derived key size, must be between + :py:const:`.BYTES_MIN` and + :py:const:`.BYTES_MAX` + :type size: int + :param password: password used to seed the key derivation procedure; + it length must be between + :py:const:`.PASSWD_MIN` and + :py:const:`.PASSWD_MAX` + :type password: bytes + :param salt: **RANDOM** salt used in the key derivation procedure; + its length must be exactly :py:const:`.SALTBYTES` + :type salt: bytes + :param opslimit: the time component (operation count) + of the key derivation procedure's computational cost; + it must be between + :py:const:`.OPSLIMIT_MIN` and + :py:const:`.OPSLIMIT_MAX` + :type opslimit: int + :param memlimit: the memory occupation component + of the key derivation procedure's computational cost; + it must be between + :py:const:`.MEMLIMIT_MIN` and + :py:const:`.MEMLIMIT_MAX` + :type memlimit: int + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + + .. versionadded:: 1.2 + """ + ensure( + AVAILABLE, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + len(salt) == SALTBYTES, + "The salt must be exactly %s, not %s bytes long" + % (SALTBYTES, len(salt)), + raising=exc.ValueError, + ) + + n_log2, r, p = nacl.bindings.nacl_bindings_pick_scrypt_params( + opslimit, memlimit + ) + maxmem = memlimit + (2**16) + + return encoder.encode( + nacl.bindings.crypto_pwhash_scryptsalsa208sha256_ll( + password, + salt, + # Cast safety: n_log2 is a positive integer, and so 2 ** n_log2 is also + # a positive integer. Mypy+typeshed can't deduce this, because there's no + # way to for them to know that n_log2: int is positive. + cast(int, 2**n_log2), + r, + p, + maxmem=maxmem, + dklen=size, + ) + ) + + +def str( + password: bytes, + opslimit: int = OPSLIMIT_INTERACTIVE, + memlimit: int = MEMLIMIT_INTERACTIVE, +) -> bytes: + """ + Hashes a password with a random salt, using the memory-hard + scryptsalsa208sha256 construct and returning an ascii string + that has all the needed info to check against a future password + + The default settings for opslimit and memlimit are those deemed + correct for the interactive user login case. + + :param bytes password: + :param int opslimit: + :param int memlimit: + :rtype: bytes + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + + .. versionadded:: 1.2 + """ + ensure( + AVAILABLE, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + return nacl.bindings.crypto_pwhash_scryptsalsa208sha256_str( + password, opslimit, memlimit + ) + + +def verify(password_hash: bytes, password: bytes) -> bool: + """ + Takes the output of scryptsalsa208sha256 and compares it against + a user provided password to see if they are the same + + :param password_hash: bytes + :param password: bytes + :rtype: boolean + :raises nacl.exceptions.UnavailableError: If called when using a + minimal build of libsodium. + + .. versionadded:: 1.2 + """ + ensure( + AVAILABLE, + "Not available in minimal build", + raising=exc.UnavailableError, + ) + + ensure( + len(password_hash) == PWHASH_SIZE, + "The password hash must be exactly %s bytes long" + % nacl.bindings.crypto_pwhash_scryptsalsa208sha256_STRBYTES, + raising=exc.ValueError, + ) + + return nacl.bindings.crypto_pwhash_scryptsalsa208sha256_str_verify( + password_hash, password + ) diff --git a/lib/nacl/py.typed b/lib/nacl/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lib/nacl/secret.py b/lib/nacl/secret.py new file mode 100644 index 0000000..5c3064f --- /dev/null +++ b/lib/nacl/secret.py @@ -0,0 +1,305 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import ClassVar, Optional + +import nacl.bindings +from nacl import encoding +from nacl import exceptions as exc +from nacl.utils import EncryptedMessage, StringFixer, random + + +class SecretBox(encoding.Encodable, StringFixer): + """ + The SecretBox class encrypts and decrypts messages using the given secret + key. + + The ciphertexts generated by :class:`~nacl.secret.Secretbox` include a 16 + byte authenticator which is checked as part of the decryption. An invalid + authenticator will cause the decrypt function to raise an exception. The + authenticator is not a signature. Once you've decrypted the message you've + demonstrated the ability to create arbitrary valid message, so messages you + send are repudiable. For non-repudiable messages, sign them after + encryption. + + Encryption is done using `XSalsa20-Poly1305`_, and there are no practical + limits on the number or size of messages (up to 2⁶⁴ messages, each up to 2⁶⁴ + bytes). + + .. _XSalsa20-Poly1305: https://doc.libsodium.org/secret-key_cryptography/secretbox#algorithm-details + + :param key: The secret key used to encrypt and decrypt messages + :param encoder: The encoder class used to decode the given key + + :cvar KEY_SIZE: The size that the key is required to be. + :cvar NONCE_SIZE: The size that the nonce is required to be. + :cvar MACBYTES: The size of the authentication MAC tag in bytes. + :cvar MESSAGEBYTES_MAX: The maximum size of a message which can be + safely encrypted with a single key/nonce + pair. + """ + + KEY_SIZE: ClassVar[int] = nacl.bindings.crypto_secretbox_KEYBYTES + NONCE_SIZE: ClassVar[int] = nacl.bindings.crypto_secretbox_NONCEBYTES + MACBYTES: ClassVar[int] = nacl.bindings.crypto_secretbox_MACBYTES + MESSAGEBYTES_MAX: ClassVar[int] = ( + nacl.bindings.crypto_secretbox_MESSAGEBYTES_MAX + ) + + def __init__( + self, key: bytes, encoder: encoding.Encoder = encoding.RawEncoder + ): + key = encoder.decode(key) + if not isinstance(key, bytes): + raise exc.TypeError("SecretBox must be created from 32 bytes") + + if len(key) != self.KEY_SIZE: + raise exc.ValueError( + "The key must be exactly %s bytes long" % self.KEY_SIZE, + ) + + self._key = key + + def __bytes__(self) -> bytes: + return self._key + + def encrypt( + self, + plaintext: bytes, + nonce: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> EncryptedMessage: + """ + Encrypts the plaintext message using the given `nonce` (or generates + one randomly if omitted) and returns the ciphertext encoded with the + encoder. + + .. warning:: It is **VITALLY** important that the nonce is a nonce, + i.e. it is a number used only once for any given key. If you fail + to do this, you compromise the privacy of the messages encrypted. + Give your nonces a different prefix, or have one side use an odd + counter and one an even counter. Just make sure they are different. + + :param plaintext: [:class:`bytes`] The plaintext message to encrypt + :param nonce: [:class:`bytes`] The nonce to use in the encryption + :param encoder: The encoder to use to encode the ciphertext + :rtype: [:class:`nacl.utils.EncryptedMessage`] + """ + if nonce is None: + nonce = random(self.NONCE_SIZE) + + if len(nonce) != self.NONCE_SIZE: + raise exc.ValueError( + "The nonce must be exactly %s bytes long" % self.NONCE_SIZE, + ) + + ciphertext = nacl.bindings.crypto_secretbox_easy( + plaintext, nonce, self._key + ) + + encoded_nonce = encoder.encode(nonce) + encoded_ciphertext = encoder.encode(ciphertext) + + return EncryptedMessage._from_parts( + encoded_nonce, + encoded_ciphertext, + encoder.encode(nonce + ciphertext), + ) + + def decrypt( + self, + ciphertext: bytes, + nonce: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> bytes: + """ + Decrypts the ciphertext using the `nonce` (explicitly, when passed as a + parameter or implicitly, when omitted, as part of the ciphertext) and + returns the plaintext message. + + :param ciphertext: [:class:`bytes`] The encrypted message to decrypt + :param nonce: [:class:`bytes`] The nonce used when encrypting the + ciphertext + :param encoder: The encoder used to decode the ciphertext. + :rtype: [:class:`bytes`] + """ + # Decode our ciphertext + ciphertext = encoder.decode(ciphertext) + + if nonce is None: + # If we were given the nonce and ciphertext combined, split them. + nonce = ciphertext[: self.NONCE_SIZE] + ciphertext = ciphertext[self.NONCE_SIZE :] + + if len(nonce) != self.NONCE_SIZE: + raise exc.ValueError( + "The nonce must be exactly %s bytes long" % self.NONCE_SIZE, + ) + + plaintext = nacl.bindings.crypto_secretbox_open_easy( + ciphertext, nonce, self._key + ) + + return plaintext + + +class Aead(encoding.Encodable, StringFixer): + """ + The AEAD class encrypts and decrypts messages using the given secret key. + + Unlike :class:`~nacl.secret.SecretBox`, AEAD supports authenticating + non-confidential data received alongside the message, such as a length + or type tag. + + Like :class:`~nacl.secret.Secretbox`, this class provides authenticated + encryption. An inauthentic message will cause the decrypt function to raise + an exception. + + Likewise, the authenticator should not be mistaken for a (public-key) + signature: recipients (with the ability to decrypt messages) are capable of + creating arbitrary valid message; in particular, this means AEAD messages + are repudiable. For non-repudiable messages, sign them after encryption. + + The cryptosystem used is `XChacha20-Poly1305`_ as specified for + `standardization`_. There are `no practical limits`_ to how much can safely + be encrypted under a given key (up to 2⁶⁴ messages each containing up + to 2⁶⁴ bytes). + + .. _standardization: https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-xchacha + .. _XChacha20-Poly1305: https://doc.libsodium.org/secret-key_cryptography/aead#xchacha-20-poly1305 + .. _no practical limits: https://doc.libsodium.org/secret-key_cryptography/aead#limitations + + :param key: The secret key used to encrypt and decrypt messages + :param encoder: The encoder class used to decode the given key + + :cvar KEY_SIZE: The size that the key is required to be. + :cvar NONCE_SIZE: The size that the nonce is required to be. + :cvar MACBYTES: The size of the authentication MAC tag in bytes. + :cvar MESSAGEBYTES_MAX: The maximum size of a message which can be + safely encrypted with a single key/nonce + pair. + """ + + KEY_SIZE = nacl.bindings.crypto_aead_xchacha20poly1305_ietf_KEYBYTES + NONCE_SIZE = nacl.bindings.crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + MACBYTES = nacl.bindings.crypto_aead_xchacha20poly1305_ietf_ABYTES + MESSAGEBYTES_MAX = ( + nacl.bindings.crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + ) + + def __init__( + self, + key: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ): + key = encoder.decode(key) + if not isinstance(key, bytes): + raise exc.TypeError("AEAD must be created from 32 bytes") + + if len(key) != self.KEY_SIZE: + raise exc.ValueError( + "The key must be exactly %s bytes long" % self.KEY_SIZE, + ) + + self._key = key + + def __bytes__(self) -> bytes: + return self._key + + def encrypt( + self, + plaintext: bytes, + aad: bytes = b"", + nonce: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> EncryptedMessage: + """ + Encrypts the plaintext message using the given `nonce` (or generates + one randomly if omitted) and returns the ciphertext encoded with the + encoder. + + .. warning:: It is vitally important for :param nonce: to be unique. + By default, it is generated randomly; [:class:`Aead`] uses XChacha20 + for extended (192b) nonce size, so the risk of reusing random nonces + is negligible. It is *strongly recommended* to keep this behaviour, + as nonce reuse will compromise the privacy of encrypted messages. + Should implicit nonces be inadequate for your application, the + second best option is using split counters; e.g. if sending messages + encrypted under a shared key between 2 users, each user can use the + number of messages it sent so far, prefixed or suffixed with a 1bit + user id. Note that the counter must **never** be rolled back (due + to overflow, on-disk state being rolled back to an earlier backup, + ...) + + :param plaintext: [:class:`bytes`] The plaintext message to encrypt + :param nonce: [:class:`bytes`] The nonce to use in the encryption + :param encoder: The encoder to use to encode the ciphertext + :rtype: [:class:`nacl.utils.EncryptedMessage`] + """ + if nonce is None: + nonce = random(self.NONCE_SIZE) + + if len(nonce) != self.NONCE_SIZE: + raise exc.ValueError( + "The nonce must be exactly %s bytes long" % self.NONCE_SIZE, + ) + + ciphertext = nacl.bindings.crypto_aead_xchacha20poly1305_ietf_encrypt( + plaintext, aad, nonce, self._key + ) + + encoded_nonce = encoder.encode(nonce) + encoded_ciphertext = encoder.encode(ciphertext) + + return EncryptedMessage._from_parts( + encoded_nonce, + encoded_ciphertext, + encoder.encode(nonce + ciphertext), + ) + + def decrypt( + self, + ciphertext: bytes, + aad: bytes = b"", + nonce: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> bytes: + """ + Decrypts the ciphertext using the `nonce` (explicitly, when passed as a + parameter or implicitly, when omitted, as part of the ciphertext) and + returns the plaintext message. + + :param ciphertext: [:class:`bytes`] The encrypted message to decrypt + :param nonce: [:class:`bytes`] The nonce used when encrypting the + ciphertext + :param encoder: The encoder used to decode the ciphertext. + :rtype: [:class:`bytes`] + """ + # Decode our ciphertext + ciphertext = encoder.decode(ciphertext) + + if nonce is None: + # If we were given the nonce and ciphertext combined, split them. + nonce = ciphertext[: self.NONCE_SIZE] + ciphertext = ciphertext[self.NONCE_SIZE :] + + if len(nonce) != self.NONCE_SIZE: + raise exc.ValueError( + "The nonce must be exactly %s bytes long" % self.NONCE_SIZE, + ) + + plaintext = nacl.bindings.crypto_aead_xchacha20poly1305_ietf_decrypt( + ciphertext, aad, nonce, self._key + ) + + return plaintext diff --git a/lib/nacl/signing.py b/lib/nacl/signing.py new file mode 100644 index 0000000..536b369 --- /dev/null +++ b/lib/nacl/signing.py @@ -0,0 +1,250 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +import nacl.bindings +from nacl import encoding +from nacl import exceptions as exc +from nacl.public import ( + PrivateKey as _Curve25519_PrivateKey, + PublicKey as _Curve25519_PublicKey, +) +from nacl.utils import StringFixer, random + + +class SignedMessage(bytes): + """ + A bytes subclass that holds a message that has been signed by a + :class:`SigningKey`. + """ + + _signature: bytes + _message: bytes + + @classmethod + def _from_parts( + cls, signature: bytes, message: bytes, combined: bytes + ) -> "SignedMessage": + obj = cls(combined) + obj._signature = signature + obj._message = message + return obj + + @property + def signature(self) -> bytes: + """ + The signature contained within the :class:`SignedMessage`. + """ + return self._signature + + @property + def message(self) -> bytes: + """ + The message contained within the :class:`SignedMessage`. + """ + return self._message + + +class VerifyKey(encoding.Encodable, StringFixer): + """ + The public key counterpart to an Ed25519 SigningKey for producing digital + signatures. + + :param key: [:class:`bytes`] Serialized Ed25519 public key + :param encoder: A class that is able to decode the `key` + """ + + def __init__( + self, key: bytes, encoder: encoding.Encoder = encoding.RawEncoder + ): + # Decode the key + key = encoder.decode(key) + if not isinstance(key, bytes): + raise exc.TypeError("VerifyKey must be created from 32 bytes") + + if len(key) != nacl.bindings.crypto_sign_PUBLICKEYBYTES: + raise exc.ValueError( + "The key must be exactly %s bytes long" + % nacl.bindings.crypto_sign_PUBLICKEYBYTES, + ) + + self._key = key + + def __bytes__(self) -> bytes: + return self._key + + def __hash__(self) -> int: + return hash(bytes(self)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, self.__class__): + return False + return nacl.bindings.sodium_memcmp(bytes(self), bytes(other)) + + def __ne__(self, other: object) -> bool: + return not (self == other) + + def verify( + self, + smessage: bytes, + signature: Optional[bytes] = None, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> bytes: + """ + Verifies the signature of a signed message, returning the message + if it has not been tampered with else raising + :class:`~nacl.exceptions.BadSignatureError`. + + :param smessage: [:class:`bytes`] Either the original messaged or a + signature and message concated together. + :param signature: [:class:`bytes`] If an unsigned message is given for + smessage then the detached signature must be provided. + :param encoder: A class that is able to decode the secret message and + signature. + :rtype: :class:`bytes` + """ + if signature is not None: + # If we were given the message and signature separately, validate + # signature size and combine them. + if not isinstance(signature, bytes): + raise exc.TypeError( + "Verification signature must be created from %d bytes" + % nacl.bindings.crypto_sign_BYTES, + ) + + if len(signature) != nacl.bindings.crypto_sign_BYTES: + raise exc.ValueError( + "The signature must be exactly %d bytes long" + % nacl.bindings.crypto_sign_BYTES, + ) + + smessage = signature + encoder.decode(smessage) + else: + # Decode the signed message + smessage = encoder.decode(smessage) + + return nacl.bindings.crypto_sign_open(smessage, self._key) + + def to_curve25519_public_key(self) -> _Curve25519_PublicKey: + """ + Converts a :class:`~nacl.signing.VerifyKey` to a + :class:`~nacl.public.PublicKey` + + :rtype: :class:`~nacl.public.PublicKey` + """ + raw_pk = nacl.bindings.crypto_sign_ed25519_pk_to_curve25519(self._key) + return _Curve25519_PublicKey(raw_pk) + + +class SigningKey(encoding.Encodable, StringFixer): + """ + Private key for producing digital signatures using the Ed25519 algorithm. + + Signing keys are produced from a 32-byte (256-bit) random seed value. This + value can be passed into the :class:`~nacl.signing.SigningKey` as a + :func:`bytes` whose length is 32. + + .. warning:: This **must** be protected and remain secret. Anyone who knows + the value of your :class:`~nacl.signing.SigningKey` or it's seed can + masquerade as you. + + :param seed: [:class:`bytes`] Random 32-byte value (i.e. private key) + :param encoder: A class that is able to decode the seed + + :ivar: verify_key: [:class:`~nacl.signing.VerifyKey`] The verify + (i.e. public) key that corresponds with this signing key. + """ + + def __init__( + self, + seed: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ): + # Decode the seed + seed = encoder.decode(seed) + if not isinstance(seed, bytes): + raise exc.TypeError( + "SigningKey must be created from a 32 byte seed" + ) + + # Verify that our seed is the proper size + if len(seed) != nacl.bindings.crypto_sign_SEEDBYTES: + raise exc.ValueError( + "The seed must be exactly %d bytes long" + % nacl.bindings.crypto_sign_SEEDBYTES + ) + + public_key, secret_key = nacl.bindings.crypto_sign_seed_keypair(seed) + + self._seed = seed + self._signing_key = secret_key + self.verify_key = VerifyKey(public_key) + + def __bytes__(self) -> bytes: + return self._seed + + def __hash__(self) -> int: + return hash(bytes(self)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, self.__class__): + return False + return nacl.bindings.sodium_memcmp(bytes(self), bytes(other)) + + def __ne__(self, other: object) -> bool: + return not (self == other) + + @classmethod + def generate(cls) -> "SigningKey": + """ + Generates a random :class:`~nacl.signing.SigningKey` object. + + :rtype: :class:`~nacl.signing.SigningKey` + """ + return cls( + random(nacl.bindings.crypto_sign_SEEDBYTES), + encoder=encoding.RawEncoder, + ) + + def sign( + self, + message: bytes, + encoder: encoding.Encoder = encoding.RawEncoder, + ) -> SignedMessage: + """ + Sign a message using this key. + + :param message: [:class:`bytes`] The data to be signed. + :param encoder: A class that is used to encode the signed message. + :rtype: :class:`~nacl.signing.SignedMessage` + """ + raw_signed = nacl.bindings.crypto_sign(message, self._signing_key) + + crypto_sign_BYTES = nacl.bindings.crypto_sign_BYTES + signature = encoder.encode(raw_signed[:crypto_sign_BYTES]) + message = encoder.encode(raw_signed[crypto_sign_BYTES:]) + signed = encoder.encode(raw_signed) + + return SignedMessage._from_parts(signature, message, signed) + + def to_curve25519_private_key(self) -> _Curve25519_PrivateKey: + """ + Converts a :class:`~nacl.signing.SigningKey` to a + :class:`~nacl.public.PrivateKey` + + :rtype: :class:`~nacl.public.PrivateKey` + """ + sk = self._signing_key + raw_private = nacl.bindings.crypto_sign_ed25519_sk_to_curve25519(sk) + return _Curve25519_PrivateKey(raw_private) diff --git a/lib/nacl/utils.py b/lib/nacl/utils.py new file mode 100644 index 0000000..d19d236 --- /dev/null +++ b/lib/nacl/utils.py @@ -0,0 +1,88 @@ +# Copyright 2013 Donald Stufft and individual contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from typing import SupportsBytes, Type, TypeVar + +import nacl.bindings +from nacl import encoding + +_EncryptedMessage = TypeVar("_EncryptedMessage", bound="EncryptedMessage") + + +class EncryptedMessage(bytes): + """ + A bytes subclass that holds a messaged that has been encrypted by a + :class:`SecretBox`. + """ + + _nonce: bytes + _ciphertext: bytes + + @classmethod + def _from_parts( + cls: Type[_EncryptedMessage], + nonce: bytes, + ciphertext: bytes, + combined: bytes, + ) -> _EncryptedMessage: + obj = cls(combined) + obj._nonce = nonce + obj._ciphertext = ciphertext + return obj + + @property + def nonce(self) -> bytes: + """ + The nonce used during the encryption of the :class:`EncryptedMessage`. + """ + return self._nonce + + @property + def ciphertext(self) -> bytes: + """ + The ciphertext contained within the :class:`EncryptedMessage`. + """ + return self._ciphertext + + +class StringFixer: + def __str__(self: SupportsBytes) -> str: + return str(self.__bytes__()) + + +def bytes_as_string(bytes_in: bytes) -> str: + return bytes_in.decode("ascii") + + +def random(size: int = 32) -> bytes: + return os.urandom(size) + + +def randombytes_deterministic( + size: int, seed: bytes, encoder: encoding.Encoder = encoding.RawEncoder +) -> bytes: + """ + Returns ``size`` number of deterministically generated pseudorandom bytes + from a seed + + :param size: int + :param seed: bytes + :param encoder: The encoder class used to encode the produced bytes + :rtype: bytes + """ + raw_data = nacl.bindings.randombytes_buf_deterministic(size, seed) + + return encoder.encode(raw_data) diff --git a/lib/paramiko-4.0.0.dist-info/INSTALLER b/lib/paramiko-4.0.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/paramiko-4.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/paramiko-4.0.0.dist-info/METADATA b/lib/paramiko-4.0.0.dist-info/METADATA new file mode 100644 index 0000000..d57a39b --- /dev/null +++ b/lib/paramiko-4.0.0.dist-info/METADATA @@ -0,0 +1,88 @@ +Metadata-Version: 2.4 +Name: paramiko +Version: 4.0.0 +Summary: SSH2 protocol library +Author-email: Jeff Forcier +License-Expression: LGPL-2.1 +Project-URL: Docs, https://docs.paramiko.org +Project-URL: Source, https://github.com/paramiko/paramiko +Project-URL: Changelog, https://www.paramiko.org/changelog.html +Project-URL: CI, https://app.circleci.com/pipelines/github/paramiko/paramiko +Project-URL: Issues, https://github.com/paramiko/paramiko/issues +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Topic :: Internet +Classifier: Topic :: Security :: Cryptography +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Requires-Python: >=3.9 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: bcrypt>=3.2 +Requires-Dist: cryptography>=3.3 +Requires-Dist: invoke>=2.0 +Requires-Dist: pynacl>=1.5 +Provides-Extra: gssapi +Requires-Dist: pyasn1>=0.1.7; extra == "gssapi" +Requires-Dist: gssapi>=1.4.1; platform_system != "Windows" and extra == "gssapi" +Requires-Dist: pywin32>=2.1.8; platform_system == "Windows" and extra == "gssapi" +Dynamic: license-file + +|version| |python| |license| |ci| |coverage| + +.. |version| image:: https://img.shields.io/pypi/v/paramiko + :target: https://pypi.org/project/paramiko/ + :alt: PyPI - Package Version +.. |python| image:: https://img.shields.io/pypi/pyversions/paramiko + :target: https://pypi.org/project/paramiko/ + :alt: PyPI - Python Version +.. |license| image:: https://img.shields.io/pypi/l/paramiko + :target: https://github.com/paramiko/paramiko/blob/main/LICENSE + :alt: PyPI - License +.. |ci| image:: https://img.shields.io/circleci/build/github/paramiko/paramiko/main + :target: https://app.circleci.com/pipelines/github/paramiko/paramiko + :alt: CircleCI +.. |coverage| image:: https://img.shields.io/codecov/c/gh/paramiko/paramiko + :target: https://app.codecov.io/gh/paramiko/paramiko + :alt: Codecov + +Welcome to Paramiko! +==================== + +Paramiko is a pure-Python [#]_ implementation of the SSHv2 protocol [#]_, +providing both client and server functionality. It provides the foundation for +the high-level SSH library `Fabric `_, which is what we +recommend you use for common client use-cases such as running remote shell +commands or transferring files. + +Direct use of Paramiko itself is only intended for users who need +advanced/low-level primitives or want to run an in-Python sshd. + +For installation information, changelogs, FAQs and similar, please visit `our +main project website `_; for API details, see `the +versioned docs `_. Additionally, the project +maintainer keeps a `roadmap `_ on his +personal site. + +.. [#] + Paramiko relies on `cryptography `_ for crypto + functionality, which makes use of C and Rust extensions but has many + precompiled options available. See `our installation page + `_ for details. + +.. [#] + OpenSSH's RFC specification page is a fantastic resource and collection of + links that we won't bother replicating here: + https://www.openssh.com/specs.html + + OpenSSH itself also happens to be our primary reference implementation: + when in doubt, we consult how they do things, unless there are good reasons + not to. There are always some gaps, but we do our best to reconcile them + when possible. diff --git a/lib/paramiko-4.0.0.dist-info/RECORD b/lib/paramiko-4.0.0.dist-info/RECORD new file mode 100644 index 0000000..2389dec --- /dev/null +++ b/lib/paramiko-4.0.0.dist-info/RECORD @@ -0,0 +1,95 @@ +paramiko-4.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +paramiko-4.0.0.dist-info/METADATA,sha256=9W89LHpZs7eu34MZOmkAWnscpX_N-pwZN5RNRAWQQTI,3900 +paramiko-4.0.0.dist-info/RECORD,, +paramiko-4.0.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +paramiko-4.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91 +paramiko-4.0.0.dist-info/licenses/LICENSE,sha256=X6Jb9fOV_SbnAcLh3kyn0WKBaYbceRwi-PQiaFetG7I,26436 +paramiko-4.0.0.dist-info/top_level.txt,sha256=R9n-eCc_1kx1DnijF7Glmm-H67k9jUz5rm2YoPL8n54,9 +paramiko/__init__.py,sha256=aU-VhYiW5aIJosmuPeSteR1h5GeLOXNqdcpkaicsCwg,3523 +paramiko/__pycache__/__init__.cpython-314.pyc,, +paramiko/__pycache__/_winapi.cpython-314.pyc,, +paramiko/__pycache__/agent.cpython-314.pyc,, +paramiko/__pycache__/auth_handler.cpython-314.pyc,, +paramiko/__pycache__/auth_strategy.cpython-314.pyc,, +paramiko/__pycache__/ber.cpython-314.pyc,, +paramiko/__pycache__/buffered_pipe.cpython-314.pyc,, +paramiko/__pycache__/channel.cpython-314.pyc,, +paramiko/__pycache__/client.cpython-314.pyc,, +paramiko/__pycache__/common.cpython-314.pyc,, +paramiko/__pycache__/compress.cpython-314.pyc,, +paramiko/__pycache__/config.cpython-314.pyc,, +paramiko/__pycache__/ecdsakey.cpython-314.pyc,, +paramiko/__pycache__/ed25519key.cpython-314.pyc,, +paramiko/__pycache__/file.cpython-314.pyc,, +paramiko/__pycache__/hostkeys.cpython-314.pyc,, +paramiko/__pycache__/kex_curve25519.cpython-314.pyc,, +paramiko/__pycache__/kex_ecdh_nist.cpython-314.pyc,, +paramiko/__pycache__/kex_gex.cpython-314.pyc,, +paramiko/__pycache__/kex_group1.cpython-314.pyc,, +paramiko/__pycache__/kex_group14.cpython-314.pyc,, +paramiko/__pycache__/kex_group16.cpython-314.pyc,, +paramiko/__pycache__/kex_gss.cpython-314.pyc,, +paramiko/__pycache__/message.cpython-314.pyc,, +paramiko/__pycache__/packet.cpython-314.pyc,, +paramiko/__pycache__/pipe.cpython-314.pyc,, +paramiko/__pycache__/pkey.cpython-314.pyc,, +paramiko/__pycache__/primes.cpython-314.pyc,, +paramiko/__pycache__/proxy.cpython-314.pyc,, +paramiko/__pycache__/rsakey.cpython-314.pyc,, +paramiko/__pycache__/server.cpython-314.pyc,, +paramiko/__pycache__/sftp.cpython-314.pyc,, +paramiko/__pycache__/sftp_attr.cpython-314.pyc,, +paramiko/__pycache__/sftp_client.cpython-314.pyc,, +paramiko/__pycache__/sftp_file.cpython-314.pyc,, +paramiko/__pycache__/sftp_handle.cpython-314.pyc,, +paramiko/__pycache__/sftp_server.cpython-314.pyc,, +paramiko/__pycache__/sftp_si.cpython-314.pyc,, +paramiko/__pycache__/ssh_exception.cpython-314.pyc,, +paramiko/__pycache__/ssh_gss.cpython-314.pyc,, +paramiko/__pycache__/transport.cpython-314.pyc,, +paramiko/__pycache__/util.cpython-314.pyc,, +paramiko/__pycache__/win_openssh.cpython-314.pyc,, +paramiko/__pycache__/win_pageant.cpython-314.pyc,, +paramiko/_winapi.py,sha256=e4PyDmHmyLcAkZo4WAX7ah_I6fq4ex7A8FhxOPYAoA8,11204 +paramiko/agent.py,sha256=4vP4knAAzZiSblzSM_srbTYK2hVnUUT561vTBdCe2i4,15877 +paramiko/auth_handler.py,sha256=kMY00x5sUkrcR9uRHIIakQw4E6649oW1tMtIQPrFMFo,43006 +paramiko/auth_strategy.py,sha256=Pjcp8q64gUwk4CneGOnOhW0WBeKBRFURieWqC9AN0Ec,11437 +paramiko/ber.py,sha256=uFb-YokU4Rg2fKjyX8VMAu05STVk37YRgghlNHmdoYo,4369 +paramiko/buffered_pipe.py,sha256=AlkTLHYWbj4W-ZD7ORQZFjEFv7kC7QSvEYypfiHpwxw,7225 +paramiko/channel.py,sha256=MXO-C5dipy8Q0Shh9ceR-CPPiBB-ssT_9oIgwzBhQ_o,49222 +paramiko/client.py,sha256=d1UAVgVf_eWf-VqpwsjhyMFo4IEZcX2-rzZtkomsffY,34337 +paramiko/common.py,sha256=sBJW8KJz_EE8TsT7wLWTPuUiL2nNsLa_cfrTCe9Fyio,7756 +paramiko/compress.py,sha256=RCHTino0cHz1dy1pLbOhFhdWfGl4u50VmBcbT7qBWNc,1282 +paramiko/config.py,sha256=QPzwsk4Vem-Ecg2NhjRu78O9SU5ZO6DmfxZTA6cHWco,27362 +paramiko/ecdsakey.py,sha256=nK8oxORGgLP-zoC2REG46bAchVrlr35jfuxTn_Ac8sM,11653 +paramiko/ed25519key.py,sha256=FYurG0gqxmhNKh_22Hp3XEON5zuvzv-r5w8y9yJQgqY,7457 +paramiko/file.py,sha256=NgbhUjYgrLh-HQtsdYlPZ3CyvS0jhXqePk45GhHPMSo,19063 +paramiko/hostkeys.py,sha256=Ez2gaZF5ntj-vTvMbVXZoLRpU6tBnhSbXJm5FUlvzhw,13144 +paramiko/kex_curve25519.py,sha256=voEFDs_zkgEdWOqDakU-5DLYO3qotWcXYiqOCUP4GDo,4436 +paramiko/kex_ecdh_nist.py,sha256=RbHPwv8Gu5iR9LwMf-N0yUjXEQgRKKBLaAT3dacv44Q,5012 +paramiko/kex_gex.py,sha256=j5fPexu48CGObvpPKn0kZTjdn1onfz0iYhh8p8kIgM0,10320 +paramiko/kex_group1.py,sha256=HfzkLH1SKaIavnN-LGuF-lAMaAECB6Izj_TELhg4Omc,5740 +paramiko/kex_group14.py,sha256=AX7xrTCqMROrMQ_3Dp8WmLkNN8dTovhPjtWgaLLpRxs,1833 +paramiko/kex_group16.py,sha256=s7qB7tSDFkG5ztlg3mV958UVWnKgn1LIA-B2t-h1eX4,2288 +paramiko/kex_gss.py,sha256=BadM1nNN-ORDRuJmb93v0xBGQlce1n29lT4ihsnmY-4,24562 +paramiko/message.py,sha256=wHTWVU_Xgfq-djOOPVF5jAsE-XgADoH47G0iI5N69gY,9349 +paramiko/packet.py,sha256=CocYnZ2Vbz7VRo-6BGMhlRWro7FLIISpxTiYeoEsyaM,24314 +paramiko/pipe.py,sha256=cmWwOyMdys62IGLC9lDznwTu11xLg6wB9mV-60lr86A,3902 +paramiko/pkey.py,sha256=E3hegNR3eS16MMVGEW2v5f_5PBcKjNwqJ_by2HXvfdc,36719 +paramiko/primes.py,sha256=6Uv0fFsTmIJxInMqeNhryw9jrzvgNksKbA7ecBI0g5E,5107 +paramiko/proxy.py,sha256=I5XxN1aDren3Fw1f3SOoQLP4O9O7jeyey9meG6Og0q4,4648 +paramiko/rsakey.py,sha256=7xoDJvfcaZVVYRGlv8xamhO3zYvE-wI_Nd814L8TxzQ,7546 +paramiko/server.py,sha256=oNkI7t2gSMYIwLov5vl_BbHU-AwFC5LxP78YIXw7mq4,30457 +paramiko/sftp.py,sha256=pyZPnR0fv94YopfPDpslloTiYelu5GuM70cXUGOaKHM,6471 +paramiko/sftp_attr.py,sha256=AX-cG_FiPinftQQq8Ndo1Mc_bZz-AhXFQQpac-oV0wg,8258 +paramiko/sftp_client.py,sha256=e_zi6V233tjx3DH9TH7rRDKRO-TCZ_zyOkBw4sSRIjo,35855 +paramiko/sftp_file.py,sha256=NgVfDhxxURhFrEqniIJQgKQ6wlgCTgOVu5GwQczW_hk,21820 +paramiko/sftp_handle.py,sha256=ho-eyiEvhYHt-_VytznNzNeGktfaIsQX5l4bespWZAk,7424 +paramiko/sftp_server.py,sha256=yH-BgsYj7BuZNGn_EHpnLRPmoNGoYB9g_XxOlK4IcYA,19492 +paramiko/sftp_si.py,sha256=Uf90bFme6Jy6yl7k4jJ28IJboq6KiyPWLjXgP9DR6gk,12544 +paramiko/ssh_exception.py,sha256=F82_vTnKr3UF7ai8dTEv6PnqwVoREyk2c9_Bo3smsrg,7494 +paramiko/ssh_gss.py,sha256=BNhiDON1FOJB2P2VQUQHLYJ7RZhTbDjc7NPMqSNwH6Y,28713 +paramiko/transport.py,sha256=BuO3Ai0aaE61rQ5i_WZ7Y-ZYhJqsxIZl0bXDwi5pLKU,135414 +paramiko/util.py,sha256=7eEtwmxiST4Jj3HIqB7irz0SMofJlmy4yuYqda-rqPs,9494 +paramiko/win_openssh.py,sha256=DbWJT0hiE6UImAbMqehcGuVLDWIl-2rObe-AhaGuWpk,1918 +paramiko/win_pageant.py,sha256=i5TG472VzJKVnK08oxM4hK_qb9IzL_Fo96B8ouaxXHo,4177 diff --git a/lib/paramiko-4.0.0.dist-info/REQUESTED b/lib/paramiko-4.0.0.dist-info/REQUESTED new file mode 100644 index 0000000..e69de29 diff --git a/lib/paramiko-4.0.0.dist-info/WHEEL b/lib/paramiko-4.0.0.dist-info/WHEEL new file mode 100644 index 0000000..e7fa31b --- /dev/null +++ b/lib/paramiko-4.0.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/paramiko-4.0.0.dist-info/licenses/LICENSE b/lib/paramiko-4.0.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000..d12bef0 --- /dev/null +++ b/lib/paramiko-4.0.0.dist-info/licenses/LICENSE @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/lib/paramiko-4.0.0.dist-info/top_level.txt b/lib/paramiko-4.0.0.dist-info/top_level.txt new file mode 100644 index 0000000..8608c1b --- /dev/null +++ b/lib/paramiko-4.0.0.dist-info/top_level.txt @@ -0,0 +1 @@ +paramiko diff --git a/lib/paramiko/__init__.py b/lib/paramiko/__init__.py new file mode 100644 index 0000000..92ff86f --- /dev/null +++ b/lib/paramiko/__init__.py @@ -0,0 +1,120 @@ +# Copyright (C) 2003-2011 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from importlib import metadata + +__version__ = metadata.version("paramiko") + +# flake8: noqa +from paramiko.transport import ( + SecurityOptions, + ServiceRequestingTransport, + Transport, +) +from paramiko.client import ( + AutoAddPolicy, + MissingHostKeyPolicy, + RejectPolicy, + SSHClient, + WarningPolicy, +) +from paramiko.auth_handler import AuthHandler +from paramiko.auth_strategy import ( + AuthFailure, + AuthStrategy, + AuthResult, + AuthSource, + InMemoryPrivateKey, + NoneAuth, + OnDiskPrivateKey, + Password, + PrivateKey, + SourceResult, +) +from paramiko.ssh_gss import GSSAuth, GSS_AUTH_AVAILABLE, GSS_EXCEPTIONS +from paramiko.channel import ( + Channel, + ChannelFile, + ChannelStderrFile, + ChannelStdinFile, +) +from paramiko.ssh_exception import ( + AuthenticationException, + BadAuthenticationType, + BadHostKeyException, + ChannelException, + ConfigParseError, + CouldNotCanonicalize, + IncompatiblePeer, + MessageOrderError, + PasswordRequiredException, + ProxyCommandFailure, + SSHException, +) +from paramiko.server import ServerInterface, SubsystemHandler, InteractiveQuery +from paramiko.rsakey import RSAKey +from paramiko.ecdsakey import ECDSAKey +from paramiko.ed25519key import Ed25519Key +from paramiko.sftp import SFTPError, BaseSFTP +from paramiko.sftp_client import SFTP, SFTPClient +from paramiko.sftp_server import SFTPServer +from paramiko.sftp_attr import SFTPAttributes +from paramiko.sftp_handle import SFTPHandle +from paramiko.sftp_si import SFTPServerInterface +from paramiko.sftp_file import SFTPFile +from paramiko.message import Message +from paramiko.packet import Packetizer +from paramiko.file import BufferedFile +from paramiko.agent import Agent, AgentKey +from paramiko.pkey import PKey, PublicBlob, UnknownKeyType +from paramiko.hostkeys import HostKeys +from paramiko.config import SSHConfig, SSHConfigDict +from paramiko.proxy import ProxyCommand + +from paramiko.common import ( + AUTH_SUCCESSFUL, + AUTH_PARTIALLY_SUCCESSFUL, + AUTH_FAILED, + OPEN_SUCCEEDED, + OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, + OPEN_FAILED_CONNECT_FAILED, + OPEN_FAILED_UNKNOWN_CHANNEL_TYPE, + OPEN_FAILED_RESOURCE_SHORTAGE, +) + +from paramiko.sftp import ( + SFTP_OK, + SFTP_EOF, + SFTP_NO_SUCH_FILE, + SFTP_PERMISSION_DENIED, + SFTP_FAILURE, + SFTP_BAD_MESSAGE, + SFTP_NO_CONNECTION, + SFTP_CONNECTION_LOST, + SFTP_OP_UNSUPPORTED, +) + +from paramiko.common import io_sleep + + +# TODO: I guess a real plugin system might be nice for future expansion... +key_classes = [RSAKey, Ed25519Key, ECDSAKey] + + +__author__ = "Jeff Forcier " +__license__ = "GNU Lesser General Public License (LGPL)" diff --git a/lib/paramiko/__pycache__/__init__.cpython-314.pyc b/lib/paramiko/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..32c0a22 Binary files /dev/null and b/lib/paramiko/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/_winapi.cpython-314.pyc b/lib/paramiko/__pycache__/_winapi.cpython-314.pyc new file mode 100644 index 0000000..a9ce577 Binary files /dev/null and b/lib/paramiko/__pycache__/_winapi.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/agent.cpython-314.pyc b/lib/paramiko/__pycache__/agent.cpython-314.pyc new file mode 100644 index 0000000..39e861e Binary files /dev/null and b/lib/paramiko/__pycache__/agent.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/auth_handler.cpython-314.pyc b/lib/paramiko/__pycache__/auth_handler.cpython-314.pyc new file mode 100644 index 0000000..97c4fb1 Binary files /dev/null and b/lib/paramiko/__pycache__/auth_handler.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/auth_strategy.cpython-314.pyc b/lib/paramiko/__pycache__/auth_strategy.cpython-314.pyc new file mode 100644 index 0000000..cc7af7b Binary files /dev/null and b/lib/paramiko/__pycache__/auth_strategy.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/ber.cpython-314.pyc b/lib/paramiko/__pycache__/ber.cpython-314.pyc new file mode 100644 index 0000000..639c598 Binary files /dev/null and b/lib/paramiko/__pycache__/ber.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/buffered_pipe.cpython-314.pyc b/lib/paramiko/__pycache__/buffered_pipe.cpython-314.pyc new file mode 100644 index 0000000..409a87b Binary files /dev/null and b/lib/paramiko/__pycache__/buffered_pipe.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/channel.cpython-314.pyc b/lib/paramiko/__pycache__/channel.cpython-314.pyc new file mode 100644 index 0000000..23a4338 Binary files /dev/null and b/lib/paramiko/__pycache__/channel.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/client.cpython-314.pyc b/lib/paramiko/__pycache__/client.cpython-314.pyc new file mode 100644 index 0000000..ea5e818 Binary files /dev/null and b/lib/paramiko/__pycache__/client.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/common.cpython-314.pyc b/lib/paramiko/__pycache__/common.cpython-314.pyc new file mode 100644 index 0000000..ce9de59 Binary files /dev/null and b/lib/paramiko/__pycache__/common.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/compress.cpython-314.pyc b/lib/paramiko/__pycache__/compress.cpython-314.pyc new file mode 100644 index 0000000..65df4b7 Binary files /dev/null and b/lib/paramiko/__pycache__/compress.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/config.cpython-314.pyc b/lib/paramiko/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000..ed978a9 Binary files /dev/null and b/lib/paramiko/__pycache__/config.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/ecdsakey.cpython-314.pyc b/lib/paramiko/__pycache__/ecdsakey.cpython-314.pyc new file mode 100644 index 0000000..d0a43f4 Binary files /dev/null and b/lib/paramiko/__pycache__/ecdsakey.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/ed25519key.cpython-314.pyc b/lib/paramiko/__pycache__/ed25519key.cpython-314.pyc new file mode 100644 index 0000000..5cbf994 Binary files /dev/null and b/lib/paramiko/__pycache__/ed25519key.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/file.cpython-314.pyc b/lib/paramiko/__pycache__/file.cpython-314.pyc new file mode 100644 index 0000000..d3dbae0 Binary files /dev/null and b/lib/paramiko/__pycache__/file.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/hostkeys.cpython-314.pyc b/lib/paramiko/__pycache__/hostkeys.cpython-314.pyc new file mode 100644 index 0000000..9f51384 Binary files /dev/null and b/lib/paramiko/__pycache__/hostkeys.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_curve25519.cpython-314.pyc b/lib/paramiko/__pycache__/kex_curve25519.cpython-314.pyc new file mode 100644 index 0000000..42ca82d Binary files /dev/null and b/lib/paramiko/__pycache__/kex_curve25519.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_ecdh_nist.cpython-314.pyc b/lib/paramiko/__pycache__/kex_ecdh_nist.cpython-314.pyc new file mode 100644 index 0000000..881db99 Binary files /dev/null and b/lib/paramiko/__pycache__/kex_ecdh_nist.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_gex.cpython-314.pyc b/lib/paramiko/__pycache__/kex_gex.cpython-314.pyc new file mode 100644 index 0000000..c50b0b6 Binary files /dev/null and b/lib/paramiko/__pycache__/kex_gex.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_group1.cpython-314.pyc b/lib/paramiko/__pycache__/kex_group1.cpython-314.pyc new file mode 100644 index 0000000..375cd55 Binary files /dev/null and b/lib/paramiko/__pycache__/kex_group1.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_group14.cpython-314.pyc b/lib/paramiko/__pycache__/kex_group14.cpython-314.pyc new file mode 100644 index 0000000..919e64d Binary files /dev/null and b/lib/paramiko/__pycache__/kex_group14.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_group16.cpython-314.pyc b/lib/paramiko/__pycache__/kex_group16.cpython-314.pyc new file mode 100644 index 0000000..a80cc9e Binary files /dev/null and b/lib/paramiko/__pycache__/kex_group16.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/kex_gss.cpython-314.pyc b/lib/paramiko/__pycache__/kex_gss.cpython-314.pyc new file mode 100644 index 0000000..19919ad Binary files /dev/null and b/lib/paramiko/__pycache__/kex_gss.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/message.cpython-314.pyc b/lib/paramiko/__pycache__/message.cpython-314.pyc new file mode 100644 index 0000000..13f496f Binary files /dev/null and b/lib/paramiko/__pycache__/message.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/packet.cpython-314.pyc b/lib/paramiko/__pycache__/packet.cpython-314.pyc new file mode 100644 index 0000000..cd912e2 Binary files /dev/null and b/lib/paramiko/__pycache__/packet.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/pipe.cpython-314.pyc b/lib/paramiko/__pycache__/pipe.cpython-314.pyc new file mode 100644 index 0000000..c942c2e Binary files /dev/null and b/lib/paramiko/__pycache__/pipe.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/pkey.cpython-314.pyc b/lib/paramiko/__pycache__/pkey.cpython-314.pyc new file mode 100644 index 0000000..a3de1e0 Binary files /dev/null and b/lib/paramiko/__pycache__/pkey.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/primes.cpython-314.pyc b/lib/paramiko/__pycache__/primes.cpython-314.pyc new file mode 100644 index 0000000..1f9a991 Binary files /dev/null and b/lib/paramiko/__pycache__/primes.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/proxy.cpython-314.pyc b/lib/paramiko/__pycache__/proxy.cpython-314.pyc new file mode 100644 index 0000000..6a8d0a9 Binary files /dev/null and b/lib/paramiko/__pycache__/proxy.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/rsakey.cpython-314.pyc b/lib/paramiko/__pycache__/rsakey.cpython-314.pyc new file mode 100644 index 0000000..dc45380 Binary files /dev/null and b/lib/paramiko/__pycache__/rsakey.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/server.cpython-314.pyc b/lib/paramiko/__pycache__/server.cpython-314.pyc new file mode 100644 index 0000000..75b4985 Binary files /dev/null and b/lib/paramiko/__pycache__/server.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp.cpython-314.pyc b/lib/paramiko/__pycache__/sftp.cpython-314.pyc new file mode 100644 index 0000000..dfacc5f Binary files /dev/null and b/lib/paramiko/__pycache__/sftp.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp_attr.cpython-314.pyc b/lib/paramiko/__pycache__/sftp_attr.cpython-314.pyc new file mode 100644 index 0000000..e596576 Binary files /dev/null and b/lib/paramiko/__pycache__/sftp_attr.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp_client.cpython-314.pyc b/lib/paramiko/__pycache__/sftp_client.cpython-314.pyc new file mode 100644 index 0000000..3f9e7aa Binary files /dev/null and b/lib/paramiko/__pycache__/sftp_client.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp_file.cpython-314.pyc b/lib/paramiko/__pycache__/sftp_file.cpython-314.pyc new file mode 100644 index 0000000..dd7e83c Binary files /dev/null and b/lib/paramiko/__pycache__/sftp_file.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp_handle.cpython-314.pyc b/lib/paramiko/__pycache__/sftp_handle.cpython-314.pyc new file mode 100644 index 0000000..80d7a6e Binary files /dev/null and b/lib/paramiko/__pycache__/sftp_handle.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp_server.cpython-314.pyc b/lib/paramiko/__pycache__/sftp_server.cpython-314.pyc new file mode 100644 index 0000000..0a0e5a1 Binary files /dev/null and b/lib/paramiko/__pycache__/sftp_server.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/sftp_si.cpython-314.pyc b/lib/paramiko/__pycache__/sftp_si.cpython-314.pyc new file mode 100644 index 0000000..3122e2d Binary files /dev/null and b/lib/paramiko/__pycache__/sftp_si.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/ssh_exception.cpython-314.pyc b/lib/paramiko/__pycache__/ssh_exception.cpython-314.pyc new file mode 100644 index 0000000..cdfe93d Binary files /dev/null and b/lib/paramiko/__pycache__/ssh_exception.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/ssh_gss.cpython-314.pyc b/lib/paramiko/__pycache__/ssh_gss.cpython-314.pyc new file mode 100644 index 0000000..ef1ccb6 Binary files /dev/null and b/lib/paramiko/__pycache__/ssh_gss.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/transport.cpython-314.pyc b/lib/paramiko/__pycache__/transport.cpython-314.pyc new file mode 100644 index 0000000..324ddf2 Binary files /dev/null and b/lib/paramiko/__pycache__/transport.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/util.cpython-314.pyc b/lib/paramiko/__pycache__/util.cpython-314.pyc new file mode 100644 index 0000000..8a0b0a1 Binary files /dev/null and b/lib/paramiko/__pycache__/util.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/win_openssh.cpython-314.pyc b/lib/paramiko/__pycache__/win_openssh.cpython-314.pyc new file mode 100644 index 0000000..dfa0eb9 Binary files /dev/null and b/lib/paramiko/__pycache__/win_openssh.cpython-314.pyc differ diff --git a/lib/paramiko/__pycache__/win_pageant.cpython-314.pyc b/lib/paramiko/__pycache__/win_pageant.cpython-314.pyc new file mode 100644 index 0000000..06acccf Binary files /dev/null and b/lib/paramiko/__pycache__/win_pageant.cpython-314.pyc differ diff --git a/lib/paramiko/_winapi.py b/lib/paramiko/_winapi.py new file mode 100644 index 0000000..4295457 --- /dev/null +++ b/lib/paramiko/_winapi.py @@ -0,0 +1,413 @@ +""" +Windows API functions implemented as ctypes functions and classes as found +in jaraco.windows (3.4.1). + +If you encounter issues with this module, please consider reporting the issues +in jaraco.windows and asking the author to port the fixes back here. +""" + +import builtins +import ctypes.wintypes + +from paramiko.util import u + + +###################### +# jaraco.windows.error + + +def format_system_message(errno): + """ + Call FormatMessage with a system error number to retrieve + the descriptive error message. + """ + # first some flags used by FormatMessageW + ALLOCATE_BUFFER = 0x100 + FROM_SYSTEM = 0x1000 + + # Let FormatMessageW allocate the buffer (we'll free it below) + # Also, let it know we want a system error message. + flags = ALLOCATE_BUFFER | FROM_SYSTEM + source = None + message_id = errno + language_id = 0 + result_buffer = ctypes.wintypes.LPWSTR() + buffer_size = 0 + arguments = None + bytes = ctypes.windll.kernel32.FormatMessageW( + flags, + source, + message_id, + language_id, + ctypes.byref(result_buffer), + buffer_size, + arguments, + ) + # note the following will cause an infinite loop if GetLastError + # repeatedly returns an error that cannot be formatted, although + # this should not happen. + handle_nonzero_success(bytes) + message = result_buffer.value + ctypes.windll.kernel32.LocalFree(result_buffer) + return message + + +class WindowsError(builtins.WindowsError): + """more info about errors at + http://msdn.microsoft.com/en-us/library/ms681381(VS.85).aspx""" + + def __init__(self, value=None): + if value is None: + value = ctypes.windll.kernel32.GetLastError() + strerror = format_system_message(value) + args = 0, strerror, None, value + super().__init__(*args) + + @property + def message(self): + return self.strerror + + @property + def code(self): + return self.winerror + + def __str__(self): + return self.message + + def __repr__(self): + return "{self.__class__.__name__}({self.winerror})".format(**vars()) + + +def handle_nonzero_success(result): + if result == 0: + raise WindowsError() + + +########################### +# jaraco.windows.api.memory + +GMEM_MOVEABLE = 0x2 + +GlobalAlloc = ctypes.windll.kernel32.GlobalAlloc +GlobalAlloc.argtypes = ctypes.wintypes.UINT, ctypes.c_size_t +GlobalAlloc.restype = ctypes.wintypes.HANDLE + +GlobalLock = ctypes.windll.kernel32.GlobalLock +GlobalLock.argtypes = (ctypes.wintypes.HGLOBAL,) +GlobalLock.restype = ctypes.wintypes.LPVOID + +GlobalUnlock = ctypes.windll.kernel32.GlobalUnlock +GlobalUnlock.argtypes = (ctypes.wintypes.HGLOBAL,) +GlobalUnlock.restype = ctypes.wintypes.BOOL + +GlobalSize = ctypes.windll.kernel32.GlobalSize +GlobalSize.argtypes = (ctypes.wintypes.HGLOBAL,) +GlobalSize.restype = ctypes.c_size_t + +CreateFileMapping = ctypes.windll.kernel32.CreateFileMappingW +CreateFileMapping.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.c_void_p, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.LPWSTR, +] +CreateFileMapping.restype = ctypes.wintypes.HANDLE + +MapViewOfFile = ctypes.windll.kernel32.MapViewOfFile +MapViewOfFile.restype = ctypes.wintypes.HANDLE + +UnmapViewOfFile = ctypes.windll.kernel32.UnmapViewOfFile +UnmapViewOfFile.argtypes = (ctypes.wintypes.HANDLE,) + +RtlMoveMemory = ctypes.windll.kernel32.RtlMoveMemory +RtlMoveMemory.argtypes = (ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t) + +ctypes.windll.kernel32.LocalFree.argtypes = (ctypes.wintypes.HLOCAL,) + +##################### +# jaraco.windows.mmap + + +class MemoryMap: + """ + A memory map object which can have security attributes overridden. + """ + + def __init__(self, name, length, security_attributes=None): + self.name = name + self.length = length + self.security_attributes = security_attributes + self.pos = 0 + + def __enter__(self): + p_SA = ( + ctypes.byref(self.security_attributes) + if self.security_attributes + else None + ) + INVALID_HANDLE_VALUE = -1 + PAGE_READWRITE = 0x4 + FILE_MAP_WRITE = 0x2 + filemap = ctypes.windll.kernel32.CreateFileMappingW( + INVALID_HANDLE_VALUE, + p_SA, + PAGE_READWRITE, + 0, + self.length, + u(self.name), + ) + handle_nonzero_success(filemap) + if filemap == INVALID_HANDLE_VALUE: + raise Exception("Failed to create file mapping") + self.filemap = filemap + self.view = MapViewOfFile(filemap, FILE_MAP_WRITE, 0, 0, 0) + return self + + def seek(self, pos): + self.pos = pos + + def write(self, msg): + assert isinstance(msg, bytes) + n = len(msg) + if self.pos + n >= self.length: # A little safety. + raise ValueError(f"Refusing to write {n} bytes") + dest = self.view + self.pos + length = ctypes.c_size_t(n) + ctypes.windll.kernel32.RtlMoveMemory(dest, msg, length) + self.pos += n + + def read(self, n): + """ + Read n bytes from mapped view. + """ + out = ctypes.create_string_buffer(n) + source = self.view + self.pos + length = ctypes.c_size_t(n) + ctypes.windll.kernel32.RtlMoveMemory(out, source, length) + self.pos += n + return out.raw + + def __exit__(self, exc_type, exc_val, tb): + ctypes.windll.kernel32.UnmapViewOfFile(self.view) + ctypes.windll.kernel32.CloseHandle(self.filemap) + + +############################# +# jaraco.windows.api.security + +# from WinNT.h +READ_CONTROL = 0x00020000 +STANDARD_RIGHTS_REQUIRED = 0x000F0000 +STANDARD_RIGHTS_READ = READ_CONTROL +STANDARD_RIGHTS_WRITE = READ_CONTROL +STANDARD_RIGHTS_EXECUTE = READ_CONTROL +STANDARD_RIGHTS_ALL = 0x001F0000 + +# from NTSecAPI.h +POLICY_VIEW_LOCAL_INFORMATION = 0x00000001 +POLICY_VIEW_AUDIT_INFORMATION = 0x00000002 +POLICY_GET_PRIVATE_INFORMATION = 0x00000004 +POLICY_TRUST_ADMIN = 0x00000008 +POLICY_CREATE_ACCOUNT = 0x00000010 +POLICY_CREATE_SECRET = 0x00000020 +POLICY_CREATE_PRIVILEGE = 0x00000040 +POLICY_SET_DEFAULT_QUOTA_LIMITS = 0x00000080 +POLICY_SET_AUDIT_REQUIREMENTS = 0x00000100 +POLICY_AUDIT_LOG_ADMIN = 0x00000200 +POLICY_SERVER_ADMIN = 0x00000400 +POLICY_LOOKUP_NAMES = 0x00000800 +POLICY_NOTIFICATION = 0x00001000 + +POLICY_ALL_ACCESS = ( + STANDARD_RIGHTS_REQUIRED + | POLICY_VIEW_LOCAL_INFORMATION + | POLICY_VIEW_AUDIT_INFORMATION + | POLICY_GET_PRIVATE_INFORMATION + | POLICY_TRUST_ADMIN + | POLICY_CREATE_ACCOUNT + | POLICY_CREATE_SECRET + | POLICY_CREATE_PRIVILEGE + | POLICY_SET_DEFAULT_QUOTA_LIMITS + | POLICY_SET_AUDIT_REQUIREMENTS + | POLICY_AUDIT_LOG_ADMIN + | POLICY_SERVER_ADMIN + | POLICY_LOOKUP_NAMES +) + + +POLICY_READ = ( + STANDARD_RIGHTS_READ + | POLICY_VIEW_AUDIT_INFORMATION + | POLICY_GET_PRIVATE_INFORMATION +) + +POLICY_WRITE = ( + STANDARD_RIGHTS_WRITE + | POLICY_TRUST_ADMIN + | POLICY_CREATE_ACCOUNT + | POLICY_CREATE_SECRET + | POLICY_CREATE_PRIVILEGE + | POLICY_SET_DEFAULT_QUOTA_LIMITS + | POLICY_SET_AUDIT_REQUIREMENTS + | POLICY_AUDIT_LOG_ADMIN + | POLICY_SERVER_ADMIN +) + +POLICY_EXECUTE = ( + STANDARD_RIGHTS_EXECUTE + | POLICY_VIEW_LOCAL_INFORMATION + | POLICY_LOOKUP_NAMES +) + + +class TokenAccess: + TOKEN_QUERY = 0x8 + + +class TokenInformationClass: + TokenUser = 1 + + +class TOKEN_USER(ctypes.Structure): + num = 1 + _fields_ = [ + ("SID", ctypes.c_void_p), + ("ATTRIBUTES", ctypes.wintypes.DWORD), + ] + + +class SECURITY_DESCRIPTOR(ctypes.Structure): + """ + typedef struct _SECURITY_DESCRIPTOR + { + UCHAR Revision; + UCHAR Sbz1; + SECURITY_DESCRIPTOR_CONTROL Control; + PSID Owner; + PSID Group; + PACL Sacl; + PACL Dacl; + } SECURITY_DESCRIPTOR; + """ + + SECURITY_DESCRIPTOR_CONTROL = ctypes.wintypes.USHORT + REVISION = 1 + + _fields_ = [ + ("Revision", ctypes.c_ubyte), + ("Sbz1", ctypes.c_ubyte), + ("Control", SECURITY_DESCRIPTOR_CONTROL), + ("Owner", ctypes.c_void_p), + ("Group", ctypes.c_void_p), + ("Sacl", ctypes.c_void_p), + ("Dacl", ctypes.c_void_p), + ] + + +class SECURITY_ATTRIBUTES(ctypes.Structure): + """ + typedef struct _SECURITY_ATTRIBUTES { + DWORD nLength; + LPVOID lpSecurityDescriptor; + BOOL bInheritHandle; + } SECURITY_ATTRIBUTES; + """ + + _fields_ = [ + ("nLength", ctypes.wintypes.DWORD), + ("lpSecurityDescriptor", ctypes.c_void_p), + ("bInheritHandle", ctypes.wintypes.BOOL), + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.nLength = ctypes.sizeof(SECURITY_ATTRIBUTES) + + @property + def descriptor(self): + return self._descriptor + + @descriptor.setter + def descriptor(self, value): + self._descriptor = value + self.lpSecurityDescriptor = ctypes.addressof(value) + + +ctypes.windll.advapi32.SetSecurityDescriptorOwner.argtypes = ( + ctypes.POINTER(SECURITY_DESCRIPTOR), + ctypes.c_void_p, + ctypes.wintypes.BOOL, +) + +######################### +# jaraco.windows.security + + +def GetTokenInformation(token, information_class): + """ + Given a token, get the token information for it. + """ + data_size = ctypes.wintypes.DWORD() + ctypes.windll.advapi32.GetTokenInformation( + token, information_class.num, 0, 0, ctypes.byref(data_size) + ) + data = ctypes.create_string_buffer(data_size.value) + handle_nonzero_success( + ctypes.windll.advapi32.GetTokenInformation( + token, + information_class.num, + ctypes.byref(data), + ctypes.sizeof(data), + ctypes.byref(data_size), + ) + ) + return ctypes.cast(data, ctypes.POINTER(TOKEN_USER)).contents + + +def OpenProcessToken(proc_handle, access): + result = ctypes.wintypes.HANDLE() + proc_handle = ctypes.wintypes.HANDLE(proc_handle) + handle_nonzero_success( + ctypes.windll.advapi32.OpenProcessToken( + proc_handle, access, ctypes.byref(result) + ) + ) + return result + + +def get_current_user(): + """ + Return a TOKEN_USER for the owner of this process. + """ + process = OpenProcessToken( + ctypes.windll.kernel32.GetCurrentProcess(), TokenAccess.TOKEN_QUERY + ) + return GetTokenInformation(process, TOKEN_USER) + + +def get_security_attributes_for_user(user=None): + """ + Return a SECURITY_ATTRIBUTES structure with the SID set to the + specified user (uses current user if none is specified). + """ + if user is None: + user = get_current_user() + + assert isinstance(user, TOKEN_USER), "user must be TOKEN_USER instance" + + SD = SECURITY_DESCRIPTOR() + SA = SECURITY_ATTRIBUTES() + # by attaching the actual security descriptor, it will be garbage- + # collected with the security attributes + SA.descriptor = SD + SA.bInheritHandle = 1 + + ctypes.windll.advapi32.InitializeSecurityDescriptor( + ctypes.byref(SD), SECURITY_DESCRIPTOR.REVISION + ) + ctypes.windll.advapi32.SetSecurityDescriptorOwner( + ctypes.byref(SD), user.SID, 0 + ) + return SA diff --git a/lib/paramiko/agent.py b/lib/paramiko/agent.py new file mode 100644 index 0000000..b29a0d1 --- /dev/null +++ b/lib/paramiko/agent.py @@ -0,0 +1,497 @@ +# Copyright (C) 2003-2007 John Rochester +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +SSH Agent interface +""" + +import os +import socket +import struct +import sys +import threading +import time +import tempfile +import stat +from logging import DEBUG +from select import select +from paramiko.common import io_sleep, byte_chr + +from paramiko.ssh_exception import SSHException, AuthenticationException +from paramiko.message import Message +from paramiko.pkey import PKey, UnknownKeyType +from paramiko.util import asbytes, get_logger + +cSSH2_AGENTC_REQUEST_IDENTITIES = byte_chr(11) +SSH2_AGENT_IDENTITIES_ANSWER = 12 +cSSH2_AGENTC_SIGN_REQUEST = byte_chr(13) +SSH2_AGENT_SIGN_RESPONSE = 14 + +SSH_AGENT_RSA_SHA2_256 = 2 +SSH_AGENT_RSA_SHA2_512 = 4 +# NOTE: RFC mildly confusing; while these flags are OR'd together, OpenSSH at +# least really treats them like "AND"s, in the sense that if it finds the +# SHA256 flag set it won't continue looking at the SHA512 one; it +# short-circuits right away. +# Thus, we never want to eg submit 6 to say "either's good". +ALGORITHM_FLAG_MAP = { + "rsa-sha2-256": SSH_AGENT_RSA_SHA2_256, + "rsa-sha2-512": SSH_AGENT_RSA_SHA2_512, +} +for key, value in list(ALGORITHM_FLAG_MAP.items()): + ALGORITHM_FLAG_MAP[f"{key}-cert-v01@openssh.com"] = value + + +# TODO 4.0: rename all these - including making some of their methods public? +class AgentSSH: + def __init__(self): + self._conn = None + self._keys = () + + def get_keys(self): + """ + Return the list of keys available through the SSH agent, if any. If + no SSH agent was running (or it couldn't be contacted), an empty list + will be returned. + + This method performs no IO, just returns the list of keys retrieved + when the connection was made. + + :return: + a tuple of `.AgentKey` objects representing keys available on the + SSH agent + """ + return self._keys + + def _connect(self, conn): + self._conn = conn + ptype, result = self._send_message(cSSH2_AGENTC_REQUEST_IDENTITIES) + if ptype != SSH2_AGENT_IDENTITIES_ANSWER: + raise SSHException("could not get keys from ssh-agent") + keys = [] + for i in range(result.get_int()): + keys.append( + AgentKey( + agent=self, + blob=result.get_binary(), + comment=result.get_text(), + ) + ) + self._keys = tuple(keys) + + def _close(self): + if self._conn is not None: + self._conn.close() + self._conn = None + self._keys = () + + def _send_message(self, msg): + msg = asbytes(msg) + self._conn.send(struct.pack(">I", len(msg)) + msg) + data = self._read_all(4) + msg = Message(self._read_all(struct.unpack(">I", data)[0])) + return ord(msg.get_byte()), msg + + def _read_all(self, wanted): + result = self._conn.recv(wanted) + while len(result) < wanted: + if len(result) == 0: + raise SSHException("lost ssh-agent") + extra = self._conn.recv(wanted - len(result)) + if len(extra) == 0: + raise SSHException("lost ssh-agent") + result += extra + return result + + +class AgentProxyThread(threading.Thread): + """ + Class in charge of communication between two channels. + """ + + def __init__(self, agent): + threading.Thread.__init__(self, target=self.run) + self._agent = agent + self._exit = False + + def run(self): + try: + (r, addr) = self.get_connection() + # Found that r should be either + # a socket from the socket library or None + self.__inr = r + # The address should be an IP address as a string? or None + self.__addr = addr + self._agent.connect() + if not isinstance(self._agent, int) and ( + self._agent._conn is None + or not hasattr(self._agent._conn, "fileno") + ): + raise AuthenticationException("Unable to connect to SSH agent") + self._communicate() + except: + # XXX Not sure what to do here ... raise or pass ? + raise + + def _communicate(self): + import fcntl + + oldflags = fcntl.fcntl(self.__inr, fcntl.F_GETFL) + fcntl.fcntl(self.__inr, fcntl.F_SETFL, oldflags | os.O_NONBLOCK) + while not self._exit: + events = select([self._agent._conn, self.__inr], [], [], 0.5) + for fd in events[0]: + if self._agent._conn == fd: + data = self._agent._conn.recv(512) + if len(data) != 0: + self.__inr.send(data) + else: + self._close() + break + elif self.__inr == fd: + data = self.__inr.recv(512) + if len(data) != 0: + self._agent._conn.send(data) + else: + self._close() + break + time.sleep(io_sleep) + + def _close(self): + self._exit = True + self.__inr.close() + self._agent._conn.close() + + +class AgentLocalProxy(AgentProxyThread): + """ + Class to be used when wanting to ask a local SSH Agent being + asked from a remote fake agent (so use a unix socket for ex.) + """ + + def __init__(self, agent): + AgentProxyThread.__init__(self, agent) + + def get_connection(self): + """ + Return a pair of socket object and string address. + + May block! + """ + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + conn.bind(self._agent._get_filename()) + conn.listen(1) + (r, addr) = conn.accept() + return r, addr + except: + raise + + +class AgentRemoteProxy(AgentProxyThread): + """ + Class to be used when wanting to ask a remote SSH Agent + """ + + def __init__(self, agent, chan): + AgentProxyThread.__init__(self, agent) + self.__chan = chan + + def get_connection(self): + return self.__chan, None + + +def get_agent_connection(): + """ + Returns some SSH agent object, or None if none were found/supported. + + .. versionadded:: 2.10 + """ + if ("SSH_AUTH_SOCK" in os.environ) and (sys.platform != "win32"): + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + conn.connect(os.environ["SSH_AUTH_SOCK"]) + return conn + except: + # probably a dangling env var: the ssh agent is gone + return + elif sys.platform == "win32": + from . import win_pageant, win_openssh + + conn = None + if win_pageant.can_talk_to_agent(): + conn = win_pageant.PageantConnection() + elif win_openssh.can_talk_to_agent(): + conn = win_openssh.OpenSSHAgentConnection() + return conn + else: + # no agent support + return + + +class AgentClientProxy: + """ + Class proxying request as a client: + + #. client ask for a request_forward_agent() + #. server creates a proxy and a fake SSH Agent + #. server ask for establishing a connection when needed, + calling the forward_agent_handler at client side. + #. the forward_agent_handler launch a thread for connecting + the remote fake agent and the local agent + #. Communication occurs ... + """ + + def __init__(self, chanRemote): + self._conn = None + self.__chanR = chanRemote + self.thread = AgentRemoteProxy(self, chanRemote) + self.thread.start() + + def __del__(self): + self.close() + + def connect(self): + """ + Method automatically called by ``AgentProxyThread.run``. + """ + conn = get_agent_connection() + if not conn: + return + self._conn = conn + + def close(self): + """ + Close the current connection and terminate the agent + Should be called manually + """ + if hasattr(self, "thread"): + self.thread._exit = True + self.thread.join(1000) + if self._conn is not None: + self._conn.close() + + +class AgentServerProxy(AgentSSH): + """ + Allows an SSH server to access a forwarded agent. + + This also creates a unix domain socket on the system to allow external + programs to also access the agent. For this reason, you probably only want + to create one of these. + + :meth:`connect` must be called before it is usable. This will also load the + list of keys the agent contains. You must also call :meth:`close` in + order to clean up the unix socket and the thread that maintains it. + (:class:`contextlib.closing` might be helpful to you.) + + :param .Transport t: Transport used for SSH Agent communication forwarding + + :raises: `.SSHException` -- mostly if we lost the agent + """ + + def __init__(self, t): + AgentSSH.__init__(self) + self.__t = t + self._dir = tempfile.mkdtemp("sshproxy") + os.chmod(self._dir, stat.S_IRWXU) + self._file = self._dir + "/sshproxy.ssh" + self.thread = AgentLocalProxy(self) + self.thread.start() + + def __del__(self): + self.close() + + def connect(self): + conn_sock = self.__t.open_forward_agent_channel() + if conn_sock is None: + raise SSHException("lost ssh-agent") + conn_sock.set_name("auth-agent") + self._connect(conn_sock) + + def close(self): + """ + Terminate the agent, clean the files, close connections + Should be called manually + """ + os.remove(self._file) + os.rmdir(self._dir) + self.thread._exit = True + self.thread.join(1000) + self._close() + + def get_env(self): + """ + Helper for the environment under unix + + :return: + a dict containing the ``SSH_AUTH_SOCK`` environment variables + """ + return {"SSH_AUTH_SOCK": self._get_filename()} + + def _get_filename(self): + return self._file + + +class AgentRequestHandler: + """ + Primary/default implementation of SSH agent forwarding functionality. + + Simply instantiate this class, handing it a live command-executing session + object, and it will handle forwarding any local SSH agent processes it + finds. + + For example:: + + # Connect + client = SSHClient() + client.connect(host, port, username) + # Obtain session + session = client.get_transport().open_session() + # Forward local agent + AgentRequestHandler(session) + # Commands executed after this point will see the forwarded agent on + # the remote end. + session.exec_command("git clone https://my.git.repository/") + """ + + def __init__(self, chanClient): + self._conn = None + self.__chanC = chanClient + chanClient.request_forward_agent(self._forward_agent_handler) + self.__clientProxys = [] + + def _forward_agent_handler(self, chanRemote): + self.__clientProxys.append(AgentClientProxy(chanRemote)) + + def __del__(self): + self.close() + + def close(self): + for p in self.__clientProxys: + p.close() + + +class Agent(AgentSSH): + """ + Client interface for using private keys from an SSH agent running on the + local machine. If an SSH agent is running, this class can be used to + connect to it and retrieve `.PKey` objects which can be used when + attempting to authenticate to remote SSH servers. + + Upon initialization, a session with the local machine's SSH agent is + opened, if one is running. If no agent is running, initialization will + succeed, but `get_keys` will return an empty tuple. + + :raises: `.SSHException` -- + if an SSH agent is found, but speaks an incompatible protocol + + .. versionchanged:: 2.10 + Added support for native openssh agent on windows (extending previous + putty pageant support) + """ + + def __init__(self): + AgentSSH.__init__(self) + + conn = get_agent_connection() + if not conn: + return + self._connect(conn) + + def close(self): + """ + Close the SSH agent connection. + """ + self._close() + + +class AgentKey(PKey): + """ + Private key held in a local SSH agent. This type of key can be used for + authenticating to a remote server (signing). Most other key operations + work as expected. + + .. versionchanged:: 3.2 + Added the ``comment`` kwarg and attribute. + + .. versionchanged:: 3.2 + Added the ``.inner_key`` attribute holding a reference to the 'real' + key instance this key is a proxy for, if one was obtainable, else None. + """ + + def __init__(self, agent, blob, comment=""): + self.agent = agent + self.blob = blob + self.comment = comment + msg = Message(blob) + self.name = msg.get_text() + self._logger = get_logger(__file__) + self.inner_key = None + try: + self.inner_key = PKey.from_type_string( + key_type=self.name, key_bytes=blob + ) + except UnknownKeyType: + # Log, but don't explode, since inner_key is a best-effort thing. + err = "Unable to derive inner_key for agent key of type {!r}" + self.log(DEBUG, err.format(self.name)) + + def log(self, *args, **kwargs): + return self._logger.log(*args, **kwargs) + + def asbytes(self): + # Prefer inner_key.asbytes, since that will differ for eg RSA-CERT + return self.inner_key.asbytes() if self.inner_key else self.blob + + def get_name(self): + return self.name + + def get_bits(self): + # Have to work around PKey's default get_bits being crap + if self.inner_key is not None: + return self.inner_key.get_bits() + return super().get_bits() + + def __getattr__(self, name): + """ + Proxy any un-implemented methods/properties to the inner_key. + """ + if self.inner_key is None: # nothing to proxy to + raise AttributeError(name) + return getattr(self.inner_key, name) + + @property + def _fields(self): + fallback = [self.get_name(), self.blob] + return self.inner_key._fields if self.inner_key else fallback + + def sign_ssh_data(self, data, algorithm=None): + msg = Message() + msg.add_byte(cSSH2_AGENTC_SIGN_REQUEST) + # NOTE: this used to be just self.blob, which is not entirely right for + # RSA-CERT 'keys' - those end up always degrading to ssh-rsa type + # signatures, for reasons probably internal to OpenSSH's agent code, + # even if everything else wants SHA2 (including our flag map). + msg.add_string(self.asbytes()) + msg.add_string(data) + msg.add_int(ALGORITHM_FLAG_MAP.get(algorithm, 0)) + ptype, result = self.agent._send_message(msg) + if ptype != SSH2_AGENT_SIGN_RESPONSE: + raise SSHException("key cannot be used for signing") + return result.get_binary() diff --git a/lib/paramiko/auth_handler.py b/lib/paramiko/auth_handler.py new file mode 100644 index 0000000..bc7f298 --- /dev/null +++ b/lib/paramiko/auth_handler.py @@ -0,0 +1,1092 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +`.AuthHandler` +""" + +import weakref +import threading +import time +import re + +from paramiko.common import ( + cMSG_SERVICE_REQUEST, + cMSG_DISCONNECT, + DISCONNECT_SERVICE_NOT_AVAILABLE, + DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, + cMSG_USERAUTH_REQUEST, + cMSG_SERVICE_ACCEPT, + DEBUG, + AUTH_SUCCESSFUL, + INFO, + cMSG_USERAUTH_SUCCESS, + cMSG_USERAUTH_FAILURE, + AUTH_PARTIALLY_SUCCESSFUL, + cMSG_USERAUTH_INFO_REQUEST, + WARNING, + AUTH_FAILED, + cMSG_USERAUTH_PK_OK, + cMSG_USERAUTH_INFO_RESPONSE, + MSG_SERVICE_REQUEST, + MSG_SERVICE_ACCEPT, + MSG_USERAUTH_REQUEST, + MSG_USERAUTH_SUCCESS, + MSG_USERAUTH_FAILURE, + MSG_USERAUTH_BANNER, + MSG_USERAUTH_INFO_REQUEST, + MSG_USERAUTH_INFO_RESPONSE, + cMSG_USERAUTH_GSSAPI_RESPONSE, + cMSG_USERAUTH_GSSAPI_TOKEN, + cMSG_USERAUTH_GSSAPI_MIC, + MSG_USERAUTH_GSSAPI_RESPONSE, + MSG_USERAUTH_GSSAPI_TOKEN, + MSG_USERAUTH_GSSAPI_ERROR, + MSG_USERAUTH_GSSAPI_ERRTOK, + MSG_USERAUTH_GSSAPI_MIC, + MSG_NAMES, + cMSG_USERAUTH_BANNER, +) +from paramiko.message import Message +from paramiko.util import b, u +from paramiko.ssh_exception import ( + SSHException, + AuthenticationException, + BadAuthenticationType, + PartialAuthentication, +) +from paramiko.server import InteractiveQuery +from paramiko.ssh_gss import GSSAuth, GSS_EXCEPTIONS + + +class AuthHandler: + """ + Internal class to handle the mechanics of authentication. + """ + + def __init__(self, transport): + self.transport = weakref.proxy(transport) + self.username = None + self.authenticated = False + self.auth_event = None + self.auth_method = "" + self.banner = None + self.password = None + self.private_key = None + self.interactive_handler = None + self.submethods = None + # for server mode: + self.auth_username = None + self.auth_fail_count = 0 + # for GSSAPI + self.gss_host = None + self.gss_deleg_creds = True + + def _log(self, *args): + return self.transport._log(*args) + + def is_authenticated(self): + return self.authenticated + + def get_username(self): + if self.transport.server_mode: + return self.auth_username + else: + return self.username + + def auth_none(self, username, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = "none" + self.username = username + self._request_auth() + finally: + self.transport.lock.release() + + def auth_publickey(self, username, key, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = "publickey" + self.username = username + self.private_key = key + self._request_auth() + finally: + self.transport.lock.release() + + def auth_password(self, username, password, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = "password" + self.username = username + self.password = password + self._request_auth() + finally: + self.transport.lock.release() + + def auth_interactive(self, username, handler, event, submethods=""): + """ + response_list = handler(title, instructions, prompt_list) + """ + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = "keyboard-interactive" + self.username = username + self.interactive_handler = handler + self.submethods = submethods + self._request_auth() + finally: + self.transport.lock.release() + + def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = "gssapi-with-mic" + self.username = username + self.gss_host = gss_host + self.gss_deleg_creds = gss_deleg_creds + self._request_auth() + finally: + self.transport.lock.release() + + def auth_gssapi_keyex(self, username, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = "gssapi-keyex" + self.username = username + self._request_auth() + finally: + self.transport.lock.release() + + def abort(self): + if self.auth_event is not None: + self.auth_event.set() + + # ...internals... + + def _request_auth(self): + m = Message() + m.add_byte(cMSG_SERVICE_REQUEST) + m.add_string("ssh-userauth") + self.transport._send_message(m) + + def _disconnect_service_not_available(self): + m = Message() + m.add_byte(cMSG_DISCONNECT) + m.add_int(DISCONNECT_SERVICE_NOT_AVAILABLE) + m.add_string("Service not available") + m.add_string("en") + self.transport._send_message(m) + self.transport.close() + + def _disconnect_no_more_auth(self): + m = Message() + m.add_byte(cMSG_DISCONNECT) + m.add_int(DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE) + m.add_string("No more auth methods available") + m.add_string("en") + self.transport._send_message(m) + self.transport.close() + + def _get_key_type_and_bits(self, key): + """ + Given any key, return its type/algorithm & bits-to-sign. + + Intended for input to or verification of, key signatures. + """ + # Use certificate contents, if available, plain pubkey otherwise + if key.public_blob: + return key.public_blob.key_type, key.public_blob.key_blob + else: + return key.get_name(), key + + def _get_session_blob(self, key, service, username, algorithm): + m = Message() + m.add_string(self.transport.session_id) + m.add_byte(cMSG_USERAUTH_REQUEST) + m.add_string(username) + m.add_string(service) + m.add_string("publickey") + m.add_boolean(True) + _, bits = self._get_key_type_and_bits(key) + m.add_string(algorithm) + m.add_string(bits) + return m.asbytes() + + def wait_for_response(self, event): + max_ts = None + if self.transport.auth_timeout is not None: + max_ts = time.time() + self.transport.auth_timeout + while True: + event.wait(0.1) + if not self.transport.is_active(): + e = self.transport.get_exception() + if (e is None) or issubclass(e.__class__, EOFError): + e = AuthenticationException( + "Authentication failed: transport shut down or saw EOF" + ) + raise e + if event.is_set(): + break + if max_ts is not None and max_ts <= time.time(): + raise AuthenticationException("Authentication timeout.") + + if not self.is_authenticated(): + e = self.transport.get_exception() + if e is None: + e = AuthenticationException("Authentication failed.") + # this is horrible. Python Exception isn't yet descended from + # object, so type(e) won't work. :( + # TODO 4.0: lol. just lmao. + if issubclass(e.__class__, PartialAuthentication): + return e.allowed_types + raise e + return [] + + def _parse_service_request(self, m): + service = m.get_text() + if self.transport.server_mode and (service == "ssh-userauth"): + # accepted + m = Message() + m.add_byte(cMSG_SERVICE_ACCEPT) + m.add_string(service) + self.transport._send_message(m) + banner, language = self.transport.server_object.get_banner() + if banner: + m = Message() + m.add_byte(cMSG_USERAUTH_BANNER) + m.add_string(banner) + m.add_string(language) + self.transport._send_message(m) + return + # dunno this one + self._disconnect_service_not_available() + + def _generate_key_from_request(self, algorithm, keyblob): + # For use in server mode. + options = self.transport.preferred_pubkeys + if algorithm.replace("-cert-v01@openssh.com", "") not in options: + err = ( + "Auth rejected: pubkey algorithm '{}' unsupported or disabled" + ) + self._log(INFO, err.format(algorithm)) + return None + return self.transport._key_info[algorithm](Message(keyblob)) + + def _choose_fallback_pubkey_algorithm(self, key_type, my_algos): + # Fallback: first one in our (possibly tweaked by caller) list + pubkey_algo = my_algos[0] + msg = "Server did not send a server-sig-algs list; defaulting to our first preferred algo ({!r})" # noqa + self._log(DEBUG, msg.format(pubkey_algo)) + self._log( + DEBUG, + "NOTE: you may use the 'disabled_algorithms' SSHClient/Transport init kwarg to disable that or other algorithms if your server does not support them!", # noqa + ) + return pubkey_algo + + def _finalize_pubkey_algorithm(self, key_type): + # Short-circuit for non-RSA keys + if "rsa" not in key_type: + return key_type + self._log( + DEBUG, + "Finalizing pubkey algorithm for key of type {!r}".format( + key_type + ), + ) + # NOTE re #2017: When the key is an RSA cert and the remote server is + # OpenSSH 7.7 or earlier, always use ssh-rsa-cert-v01@openssh.com. + # Those versions of the server won't support rsa-sha2 family sig algos + # for certs specifically, and in tandem with various server bugs + # regarding server-sig-algs, it's impossible to fit this into the rest + # of the logic here. + if key_type.endswith("-cert-v01@openssh.com") and re.search( + r"-OpenSSH_(?:[1-6]|7\.[0-7])", self.transport.remote_version + ): + pubkey_algo = "ssh-rsa-cert-v01@openssh.com" + self.transport._agreed_pubkey_algorithm = pubkey_algo + self._log(DEBUG, "OpenSSH<7.8 + RSA cert = forcing ssh-rsa!") + self._log( + DEBUG, "Agreed upon {!r} pubkey algorithm".format(pubkey_algo) + ) + return pubkey_algo + # Normal attempts to handshake follow from here. + # Only consider RSA algos from our list, lest we agree on another! + my_algos = [x for x in self.transport.preferred_pubkeys if "rsa" in x] + self._log(DEBUG, "Our pubkey algorithm list: {}".format(my_algos)) + # Short-circuit negatively if user disabled all RSA algos (heh) + if not my_algos: + raise SSHException( + "An RSA key was specified, but no RSA pubkey algorithms are configured!" # noqa + ) + # Check for server-sig-algs if supported & sent + server_algo_str = u( + self.transport.server_extensions.get("server-sig-algs", b("")) + ) + pubkey_algo = None + # Prefer to match against server-sig-algs + if server_algo_str: + server_algos = server_algo_str.split(",") + self._log( + DEBUG, "Server-side algorithm list: {}".format(server_algos) + ) + # Only use algos from our list that the server likes, in our own + # preference order. (NOTE: purposefully using same style as in + # Transport...expect to refactor later) + agreement = list(filter(server_algos.__contains__, my_algos)) + if agreement: + pubkey_algo = agreement[0] + self._log( + DEBUG, + "Agreed upon {!r} pubkey algorithm".format(pubkey_algo), + ) + else: + self._log(DEBUG, "No common pubkey algorithms exist! Dying.") + # TODO: MAY want to use IncompatiblePeer again here but that's + # technically for initial key exchange, not pubkey auth. + err = "Unable to agree on a pubkey algorithm for signing a {!r} key!" # noqa + raise AuthenticationException(err.format(key_type)) + # Fallback to something based purely on the key & our configuration + else: + pubkey_algo = self._choose_fallback_pubkey_algorithm( + key_type, my_algos + ) + if key_type.endswith("-cert-v01@openssh.com"): + pubkey_algo += "-cert-v01@openssh.com" + self.transport._agreed_pubkey_algorithm = pubkey_algo + return pubkey_algo + + def _parse_service_accept(self, m): + service = m.get_text() + if service == "ssh-userauth": + self._log(DEBUG, "userauth is OK") + m = Message() + m.add_byte(cMSG_USERAUTH_REQUEST) + m.add_string(self.username) + m.add_string("ssh-connection") + m.add_string(self.auth_method) + if self.auth_method == "password": + m.add_boolean(False) + password = b(self.password) + m.add_string(password) + elif self.auth_method == "publickey": + m.add_boolean(True) + key_type, bits = self._get_key_type_and_bits(self.private_key) + algorithm = self._finalize_pubkey_algorithm(key_type) + m.add_string(algorithm) + m.add_string(bits) + blob = self._get_session_blob( + self.private_key, + "ssh-connection", + self.username, + algorithm, + ) + sig = self.private_key.sign_ssh_data(blob, algorithm) + m.add_string(sig) + elif self.auth_method == "keyboard-interactive": + m.add_string("") + m.add_string(self.submethods) + elif self.auth_method == "gssapi-with-mic": + sshgss = GSSAuth(self.auth_method, self.gss_deleg_creds) + m.add_bytes(sshgss.ssh_gss_oids()) + # send the supported GSSAPI OIDs to the server + self.transport._send_message(m) + ptype, m = self.transport.packetizer.read_message() + if ptype == MSG_USERAUTH_BANNER: + self._parse_userauth_banner(m) + ptype, m = self.transport.packetizer.read_message() + if ptype == MSG_USERAUTH_GSSAPI_RESPONSE: + # Read the mechanism selected by the server. We send just + # the Kerberos V5 OID, so the server can only respond with + # this OID. + mech = m.get_string() + m = Message() + m.add_byte(cMSG_USERAUTH_GSSAPI_TOKEN) + try: + m.add_string( + sshgss.ssh_init_sec_context( + self.gss_host, mech, self.username + ) + ) + except GSS_EXCEPTIONS as e: + return self._handle_local_gss_failure(e) + self.transport._send_message(m) + while True: + ptype, m = self.transport.packetizer.read_message() + if ptype == MSG_USERAUTH_GSSAPI_TOKEN: + srv_token = m.get_string() + try: + next_token = sshgss.ssh_init_sec_context( + self.gss_host, + mech, + self.username, + srv_token, + ) + except GSS_EXCEPTIONS as e: + return self._handle_local_gss_failure(e) + # After this step the GSSAPI should not return any + # token. If it does, we keep sending the token to + # the server until no more token is returned. + if next_token is None: + break + else: + m = Message() + m.add_byte(cMSG_USERAUTH_GSSAPI_TOKEN) + m.add_string(next_token) + self.transport.send_message(m) + else: + raise SSHException( + "Received Package: {}".format(MSG_NAMES[ptype]) + ) + m = Message() + m.add_byte(cMSG_USERAUTH_GSSAPI_MIC) + # send the MIC to the server + m.add_string(sshgss.ssh_get_mic(self.transport.session_id)) + elif ptype == MSG_USERAUTH_GSSAPI_ERRTOK: + # RFC 4462 says we are not required to implement GSS-API + # error messages. + # See RFC 4462 Section 3.8 in + # http://www.ietf.org/rfc/rfc4462.txt + raise SSHException("Server returned an error token") + elif ptype == MSG_USERAUTH_GSSAPI_ERROR: + maj_status = m.get_int() + min_status = m.get_int() + err_msg = m.get_string() + m.get_string() # Lang tag - discarded + raise SSHException( + """GSS-API Error: +Major Status: {} +Minor Status: {} +Error Message: {} +""".format( + maj_status, min_status, err_msg + ) + ) + elif ptype == MSG_USERAUTH_FAILURE: + self._parse_userauth_failure(m) + return + else: + raise SSHException( + "Received Package: {}".format(MSG_NAMES[ptype]) + ) + elif ( + self.auth_method == "gssapi-keyex" + and self.transport.gss_kex_used + ): + kexgss = self.transport.kexgss_ctxt + kexgss.set_username(self.username) + mic_token = kexgss.ssh_get_mic(self.transport.session_id) + m.add_string(mic_token) + elif self.auth_method == "none": + pass + else: + raise SSHException( + 'Unknown auth method "{}"'.format(self.auth_method) + ) + self.transport._send_message(m) + else: + self._log( + DEBUG, 'Service request "{}" accepted (?)'.format(service) + ) + + def _send_auth_result(self, username, method, result): + # okay, send result + m = Message() + if result == AUTH_SUCCESSFUL: + self._log(INFO, "Auth granted ({}).".format(method)) + m.add_byte(cMSG_USERAUTH_SUCCESS) + self.authenticated = True + else: + self._log(INFO, "Auth rejected ({}).".format(method)) + m.add_byte(cMSG_USERAUTH_FAILURE) + m.add_string( + self.transport.server_object.get_allowed_auths(username) + ) + if result == AUTH_PARTIALLY_SUCCESSFUL: + m.add_boolean(True) + else: + m.add_boolean(False) + self.auth_fail_count += 1 + self.transport._send_message(m) + if self.auth_fail_count >= 10: + self._disconnect_no_more_auth() + if result == AUTH_SUCCESSFUL: + self.transport._auth_trigger() + + def _interactive_query(self, q): + # make interactive query instead of response + m = Message() + m.add_byte(cMSG_USERAUTH_INFO_REQUEST) + m.add_string(q.name) + m.add_string(q.instructions) + m.add_string(bytes()) + m.add_int(len(q.prompts)) + for p in q.prompts: + m.add_string(p[0]) + m.add_boolean(p[1]) + self.transport._send_message(m) + + def _parse_userauth_request(self, m): + if not self.transport.server_mode: + # er, uh... what? + m = Message() + m.add_byte(cMSG_USERAUTH_FAILURE) + m.add_string("none") + m.add_boolean(False) + self.transport._send_message(m) + return + if self.authenticated: + # ignore + return + username = m.get_text() + service = m.get_text() + method = m.get_text() + self._log( + DEBUG, + "Auth request (type={}) service={}, username={}".format( + method, service, username + ), + ) + if service != "ssh-connection": + self._disconnect_service_not_available() + return + if (self.auth_username is not None) and ( + self.auth_username != username + ): + self._log( + WARNING, + "Auth rejected because the client attempted to change username in mid-flight", # noqa + ) + self._disconnect_no_more_auth() + return + self.auth_username = username + # check if GSS-API authentication is enabled + gss_auth = self.transport.server_object.enable_auth_gssapi() + + if method == "none": + result = self.transport.server_object.check_auth_none(username) + elif method == "password": + changereq = m.get_boolean() + password = m.get_binary() + try: + password = password.decode("UTF-8") + except UnicodeError: + # some clients/servers expect non-utf-8 passwords! + # in this case, just return the raw byte string. + pass + if changereq: + # always treated as failure, since we don't support changing + # passwords, but collect the list of valid auth types from + # the callback anyway + self._log(DEBUG, "Auth request to change passwords (rejected)") + newpassword = m.get_binary() + try: + newpassword = newpassword.decode("UTF-8", "replace") + except UnicodeError: + pass + result = AUTH_FAILED + else: + result = self.transport.server_object.check_auth_password( + username, password + ) + elif method == "publickey": + sig_attached = m.get_boolean() + # NOTE: server never wants to guess a client's algo, they're + # telling us directly. No need for _finalize_pubkey_algorithm + # anywhere in this flow. + algorithm = m.get_text() + keyblob = m.get_binary() + try: + key = self._generate_key_from_request(algorithm, keyblob) + except SSHException as e: + self._log(INFO, "Auth rejected: public key: {}".format(str(e))) + key = None + except Exception as e: + msg = "Auth rejected: unsupported or mangled public key ({}: {})" # noqa + self._log(INFO, msg.format(e.__class__.__name__, e)) + key = None + if key is None: + self._disconnect_no_more_auth() + return + # first check if this key is okay... if not, we can skip the verify + result = self.transport.server_object.check_auth_publickey( + username, key + ) + if result != AUTH_FAILED: + # key is okay, verify it + if not sig_attached: + # client wants to know if this key is acceptable, before it + # signs anything... send special "ok" message + m = Message() + m.add_byte(cMSG_USERAUTH_PK_OK) + m.add_string(algorithm) + m.add_string(keyblob) + self.transport._send_message(m) + return + sig = Message(m.get_binary()) + blob = self._get_session_blob( + key, service, username, algorithm + ) + if not key.verify_ssh_sig(blob, sig): + self._log(INFO, "Auth rejected: invalid signature") + result = AUTH_FAILED + elif method == "keyboard-interactive": + submethods = m.get_string() + result = self.transport.server_object.check_auth_interactive( + username, submethods + ) + if isinstance(result, InteractiveQuery): + # make interactive query instead of response + self._interactive_query(result) + return + elif method == "gssapi-with-mic" and gss_auth: + sshgss = GSSAuth(method) + # Read the number of OID mechanisms supported by the client. + # OpenSSH sends just one OID. It's the Kerveros V5 OID and that's + # the only OID we support. + mechs = m.get_int() + # We can't accept more than one OID, so if the SSH client sends + # more than one, disconnect. + if mechs > 1: + self._log( + INFO, + "Disconnect: Received more than one GSS-API OID mechanism", + ) + self._disconnect_no_more_auth() + desired_mech = m.get_string() + mech_ok = sshgss.ssh_check_mech(desired_mech) + # if we don't support the mechanism, disconnect. + if not mech_ok: + self._log( + INFO, + "Disconnect: Received an invalid GSS-API OID mechanism", + ) + self._disconnect_no_more_auth() + # send the Kerberos V5 GSSAPI OID to the client + supported_mech = sshgss.ssh_gss_oids("server") + # RFC 4462 says we are not required to implement GSS-API error + # messages. See section 3.8 in http://www.ietf.org/rfc/rfc4462.txt + m = Message() + m.add_byte(cMSG_USERAUTH_GSSAPI_RESPONSE) + m.add_bytes(supported_mech) + self.transport.auth_handler = GssapiWithMicAuthHandler( + self, sshgss + ) + self.transport._expected_packet = ( + MSG_USERAUTH_GSSAPI_TOKEN, + MSG_USERAUTH_REQUEST, + MSG_SERVICE_REQUEST, + ) + self.transport._send_message(m) + return + elif method == "gssapi-keyex" and gss_auth: + mic_token = m.get_string() + sshgss = self.transport.kexgss_ctxt + if sshgss is None: + # If there is no valid context, we reject the authentication + result = AUTH_FAILED + self._send_auth_result(username, method, result) + try: + sshgss.ssh_check_mic( + mic_token, self.transport.session_id, self.auth_username + ) + except Exception: + result = AUTH_FAILED + self._send_auth_result(username, method, result) + raise + result = AUTH_SUCCESSFUL + self.transport.server_object.check_auth_gssapi_keyex( + username, result + ) + else: + result = self.transport.server_object.check_auth_none(username) + # okay, send result + self._send_auth_result(username, method, result) + + def _parse_userauth_success(self, m): + self._log( + INFO, "Authentication ({}) successful!".format(self.auth_method) + ) + self.authenticated = True + self.transport._auth_trigger() + if self.auth_event is not None: + self.auth_event.set() + + def _parse_userauth_failure(self, m): + authlist = m.get_list() + # TODO 4.0: we aren't giving callers access to authlist _unless_ it's + # partial authentication, so eg authtype=none can't work unless we + # tweak this. + partial = m.get_boolean() + if partial: + self._log(INFO, "Authentication continues...") + self._log(DEBUG, "Methods: " + str(authlist)) + self.transport.saved_exception = PartialAuthentication(authlist) + elif self.auth_method not in authlist: + for msg in ( + "Authentication type ({}) not permitted.".format( + self.auth_method + ), + "Allowed methods: {}".format(authlist), + ): + self._log(DEBUG, msg) + self.transport.saved_exception = BadAuthenticationType( + "Bad authentication type", authlist + ) + else: + self._log( + INFO, "Authentication ({}) failed.".format(self.auth_method) + ) + self.authenticated = False + self.username = None + if self.auth_event is not None: + self.auth_event.set() + + def _parse_userauth_banner(self, m): + banner = m.get_string() + self.banner = banner + self._log(INFO, "Auth banner: {}".format(banner)) + # who cares. + + def _parse_userauth_info_request(self, m): + if self.auth_method != "keyboard-interactive": + raise SSHException("Illegal info request from server") + title = m.get_text() + instructions = m.get_text() + m.get_binary() # lang + prompts = m.get_int() + prompt_list = [] + for i in range(prompts): + prompt_list.append((m.get_text(), m.get_boolean())) + response_list = self.interactive_handler( + title, instructions, prompt_list + ) + + m = Message() + m.add_byte(cMSG_USERAUTH_INFO_RESPONSE) + m.add_int(len(response_list)) + for r in response_list: + m.add_string(r) + self.transport._send_message(m) + + def _parse_userauth_info_response(self, m): + if not self.transport.server_mode: + raise SSHException("Illegal info response from server") + n = m.get_int() + responses = [] + for i in range(n): + responses.append(m.get_text()) + result = self.transport.server_object.check_auth_interactive_response( + responses + ) + if isinstance(result, InteractiveQuery): + # make interactive query instead of response + self._interactive_query(result) + return + self._send_auth_result( + self.auth_username, "keyboard-interactive", result + ) + + def _handle_local_gss_failure(self, e): + self.transport.saved_exception = e + self._log(DEBUG, "GSSAPI failure: {}".format(e)) + self._log(INFO, "Authentication ({}) failed.".format(self.auth_method)) + self.authenticated = False + self.username = None + if self.auth_event is not None: + self.auth_event.set() + return + + # TODO 4.0: MAY make sense to make these tables into actual + # classes/instances that can be fed a mode bool or whatever. Or, + # alternately (both?) make the message types small classes or enums that + # embed this info within themselves (which could also then tidy up the + # current 'integer -> human readable short string' stuff in common.py). + # TODO: if we do that, also expose 'em publicly. + + # Messages which should be handled _by_ servers (sent by clients) + @property + def _server_handler_table(self): + return { + # TODO 4.0: MSG_SERVICE_REQUEST ought to eventually move into + # Transport's server mode like the client side did, just for + # consistency. + MSG_SERVICE_REQUEST: self._parse_service_request, + MSG_USERAUTH_REQUEST: self._parse_userauth_request, + MSG_USERAUTH_INFO_RESPONSE: self._parse_userauth_info_response, + } + + # Messages which should be handled _by_ clients (sent by servers) + @property + def _client_handler_table(self): + return { + MSG_SERVICE_ACCEPT: self._parse_service_accept, + MSG_USERAUTH_SUCCESS: self._parse_userauth_success, + MSG_USERAUTH_FAILURE: self._parse_userauth_failure, + MSG_USERAUTH_BANNER: self._parse_userauth_banner, + MSG_USERAUTH_INFO_REQUEST: self._parse_userauth_info_request, + } + + # NOTE: prior to the fix for #1283, this was a static dict instead of a + # property. Should be backwards compatible in most/all cases. + @property + def _handler_table(self): + if self.transport.server_mode: + return self._server_handler_table + else: + return self._client_handler_table + + +class GssapiWithMicAuthHandler: + """A specialized Auth handler for gssapi-with-mic + + During the GSSAPI token exchange we need a modified dispatch table, + because the packet type numbers are not unique. + """ + + method = "gssapi-with-mic" + + def __init__(self, delegate, sshgss): + self._delegate = delegate + self.sshgss = sshgss + + def abort(self): + self._restore_delegate_auth_handler() + return self._delegate.abort() + + @property + def transport(self): + return self._delegate.transport + + @property + def _send_auth_result(self): + return self._delegate._send_auth_result + + @property + def auth_username(self): + return self._delegate.auth_username + + @property + def gss_host(self): + return self._delegate.gss_host + + def _restore_delegate_auth_handler(self): + self.transport.auth_handler = self._delegate + + def _parse_userauth_gssapi_token(self, m): + client_token = m.get_string() + # use the client token as input to establish a secure + # context. + sshgss = self.sshgss + try: + token = sshgss.ssh_accept_sec_context( + self.gss_host, client_token, self.auth_username + ) + except Exception as e: + self.transport.saved_exception = e + result = AUTH_FAILED + self._restore_delegate_auth_handler() + self._send_auth_result(self.auth_username, self.method, result) + raise + if token is not None: + m = Message() + m.add_byte(cMSG_USERAUTH_GSSAPI_TOKEN) + m.add_string(token) + self.transport._expected_packet = ( + MSG_USERAUTH_GSSAPI_TOKEN, + MSG_USERAUTH_GSSAPI_MIC, + MSG_USERAUTH_REQUEST, + ) + self.transport._send_message(m) + + def _parse_userauth_gssapi_mic(self, m): + mic_token = m.get_string() + sshgss = self.sshgss + username = self.auth_username + self._restore_delegate_auth_handler() + try: + sshgss.ssh_check_mic( + mic_token, self.transport.session_id, username + ) + except Exception as e: + self.transport.saved_exception = e + result = AUTH_FAILED + self._send_auth_result(username, self.method, result) + raise + # TODO: Implement client credential saving. + # The OpenSSH server is able to create a TGT with the delegated + # client credentials, but this is not supported by GSS-API. + result = AUTH_SUCCESSFUL + self.transport.server_object.check_auth_gssapi_with_mic( + username, result + ) + # okay, send result + self._send_auth_result(username, self.method, result) + + def _parse_service_request(self, m): + self._restore_delegate_auth_handler() + return self._delegate._parse_service_request(m) + + def _parse_userauth_request(self, m): + self._restore_delegate_auth_handler() + return self._delegate._parse_userauth_request(m) + + __handler_table = { + MSG_SERVICE_REQUEST: _parse_service_request, + MSG_USERAUTH_REQUEST: _parse_userauth_request, + MSG_USERAUTH_GSSAPI_TOKEN: _parse_userauth_gssapi_token, + MSG_USERAUTH_GSSAPI_MIC: _parse_userauth_gssapi_mic, + } + + @property + def _handler_table(self): + # TODO: determine if we can cut this up like we did for the primary + # AuthHandler class. + return self.__handler_table + + +class AuthOnlyHandler(AuthHandler): + """ + AuthHandler, and just auth, no service requests! + + .. versionadded:: 3.2 + """ + + # NOTE: this purposefully duplicates some of the parent class in order to + # modernize, refactor, etc. The intent is that eventually we will collapse + # this one onto the parent in a backwards incompatible release. + + @property + def _client_handler_table(self): + my_table = super()._client_handler_table.copy() + del my_table[MSG_SERVICE_ACCEPT] + return my_table + + def send_auth_request(self, username, method, finish_message=None): + """ + Submit a userauth request message & wait for response. + + Performs the transport message send call, sets self.auth_event, and + will lock-n-block as necessary to both send, and wait for response to, + the USERAUTH_REQUEST. + + Most callers will want to supply a callback to ``finish_message``, + which accepts a Message ``m`` and may call mutator methods on it to add + more fields. + """ + # Store a few things for reference in handlers, including auth failure + # handler (which needs to know if we were using a bad method, etc) + self.auth_method = method + self.username = username + # Generic userauth request fields + m = Message() + m.add_byte(cMSG_USERAUTH_REQUEST) + m.add_string(username) + m.add_string("ssh-connection") + m.add_string(method) + # Caller usually has more to say, such as injecting password, key etc + finish_message(m) + # TODO 4.0: seems odd to have the client handle the lock and not + # Transport; that _may_ have been an artifact of allowing user + # threading event injection? Regardless, we don't want to move _this_ + # locking into Transport._send_message now, because lots of other + # untouched code also uses that method and we might end up + # double-locking (?) but 4.0 would be a good time to revisit. + with self.transport.lock: + self.transport._send_message(m) + # We have cut out the higher level event args, but self.auth_event is + # still required for self.wait_for_response to function correctly (it's + # the mechanism used by the auth success/failure handlers, the abort + # handler, and a few other spots like in gssapi. + # TODO: interestingly, wait_for_response itself doesn't actually + # enforce that its event argument and self.auth_event are the same... + self.auth_event = threading.Event() + return self.wait_for_response(self.auth_event) + + def auth_none(self, username): + return self.send_auth_request(username, "none") + + def auth_publickey(self, username, key): + key_type, bits = self._get_key_type_and_bits(key) + algorithm = self._finalize_pubkey_algorithm(key_type) + blob = self._get_session_blob( + key, + "ssh-connection", + username, + algorithm, + ) + + def finish(m): + # This field doesn't appear to be named, but is False when querying + # for permission (ie knowing whether to even prompt a user for + # passphrase, etc) or True when just going for it. Paramiko has + # never bothered with the former type of message, apparently. + m.add_boolean(True) + m.add_string(algorithm) + m.add_string(bits) + m.add_string(key.sign_ssh_data(blob, algorithm)) + + return self.send_auth_request(username, "publickey", finish) + + def auth_password(self, username, password): + def finish(m): + # Unnamed field that equates to "I am changing my password", which + # Paramiko clientside never supported and serverside only sort of + # supported. + m.add_boolean(False) + m.add_string(b(password)) + + return self.send_auth_request(username, "password", finish) + + def auth_interactive(self, username, handler, submethods=""): + """ + response_list = handler(title, instructions, prompt_list) + """ + # Unlike most siblings, this auth method _does_ require other + # superclass handlers (eg userauth info request) to understand + # what's going on, so we still set some self attributes. + self.auth_method = "keyboard_interactive" + self.interactive_handler = handler + + def finish(m): + # Empty string for deprecated language tag field, per RFC 4256: + # https://www.rfc-editor.org/rfc/rfc4256#section-3.1 + m.add_string("") + m.add_string(submethods) + + return self.send_auth_request(username, "keyboard-interactive", finish) + + # NOTE: not strictly 'auth only' related, but allows users to opt-in. + def _choose_fallback_pubkey_algorithm(self, key_type, my_algos): + msg = "Server did not send a server-sig-algs list; defaulting to something in our preferred algorithms list" # noqa + self._log(DEBUG, msg) + noncert_key_type = key_type.replace("-cert-v01@openssh.com", "") + if key_type in my_algos or noncert_key_type in my_algos: + actual = key_type if key_type in my_algos else noncert_key_type + msg = f"Current key type, {actual!r}, is in our preferred list; using that" # noqa + algo = actual + else: + algo = my_algos[0] + msg = f"{key_type!r} not in our list - trying first list item instead, {algo!r}" # noqa + self._log(DEBUG, msg) + return algo diff --git a/lib/paramiko/auth_strategy.py b/lib/paramiko/auth_strategy.py new file mode 100644 index 0000000..03c1d87 --- /dev/null +++ b/lib/paramiko/auth_strategy.py @@ -0,0 +1,306 @@ +""" +Modern, adaptable authentication machinery. + +Replaces certain parts of `.SSHClient`. For a concrete implementation, see the +``OpenSSHAuthStrategy`` class in `Fabric `_. +""" + +from collections import namedtuple + +from .agent import AgentKey +from .util import get_logger +from .ssh_exception import AuthenticationException + + +class AuthSource: + """ + Some SSH authentication source, such as a password, private key, or agent. + + See subclasses in this module for concrete implementations. + + All implementations must accept at least a ``username`` (``str``) kwarg. + """ + + def __init__(self, username): + self.username = username + + def _repr(self, **kwargs): + # TODO: are there any good libs for this? maybe some helper from + # structlog? + pairs = [f"{k}={v!r}" for k, v in kwargs.items()] + joined = ", ".join(pairs) + return f"{self.__class__.__name__}({joined})" + + def __repr__(self): + return self._repr() + + def authenticate(self, transport): + """ + Perform authentication. + """ + raise NotImplementedError + + +class NoneAuth(AuthSource): + """ + Auth type "none", ie https://www.rfc-editor.org/rfc/rfc4252#section-5.2 . + """ + + def authenticate(self, transport): + return transport.auth_none(self.username) + + +class Password(AuthSource): + """ + Password authentication. + + :param callable password_getter: + A lazy callable that should return a `str` password value at + authentication time, such as a `functools.partial` wrapping + `getpass.getpass`, an API call to a secrets store, or similar. + + If you already know the password at instantiation time, you should + simply use something like ``lambda: "my literal"`` (for a literal, but + also, shame on you!) or ``lambda: variable_name`` (for something stored + in a variable). + """ + + def __init__(self, username, password_getter): + super().__init__(username=username) + self.password_getter = password_getter + + def __repr__(self): + # Password auth is marginally more 'username-caring' than pkeys, so may + # as well log that info here. + return super()._repr(user=self.username) + + def authenticate(self, transport): + # Lazily get the password, in case it's prompting a user + # TODO: be nice to log source _of_ the password? + password = self.password_getter() + return transport.auth_password(self.username, password) + + +# TODO 4.0: twiddle this, or PKey, or both, so they're more obviously distinct. +# TODO 4.0: the obvious is to make this more wordy (PrivateKeyAuth), the +# minimalist approach might be to rename PKey to just Key (esp given all the +# subclasses are WhateverKey and not WhateverPKey) +class PrivateKey(AuthSource): + """ + Essentially a mixin for private keys. + + Knows how to auth, but leaves key material discovery/loading/decryption to + subclasses. + + Subclasses **must** ensure that they've set ``self.pkey`` to a decrypted + `.PKey` instance before calling ``super().authenticate``; typically + either in their ``__init__``, or in an overridden ``authenticate`` prior to + its `super` call. + """ + + def authenticate(self, transport): + return transport.auth_publickey(self.username, self.pkey) + + +class InMemoryPrivateKey(PrivateKey): + """ + An in-memory, decrypted `.PKey` object. + """ + + def __init__(self, username, pkey): + super().__init__(username=username) + # No decryption (presumably) necessary! + self.pkey = pkey + + def __repr__(self): + # NOTE: most of interesting repr-bits for private keys is in PKey. + # TODO: tacking on agent-ness like this is a bit awkward, but, eh? + rep = super()._repr(pkey=self.pkey) + if isinstance(self.pkey, AgentKey): + rep += " [agent]" + return rep + + +class OnDiskPrivateKey(PrivateKey): + """ + Some on-disk private key that needs opening and possibly decrypting. + + :param str source: + String tracking where this key's path was specified; should be one of + ``"ssh-config"``, ``"python-config"``, or ``"implicit-home"``. + :param Path path: + The filesystem path this key was loaded from. + :param PKey pkey: + The `PKey` object this auth source uses/represents. + """ + + def __init__(self, username, source, path, pkey): + super().__init__(username=username) + self.source = source + allowed = ("ssh-config", "python-config", "implicit-home") + if source not in allowed: + raise ValueError(f"source argument must be one of: {allowed!r}") + self.path = path + # Superclass wants .pkey, other two are mostly for display/debugging. + self.pkey = pkey + + def __repr__(self): + return self._repr( + key=self.pkey, source=self.source, path=str(self.path) + ) + + +# TODO re sources: is there anything in an OpenSSH config file that doesn't fit +# into what Paramiko already had kwargs for? + + +SourceResult = namedtuple("SourceResult", ["source", "result"]) + +# TODO: tempting to make this an OrderedDict, except the keys essentially want +# to be rich objects (AuthSources) which do not make for useful user indexing? +# TODO: members being vanilla tuples is pretty old-school/expedient; they +# "really" want to be something that's type friendlier (unless the tuple's 2nd +# member being a Union of two types is "fine"?), which I assume means yet more +# classes, eg an abstract SourceResult with concrete AuthSuccess and +# AuthFailure children? +# TODO: arguably we want __init__ typechecking of the members (or to leverage +# mypy by classifying this literally as list-of-AuthSource?) +class AuthResult(list): + """ + Represents a partial or complete SSH authentication attempt. + + This class conceptually extends `AuthStrategy` by pairing the former's + authentication **sources** with the **results** of trying to authenticate + with them. + + `AuthResult` is a (subclass of) `list` of `namedtuple`, which are of the + form ``namedtuple('SourceResult', 'source', 'result')`` (where the + ``source`` member is an `AuthSource` and the ``result`` member is either a + return value from the relevant `.Transport` method, or an exception + object). + + .. note:: + Transport auth method results are always themselves a ``list`` of "next + allowable authentication methods". + + In the simple case of "you just authenticated successfully", it's an + empty list; if your auth was rejected but you're allowed to try again, + it will be a list of string method names like ``pubkey`` or + ``password``. + + The ``__str__`` of this class represents the empty-list scenario as the + word ``success``, which should make reading the result of an + authentication session more obvious to humans. + + Instances also have a `strategy` attribute referencing the `AuthStrategy` + which was attempted. + """ + + def __init__(self, strategy, *args, **kwargs): + self.strategy = strategy + super().__init__(*args, **kwargs) + + def __str__(self): + # NOTE: meaningfully distinct from __repr__, which still wants to use + # superclass' implementation. + # TODO: go hog wild, use rich.Table? how is that on degraded term's? + # TODO: test this lol + return "\n".join( + f"{x.source} -> {x.result or 'success'}" for x in self + ) + + +# TODO 4.0: descend from SSHException or even just Exception +class AuthFailure(AuthenticationException): + """ + Basic exception wrapping an `AuthResult` indicating overall auth failure. + + Note that `AuthFailure` descends from `AuthenticationException` but is + generally "higher level"; the latter is now only raised by individual + `AuthSource` attempts and should typically only be seen by users when + encapsulated in this class. It subclasses `AuthenticationException` + primarily for backwards compatibility reasons. + """ + + def __init__(self, result): + self.result = result + + def __str__(self): + return "\n" + str(self.result) + + +class AuthStrategy: + """ + This class represents one or more attempts to auth with an SSH server. + + By default, subclasses must at least accept an ``ssh_config`` + (`.SSHConfig`) keyword argument, but may opt to accept more as needed for + their particular strategy. + """ + + def __init__( + self, + ssh_config, + ): + self.ssh_config = ssh_config + self.log = get_logger(__name__) + + def get_sources(self): + """ + Generator yielding `AuthSource` instances, in the order to try. + + This is the primary override point for subclasses: you figure out what + sources you need, and ``yield`` them. + + Subclasses _of_ subclasses may find themselves wanting to do things + like filtering or discarding around a call to `super`. + """ + raise NotImplementedError + + def authenticate(self, transport): + """ + Handles attempting `AuthSource` instances yielded from `get_sources`. + + You *normally* won't need to override this, but it's an option for + advanced users. + """ + succeeded = False + overall_result = AuthResult(strategy=self) + # TODO: arguably we could fit in a "send none auth, record allowed auth + # types sent back" thing here as OpenSSH-client does, but that likely + # wants to live in fabric.OpenSSHAuthStrategy as not all target servers + # will implement it! + # TODO: needs better "server told us too many attempts" checking! + for source in self.get_sources(): + self.log.debug(f"Trying {source}") + try: # NOTE: this really wants to _only_ wrap the authenticate()! + result = source.authenticate(transport) + succeeded = True + # TODO: 'except PartialAuthentication' is needed for 2FA and + # similar, as per old SSHClient.connect - it is the only way + # AuthHandler supplies access to the 'name-list' field from + # MSG_USERAUTH_FAILURE, at present. + except Exception as e: + result = e + # TODO: look at what this could possibly raise, we don't really + # want Exception here, right? just SSHException subclasses? or + # do we truly want to capture anything at all with assumption + # it's easy enough for users to look afterwards? + # NOTE: showing type, not message, for tersity & also most of + # the time it's basically just "Authentication failed." + source_class = e.__class__.__name__ + self.log.info( + f"Authentication via {source} failed with {source_class}" + ) + overall_result.append(SourceResult(source, result)) + if succeeded: + break + # Gotta die here if nothing worked, otherwise Transport's main loop + # just kinda hangs out until something times out! + if not succeeded: + raise AuthFailure(result=overall_result) + # Success: give back what was done, in case they care. + return overall_result + + # TODO: is there anything OpenSSH client does which _can't_ cleanly map to + # iterating a generator? diff --git a/lib/paramiko/ber.py b/lib/paramiko/ber.py new file mode 100644 index 0000000..b8287f5 --- /dev/null +++ b/lib/paramiko/ber.py @@ -0,0 +1,139 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +from paramiko.common import max_byte, zero_byte, byte_ord, byte_chr + +import paramiko.util as util +from paramiko.util import b +from paramiko.sftp import int64 + + +class BERException(Exception): + pass + + +class BER: + """ + Robey's tiny little attempt at a BER decoder. + """ + + def __init__(self, content=bytes()): + self.content = b(content) + self.idx = 0 + + def asbytes(self): + return self.content + + def __str__(self): + return self.asbytes() + + def __repr__(self): + return "BER('" + repr(self.content) + "')" + + def decode(self): + return self.decode_next() + + def decode_next(self): + if self.idx >= len(self.content): + return None + ident = byte_ord(self.content[self.idx]) + self.idx += 1 + if (ident & 31) == 31: + # identifier > 30 + ident = 0 + while self.idx < len(self.content): + t = byte_ord(self.content[self.idx]) + self.idx += 1 + ident = (ident << 7) | (t & 0x7F) + if not (t & 0x80): + break + if self.idx >= len(self.content): + return None + # now fetch length + size = byte_ord(self.content[self.idx]) + self.idx += 1 + if size & 0x80: + # more complimicated... + # FIXME: theoretically should handle indefinite-length (0x80) + t = size & 0x7F + if self.idx + t > len(self.content): + return None + size = util.inflate_long( + self.content[self.idx : self.idx + t], True + ) + self.idx += t + if self.idx + size > len(self.content): + # can't fit + return None + data = self.content[self.idx : self.idx + size] + self.idx += size + # now switch on id + if ident == 0x30: + # sequence + return self.decode_sequence(data) + elif ident == 2: + # int + return util.inflate_long(data) + else: + # 1: boolean (00 false, otherwise true) + msg = "Unknown ber encoding type {:d} (robey is lazy)" + raise BERException(msg.format(ident)) + + @staticmethod + def decode_sequence(data): + out = [] + ber = BER(data) + while True: + x = ber.decode_next() + if x is None: + break + out.append(x) + return out + + def encode_tlv(self, ident, val): + # no need to support ident > 31 here + self.content += byte_chr(ident) + if len(val) > 0x7F: + lenstr = util.deflate_long(len(val)) + self.content += byte_chr(0x80 + len(lenstr)) + lenstr + else: + self.content += byte_chr(len(val)) + self.content += val + + def encode(self, x): + if type(x) is bool: + if x: + self.encode_tlv(1, max_byte) + else: + self.encode_tlv(1, zero_byte) + elif (type(x) is int) or (type(x) is int64): + self.encode_tlv(2, util.deflate_long(x)) + elif type(x) is str: + self.encode_tlv(4, x) + elif (type(x) is list) or (type(x) is tuple): + self.encode_tlv(0x30, self.encode_sequence(x)) + else: + raise BERException( + "Unknown type for encoding: {!r}".format(type(x)) + ) + + @staticmethod + def encode_sequence(data): + ber = BER() + for item in data: + ber.encode(item) + return ber.asbytes() diff --git a/lib/paramiko/buffered_pipe.py b/lib/paramiko/buffered_pipe.py new file mode 100644 index 0000000..c19279c --- /dev/null +++ b/lib/paramiko/buffered_pipe.py @@ -0,0 +1,212 @@ +# Copyright (C) 2006-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Attempt to generalize the "feeder" part of a `.Channel`: an object which can be +read from and closed, but is reading from a buffer fed by another thread. The +read operations are blocking and can have a timeout set. +""" + +import array +import threading +import time +from paramiko.util import b + + +class PipeTimeout(IOError): + """ + Indicates that a timeout was reached on a read from a `.BufferedPipe`. + """ + + pass + + +class BufferedPipe: + """ + A buffer that obeys normal read (with timeout) & close semantics for a + file or socket, but is fed data from another thread. This is used by + `.Channel`. + """ + + def __init__(self): + self._lock = threading.Lock() + self._cv = threading.Condition(self._lock) + self._event = None + self._buffer = array.array("B") + self._closed = False + + def _buffer_frombytes(self, data): + self._buffer.frombytes(data) + + def _buffer_tobytes(self, limit=None): + return self._buffer[:limit].tobytes() + + def set_event(self, event): + """ + Set an event on this buffer. When data is ready to be read (or the + buffer has been closed), the event will be set. When no data is + ready, the event will be cleared. + + :param threading.Event event: the event to set/clear + """ + self._lock.acquire() + try: + self._event = event + # Make sure the event starts in `set` state if we appear to already + # be closed; otherwise, if we start in `clear` state & are closed, + # nothing will ever call `.feed` and the event (& OS pipe, if we're + # wrapping one - see `Channel.fileno`) will permanently stay in + # `clear`, causing deadlock if e.g. `select`ed upon. + if self._closed or len(self._buffer) > 0: + event.set() + else: + event.clear() + finally: + self._lock.release() + + def feed(self, data): + """ + Feed new data into this pipe. This method is assumed to be called + from a separate thread, so synchronization is done. + + :param data: the data to add, as a ``str`` or ``bytes`` + """ + self._lock.acquire() + try: + if self._event is not None: + self._event.set() + self._buffer_frombytes(b(data)) + self._cv.notify_all() + finally: + self._lock.release() + + def read_ready(self): + """ + Returns true if data is buffered and ready to be read from this + feeder. A ``False`` result does not mean that the feeder has closed; + it means you may need to wait before more data arrives. + + :return: + ``True`` if a `read` call would immediately return at least one + byte; ``False`` otherwise. + """ + self._lock.acquire() + try: + if len(self._buffer) == 0: + return False + return True + finally: + self._lock.release() + + def read(self, nbytes, timeout=None): + """ + Read data from the pipe. The return value is a string representing + the data received. The maximum amount of data to be received at once + is specified by ``nbytes``. If a string of length zero is returned, + the pipe has been closed. + + The optional ``timeout`` argument can be a nonnegative float expressing + seconds, or ``None`` for no timeout. If a float is given, a + `.PipeTimeout` will be raised if the timeout period value has elapsed + before any data arrives. + + :param int nbytes: maximum number of bytes to read + :param float timeout: + maximum seconds to wait (or ``None``, the default, to wait forever) + :return: the read data, as a ``str`` or ``bytes`` + + :raises: + `.PipeTimeout` -- if a timeout was specified and no data was ready + before that timeout + """ + out = bytes() + self._lock.acquire() + try: + if len(self._buffer) == 0: + if self._closed: + return out + # should we block? + if timeout == 0.0: + raise PipeTimeout() + # loop here in case we get woken up but a different thread has + # grabbed everything in the buffer. + while (len(self._buffer) == 0) and not self._closed: + then = time.time() + self._cv.wait(timeout) + if timeout is not None: + timeout -= time.time() - then + if timeout <= 0.0: + raise PipeTimeout() + + # something's in the buffer and we have the lock! + if len(self._buffer) <= nbytes: + out = self._buffer_tobytes() + del self._buffer[:] + if (self._event is not None) and not self._closed: + self._event.clear() + else: + out = self._buffer_tobytes(nbytes) + del self._buffer[:nbytes] + finally: + self._lock.release() + + return out + + def empty(self): + """ + Clear out the buffer and return all data that was in it. + + :return: + any data that was in the buffer prior to clearing it out, as a + `str` + """ + self._lock.acquire() + try: + out = self._buffer_tobytes() + del self._buffer[:] + if (self._event is not None) and not self._closed: + self._event.clear() + return out + finally: + self._lock.release() + + def close(self): + """ + Close this pipe object. Future calls to `read` after the buffer + has been emptied will return immediately with an empty string. + """ + self._lock.acquire() + try: + self._closed = True + self._cv.notify_all() + if self._event is not None: + self._event.set() + finally: + self._lock.release() + + def __len__(self): + """ + Return the number of bytes buffered. + + :return: number (`int`) of bytes buffered + """ + self._lock.acquire() + try: + return len(self._buffer) + finally: + self._lock.release() diff --git a/lib/paramiko/channel.py b/lib/paramiko/channel.py new file mode 100644 index 0000000..25326ca --- /dev/null +++ b/lib/paramiko/channel.py @@ -0,0 +1,1390 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Abstraction for an SSH2 channel. +""" + +import binascii +import os +import socket +import time +import threading + +from functools import wraps + +from paramiko import util +from paramiko.common import ( + cMSG_CHANNEL_REQUEST, + cMSG_CHANNEL_WINDOW_ADJUST, + cMSG_CHANNEL_DATA, + cMSG_CHANNEL_EXTENDED_DATA, + DEBUG, + ERROR, + cMSG_CHANNEL_SUCCESS, + cMSG_CHANNEL_FAILURE, + cMSG_CHANNEL_EOF, + cMSG_CHANNEL_CLOSE, +) +from paramiko.message import Message +from paramiko.ssh_exception import SSHException +from paramiko.file import BufferedFile +from paramiko.buffered_pipe import BufferedPipe, PipeTimeout +from paramiko import pipe +from paramiko.util import ClosingContextManager + + +def open_only(func): + """ + Decorator for `.Channel` methods which performs an openness check. + + :raises: + `.SSHException` -- If the wrapped method is called on an unopened + `.Channel`. + """ + + @wraps(func) + def _check(self, *args, **kwds): + if ( + self.closed + or self.eof_received + or self.eof_sent + or not self.active + ): + raise SSHException("Channel is not open") + return func(self, *args, **kwds) + + return _check + + +class Channel(ClosingContextManager): + """ + A secure tunnel across an SSH `.Transport`. A Channel is meant to behave + like a socket, and has an API that should be indistinguishable from the + Python socket API. + + Because SSH2 has a windowing kind of flow control, if you stop reading data + from a Channel and its buffer fills up, the server will be unable to send + you any more data until you read some of it. (This won't affect other + channels on the same transport -- all channels on a single transport are + flow-controlled independently.) Similarly, if the server isn't reading + data you send, calls to `send` may block, unless you set a timeout. This + is exactly like a normal network socket, so it shouldn't be too surprising. + + Instances of this class may be used as context managers. + """ + + def __init__(self, chanid): + """ + Create a new channel. The channel is not associated with any + particular session or `.Transport` until the Transport attaches it. + Normally you would only call this method from the constructor of a + subclass of `.Channel`. + + :param int chanid: + the ID of this channel, as passed by an existing `.Transport`. + """ + #: Channel ID + self.chanid = chanid + #: Remote channel ID + self.remote_chanid = 0 + #: `.Transport` managing this channel + self.transport = None + #: Whether the connection is presently active + self.active = False + self.eof_received = 0 + self.eof_sent = 0 + self.in_buffer = BufferedPipe() + self.in_stderr_buffer = BufferedPipe() + self.timeout = None + #: Whether the connection has been closed + self.closed = False + self.ultra_debug = False + self.lock = threading.Lock() + self.out_buffer_cv = threading.Condition(self.lock) + self.in_window_size = 0 + self.out_window_size = 0 + self.in_max_packet_size = 0 + self.out_max_packet_size = 0 + self.in_window_threshold = 0 + self.in_window_sofar = 0 + self.status_event = threading.Event() + self._name = str(chanid) + self.logger = util.get_logger("paramiko.transport") + self._pipe = None + self.event = threading.Event() + self.event_ready = False + self.combine_stderr = False + self.exit_status = -1 + self.origin_addr = None + + def __del__(self): + try: + self.close() + except: + pass + + def __repr__(self): + """ + Return a string representation of this object, for debugging. + """ + out = " 0: + out += " in-buffer={}".format(len(self.in_buffer)) + out += " -> " + repr(self.transport) + out += ">" + return out + + @open_only + def get_pty( + self, + term="vt100", + width=80, + height=24, + width_pixels=0, + height_pixels=0, + ): + """ + Request a pseudo-terminal from the server. This is usually used right + after creating a client channel, to ask the server to provide some + basic terminal semantics for a shell invoked with `invoke_shell`. + It isn't necessary (or desirable) to call this method if you're going + to execute a single command with `exec_command`. + + :param str term: the terminal type to emulate + (for example, ``'vt100'``) + :param int width: width (in characters) of the terminal screen + :param int height: height (in characters) of the terminal screen + :param int width_pixels: width (in pixels) of the terminal screen + :param int height_pixels: height (in pixels) of the terminal screen + + :raises: + `.SSHException` -- if the request was rejected or the channel was + closed + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("pty-req") + m.add_boolean(True) + m.add_string(term) + m.add_int(width) + m.add_int(height) + m.add_int(width_pixels) + m.add_int(height_pixels) + m.add_string(bytes()) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + @open_only + def invoke_shell(self): + """ + Request an interactive shell session on this channel. If the server + allows it, the channel will then be directly connected to the stdin, + stdout, and stderr of the shell. + + Normally you would call `get_pty` before this, in which case the + shell will operate through the pty, and the channel will be connected + to the stdin and stdout of the pty. + + When the shell exits, the channel will be closed and can't be reused. + You must open a new channel if you wish to open another shell. + + :raises: + `.SSHException` -- if the request was rejected or the channel was + closed + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("shell") + m.add_boolean(True) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + @open_only + def exec_command(self, command): + """ + Execute a command on the server. If the server allows it, the channel + will then be directly connected to the stdin, stdout, and stderr of + the command being executed. + + When the command finishes executing, the channel will be closed and + can't be reused. You must open a new channel if you wish to execute + another command. + + :param str command: a shell command to execute. + + :raises: + `.SSHException` -- if the request was rejected or the channel was + closed + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("exec") + m.add_boolean(True) + m.add_string(command) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + @open_only + def invoke_subsystem(self, subsystem): + """ + Request a subsystem on the server (for example, ``sftp``). If the + server allows it, the channel will then be directly connected to the + requested subsystem. + + When the subsystem finishes, the channel will be closed and can't be + reused. + + :param str subsystem: name of the subsystem being requested. + + :raises: + `.SSHException` -- if the request was rejected or the channel was + closed + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("subsystem") + m.add_boolean(True) + m.add_string(subsystem) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + @open_only + def resize_pty(self, width=80, height=24, width_pixels=0, height_pixels=0): + """ + Resize the pseudo-terminal. This can be used to change the width and + height of the terminal emulation created in a previous `get_pty` call. + + :param int width: new width (in characters) of the terminal screen + :param int height: new height (in characters) of the terminal screen + :param int width_pixels: new width (in pixels) of the terminal screen + :param int height_pixels: new height (in pixels) of the terminal screen + + :raises: + `.SSHException` -- if the request was rejected or the channel was + closed + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("window-change") + m.add_boolean(False) + m.add_int(width) + m.add_int(height) + m.add_int(width_pixels) + m.add_int(height_pixels) + self.transport._send_user_message(m) + + @open_only + def update_environment(self, environment): + """ + Updates this channel's remote shell environment. + + .. note:: + This operation is additive - i.e. the current environment is not + reset before the given environment variables are set. + + .. warning:: + Servers may silently reject some environment variables; see the + warning in `set_environment_variable` for details. + + :param dict environment: + a dictionary containing the name and respective values to set + :raises: + `.SSHException` -- if any of the environment variables was rejected + by the server or the channel was closed + """ + for name, value in environment.items(): + try: + self.set_environment_variable(name, value) + except SSHException as e: + err = 'Failed to set environment variable "{}".' + raise SSHException(err.format(name), e) + + @open_only + def set_environment_variable(self, name, value): + """ + Set the value of an environment variable. + + .. warning:: + The server may reject this request depending on its ``AcceptEnv`` + setting; such rejections will fail silently (which is common client + practice for this particular request type). Make sure you + understand your server's configuration before using! + + :param str name: name of the environment variable + :param str value: value of the environment variable + + :raises: + `.SSHException` -- if the request was rejected or the channel was + closed + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("env") + m.add_boolean(False) + m.add_string(name) + m.add_string(value) + self.transport._send_user_message(m) + + def exit_status_ready(self): + """ + Return true if the remote process has exited and returned an exit + status. You may use this to poll the process status if you don't + want to block in `recv_exit_status`. Note that the server may not + return an exit status in some cases (like bad servers). + + :return: + ``True`` if `recv_exit_status` will return immediately, else + ``False``. + + .. versionadded:: 1.7.3 + """ + return self.closed or self.status_event.is_set() + + def recv_exit_status(self): + """ + Return the exit status from the process on the server. This is + mostly useful for retrieving the results of an `exec_command`. + If the command hasn't finished yet, this method will wait until + it does, or until the channel is closed. If no exit status is + provided by the server, -1 is returned. + + .. warning:: + In some situations, receiving remote output larger than the current + `.Transport` or session's ``window_size`` (e.g. that set by the + ``default_window_size`` kwarg for `.Transport.__init__`) will cause + `.recv_exit_status` to hang indefinitely if it is called prior to a + sufficiently large `.Channel.recv` (or if there are no threads + calling `.Channel.recv` in the background). + + In these cases, ensuring that `.recv_exit_status` is called *after* + `.Channel.recv` (or, again, using threads) can avoid the hang. + + :return: the exit code (as an `int`) of the process on the server. + + .. versionadded:: 1.2 + """ + self.status_event.wait() + assert self.status_event.is_set() + return self.exit_status + + def send_exit_status(self, status): + """ + Send the exit status of an executed command to the client. (This + really only makes sense in server mode.) Many clients expect to + get some sort of status code back from an executed command after + it completes. + + :param int status: the exit code of the process + + .. versionadded:: 1.2 + """ + # in many cases, the channel will not still be open here. + # that's fine. + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("exit-status") + m.add_boolean(False) + m.add_int(status) + self.transport._send_user_message(m) + + @open_only + def request_x11( + self, + screen_number=0, + auth_protocol=None, + auth_cookie=None, + single_connection=False, + handler=None, + ): + """ + Request an x11 session on this channel. If the server allows it, + further x11 requests can be made from the server to the client, + when an x11 application is run in a shell session. + + From :rfc:`4254`:: + + It is RECOMMENDED that the 'x11 authentication cookie' that is + sent be a fake, random cookie, and that the cookie be checked and + replaced by the real cookie when a connection request is received. + + If you omit the auth_cookie, a new secure random 128-bit value will be + generated, used, and returned. You will need to use this value to + verify incoming x11 requests and replace them with the actual local + x11 cookie (which requires some knowledge of the x11 protocol). + + If a handler is passed in, the handler is called from another thread + whenever a new x11 connection arrives. The default handler queues up + incoming x11 connections, which may be retrieved using + `.Transport.accept`. The handler's calling signature is:: + + handler(channel: Channel, (address: str, port: int)) + + :param int screen_number: the x11 screen number (0, 10, etc.) + :param str auth_protocol: + the name of the X11 authentication method used; if none is given, + ``"MIT-MAGIC-COOKIE-1"`` is used + :param str auth_cookie: + hexadecimal string containing the x11 auth cookie; if none is + given, a secure random 128-bit value is generated + :param bool single_connection: + if True, only a single x11 connection will be forwarded (by + default, any number of x11 connections can arrive over this + session) + :param handler: + an optional callable handler to use for incoming X11 connections + :return: the auth_cookie used + """ + if auth_protocol is None: + auth_protocol = "MIT-MAGIC-COOKIE-1" + if auth_cookie is None: + auth_cookie = binascii.hexlify(os.urandom(16)) + + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("x11-req") + m.add_boolean(True) + m.add_boolean(single_connection) + m.add_string(auth_protocol) + m.add_string(auth_cookie) + m.add_int(screen_number) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + self.transport._set_x11_handler(handler) + return auth_cookie + + @open_only + def request_forward_agent(self, handler): + """ + Request for a forward SSH Agent on this channel. + This is only valid for an ssh-agent from OpenSSH !!! + + :param handler: + a required callable handler to use for incoming SSH Agent + connections + + :return: True if we are ok, else False + (at that time we always return ok) + + :raises: SSHException in case of channel problem. + """ + m = Message() + m.add_byte(cMSG_CHANNEL_REQUEST) + m.add_int(self.remote_chanid) + m.add_string("auth-agent-req@openssh.com") + m.add_boolean(False) + self.transport._send_user_message(m) + self.transport._set_forward_agent_handler(handler) + return True + + def get_transport(self): + """ + Return the `.Transport` associated with this channel. + """ + return self.transport + + def set_name(self, name): + """ + Set a name for this channel. Currently it's only used to set the name + of the channel in logfile entries. The name can be fetched with the + `get_name` method. + + :param str name: new channel name + """ + self._name = name + + def get_name(self): + """ + Get the name of this channel that was previously set by `set_name`. + """ + return self._name + + def get_id(self): + """ + Return the `int` ID # for this channel. + + The channel ID is unique across a `.Transport` and usually a small + number. It's also the number passed to + `.ServerInterface.check_channel_request` when determining whether to + accept a channel request in server mode. + """ + return self.chanid + + def set_combine_stderr(self, combine): + """ + Set whether stderr should be combined into stdout on this channel. + The default is ``False``, but in some cases it may be convenient to + have both streams combined. + + If this is ``False``, and `exec_command` is called (or ``invoke_shell`` + with no pty), output to stderr will not show up through the `recv` + and `recv_ready` calls. You will have to use `recv_stderr` and + `recv_stderr_ready` to get stderr output. + + If this is ``True``, data will never show up via `recv_stderr` or + `recv_stderr_ready`. + + :param bool combine: + ``True`` if stderr output should be combined into stdout on this + channel. + :return: the previous setting (a `bool`). + + .. versionadded:: 1.1 + """ + data = bytes() + self.lock.acquire() + try: + old = self.combine_stderr + self.combine_stderr = combine + if combine and not old: + # copy old stderr buffer into primary buffer + data = self.in_stderr_buffer.empty() + finally: + self.lock.release() + if len(data) > 0: + self._feed(data) + return old + + # ...socket API... + + def settimeout(self, timeout): + """ + Set a timeout on blocking read/write operations. The ``timeout`` + argument can be a nonnegative float expressing seconds, or ``None``. + If a float is given, subsequent channel read/write operations will + raise a timeout exception if the timeout period value has elapsed + before the operation has completed. Setting a timeout of ``None`` + disables timeouts on socket operations. + + ``chan.settimeout(0.0)`` is equivalent to ``chan.setblocking(0)``; + ``chan.settimeout(None)`` is equivalent to ``chan.setblocking(1)``. + + :param float timeout: + seconds to wait for a pending read/write operation before raising + ``socket.timeout``, or ``None`` for no timeout. + """ + self.timeout = timeout + + def gettimeout(self): + """ + Returns the timeout in seconds (as a float) associated with socket + operations, or ``None`` if no timeout is set. This reflects the last + call to `setblocking` or `settimeout`. + """ + return self.timeout + + def setblocking(self, blocking): + """ + Set blocking or non-blocking mode of the channel: if ``blocking`` is 0, + the channel is set to non-blocking mode; otherwise it's set to blocking + mode. Initially all channels are in blocking mode. + + In non-blocking mode, if a `recv` call doesn't find any data, or if a + `send` call can't immediately dispose of the data, an error exception + is raised. In blocking mode, the calls block until they can proceed. An + EOF condition is considered "immediate data" for `recv`, so if the + channel is closed in the read direction, it will never block. + + ``chan.setblocking(0)`` is equivalent to ``chan.settimeout(0)``; + ``chan.setblocking(1)`` is equivalent to ``chan.settimeout(None)``. + + :param int blocking: + 0 to set non-blocking mode; non-0 to set blocking mode. + """ + if blocking: + self.settimeout(None) + else: + self.settimeout(0.0) + + def getpeername(self): + """ + Return the address of the remote side of this Channel, if possible. + + This simply wraps `.Transport.getpeername`, used to provide enough of a + socket-like interface to allow asyncore to work. (asyncore likes to + call ``'getpeername'``.) + """ + return self.transport.getpeername() + + def close(self): + """ + Close the channel. All future read/write operations on the channel + will fail. The remote end will receive no more data (after queued data + is flushed). Channels are automatically closed when their `.Transport` + is closed or when they are garbage collected. + """ + self.lock.acquire() + try: + # only close the pipe when the user explicitly closes the channel. + # otherwise they will get unpleasant surprises. (and do it before + # checking self.closed, since the remote host may have already + # closed the connection.) + if self._pipe is not None: + self._pipe.close() + self._pipe = None + + if not self.active or self.closed: + return + msgs = self._close_internal() + finally: + self.lock.release() + for m in msgs: + if m is not None: + self.transport._send_user_message(m) + + def recv_ready(self): + """ + Returns true if data is buffered and ready to be read from this + channel. A ``False`` result does not mean that the channel has closed; + it means you may need to wait before more data arrives. + + :return: + ``True`` if a `recv` call on this channel would immediately return + at least one byte; ``False`` otherwise. + """ + return self.in_buffer.read_ready() + + def recv(self, nbytes): + """ + Receive data from the channel. The return value is a string + representing the data received. The maximum amount of data to be + received at once is specified by ``nbytes``. If a string of + length zero is returned, the channel stream has closed. + + :param int nbytes: maximum number of bytes to read. + :return: received data, as a `bytes`. + + :raises socket.timeout: + if no data is ready before the timeout set by `settimeout`. + """ + try: + out = self.in_buffer.read(nbytes, self.timeout) + except PipeTimeout: + raise socket.timeout() + + ack = self._check_add_window(len(out)) + # no need to hold the channel lock when sending this + if ack > 0: + m = Message() + m.add_byte(cMSG_CHANNEL_WINDOW_ADJUST) + m.add_int(self.remote_chanid) + m.add_int(ack) + self.transport._send_user_message(m) + + return out + + def recv_stderr_ready(self): + """ + Returns true if data is buffered and ready to be read from this + channel's stderr stream. Only channels using `exec_command` or + `invoke_shell` without a pty will ever have data on the stderr + stream. + + :return: + ``True`` if a `recv_stderr` call on this channel would immediately + return at least one byte; ``False`` otherwise. + + .. versionadded:: 1.1 + """ + return self.in_stderr_buffer.read_ready() + + def recv_stderr(self, nbytes): + """ + Receive data from the channel's stderr stream. Only channels using + `exec_command` or `invoke_shell` without a pty will ever have data + on the stderr stream. The return value is a string representing the + data received. The maximum amount of data to be received at once is + specified by ``nbytes``. If a string of length zero is returned, the + channel stream has closed. + + :param int nbytes: maximum number of bytes to read. + :return: received data as a `bytes` + + :raises socket.timeout: if no data is ready before the timeout set by + `settimeout`. + + .. versionadded:: 1.1 + """ + try: + out = self.in_stderr_buffer.read(nbytes, self.timeout) + except PipeTimeout: + raise socket.timeout() + + ack = self._check_add_window(len(out)) + # no need to hold the channel lock when sending this + if ack > 0: + m = Message() + m.add_byte(cMSG_CHANNEL_WINDOW_ADJUST) + m.add_int(self.remote_chanid) + m.add_int(ack) + self.transport._send_user_message(m) + + return out + + def send_ready(self): + """ + Returns true if data can be written to this channel without blocking. + This means the channel is either closed (so any write attempt would + return immediately) or there is at least one byte of space in the + outbound buffer. If there is at least one byte of space in the + outbound buffer, a `send` call will succeed immediately and return + the number of bytes actually written. + + :return: + ``True`` if a `send` call on this channel would immediately succeed + or fail + """ + self.lock.acquire() + try: + if self.closed or self.eof_sent: + return True + return self.out_window_size > 0 + finally: + self.lock.release() + + def send(self, s): + """ + Send data to the channel. Returns the number of bytes sent, or 0 if + the channel stream is closed. Applications are responsible for + checking that all data has been sent: if only some of the data was + transmitted, the application needs to attempt delivery of the remaining + data. + + :param bytes s: data to send + :return: number of bytes actually sent, as an `int` + + :raises socket.timeout: if no data could be sent before the timeout set + by `settimeout`. + """ + + m = Message() + m.add_byte(cMSG_CHANNEL_DATA) + m.add_int(self.remote_chanid) + return self._send(s, m) + + def send_stderr(self, s): + """ + Send data to the channel on the "stderr" stream. This is normally + only used by servers to send output from shell commands -- clients + won't use this. Returns the number of bytes sent, or 0 if the channel + stream is closed. Applications are responsible for checking that all + data has been sent: if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. + + :param bytes s: data to send. + :return: number of bytes actually sent, as an `int`. + + :raises socket.timeout: + if no data could be sent before the timeout set by `settimeout`. + + .. versionadded:: 1.1 + """ + + m = Message() + m.add_byte(cMSG_CHANNEL_EXTENDED_DATA) + m.add_int(self.remote_chanid) + m.add_int(1) + return self._send(s, m) + + def sendall(self, s): + """ + Send data to the channel, without allowing partial results. Unlike + `send`, this method continues to send data from the given string until + either all data has been sent or an error occurs. Nothing is returned. + + :param bytes s: data to send. + + :raises socket.timeout: + if sending stalled for longer than the timeout set by `settimeout`. + :raises socket.error: + if an error occurred before the entire string was sent. + + .. note:: + If the channel is closed while only part of the data has been + sent, there is no way to determine how much data (if any) was sent. + This is irritating, but identically follows Python's API. + """ + while s: + sent = self.send(s) + s = s[sent:] + return None + + def sendall_stderr(self, s): + """ + Send data to the channel's "stderr" stream, without allowing partial + results. Unlike `send_stderr`, this method continues to send data + from the given bytestring until all data has been sent or an error + occurs. Nothing is returned. + + :param bytes s: data to send to the client as "stderr" output. + + :raises socket.timeout: + if sending stalled for longer than the timeout set by `settimeout`. + :raises socket.error: + if an error occurred before the entire string was sent. + + .. versionadded:: 1.1 + """ + while s: + sent = self.send_stderr(s) + s = s[sent:] + return None + + def makefile(self, *params): + """ + Return a file-like object associated with this channel. The optional + ``mode`` and ``bufsize`` arguments are interpreted the same way as by + the built-in ``file()`` function in Python. + + :return: `.ChannelFile` object which can be used for Python file I/O. + """ + return ChannelFile(*([self] + list(params))) + + def makefile_stderr(self, *params): + """ + Return a file-like object associated with this channel's stderr + stream. Only channels using `exec_command` or `invoke_shell` + without a pty will ever have data on the stderr stream. + + The optional ``mode`` and ``bufsize`` arguments are interpreted the + same way as by the built-in ``file()`` function in Python. For a + client, it only makes sense to open this file for reading. For a + server, it only makes sense to open this file for writing. + + :returns: + `.ChannelStderrFile` object which can be used for Python file I/O. + + .. versionadded:: 1.1 + """ + return ChannelStderrFile(*([self] + list(params))) + + def makefile_stdin(self, *params): + """ + Return a file-like object associated with this channel's stdin + stream. + + The optional ``mode`` and ``bufsize`` arguments are interpreted the + same way as by the built-in ``file()`` function in Python. For a + client, it only makes sense to open this file for writing. For a + server, it only makes sense to open this file for reading. + + :returns: + `.ChannelStdinFile` object which can be used for Python file I/O. + + .. versionadded:: 2.6 + """ + return ChannelStdinFile(*([self] + list(params))) + + def fileno(self): + """ + Returns an OS-level file descriptor which can be used for polling, but + but not for reading or writing. This is primarily to allow Python's + ``select`` module to work. + + The first time ``fileno`` is called on a channel, a pipe is created to + simulate real OS-level file descriptor (FD) behavior. Because of this, + two OS-level FDs are created, which will use up FDs faster than normal. + (You won't notice this effect unless you have hundreds of channels + open at the same time.) + + :return: an OS-level file descriptor (`int`) + + .. warning:: + This method causes channel reads to be slightly less efficient. + """ + self.lock.acquire() + try: + if self._pipe is not None: + return self._pipe.fileno() + # create the pipe and feed in any existing data + self._pipe = pipe.make_pipe() + p1, p2 = pipe.make_or_pipe(self._pipe) + self.in_buffer.set_event(p1) + self.in_stderr_buffer.set_event(p2) + return self._pipe.fileno() + finally: + self.lock.release() + + def shutdown(self, how): + """ + Shut down one or both halves of the connection. If ``how`` is 0, + further receives are disallowed. If ``how`` is 1, further sends + are disallowed. If ``how`` is 2, further sends and receives are + disallowed. This closes the stream in one or both directions. + + :param int how: + 0 (stop receiving), 1 (stop sending), or 2 (stop receiving and + sending). + """ + if (how == 0) or (how == 2): + # feign "read" shutdown + self.eof_received = 1 + if (how == 1) or (how == 2): + self.lock.acquire() + try: + m = self._send_eof() + finally: + self.lock.release() + if m is not None and self.transport is not None: + self.transport._send_user_message(m) + + def shutdown_read(self): + """ + Shutdown the receiving side of this socket, closing the stream in + the incoming direction. After this call, future reads on this + channel will fail instantly. This is a convenience method, equivalent + to ``shutdown(0)``, for people who don't make it a habit to + memorize unix constants from the 1970s. + + .. versionadded:: 1.2 + """ + self.shutdown(0) + + def shutdown_write(self): + """ + Shutdown the sending side of this socket, closing the stream in + the outgoing direction. After this call, future writes on this + channel will fail instantly. This is a convenience method, equivalent + to ``shutdown(1)``, for people who don't make it a habit to + memorize unix constants from the 1970s. + + .. versionadded:: 1.2 + """ + self.shutdown(1) + + @property + def _closed(self): + # Concession to Python 3's socket API, which has a private ._closed + # attribute instead of a semipublic .closed attribute. + return self.closed + + # ...calls from Transport + + def _set_transport(self, transport): + self.transport = transport + self.logger = util.get_logger(self.transport.get_log_channel()) + + def _set_window(self, window_size, max_packet_size): + self.in_window_size = window_size + self.in_max_packet_size = max_packet_size + # threshold of bytes we receive before we bother to send + # a window update + self.in_window_threshold = window_size // 10 + self.in_window_sofar = 0 + self._log(DEBUG, "Max packet in: {} bytes".format(max_packet_size)) + + def _set_remote_channel(self, chanid, window_size, max_packet_size): + self.remote_chanid = chanid + self.out_window_size = window_size + self.out_max_packet_size = self.transport._sanitize_packet_size( + max_packet_size + ) + self.active = 1 + self._log( + DEBUG, "Max packet out: {} bytes".format(self.out_max_packet_size) + ) + + def _request_success(self, m): + self._log(DEBUG, "Sesch channel {} request ok".format(self.chanid)) + self.event_ready = True + self.event.set() + return + + def _request_failed(self, m): + self.lock.acquire() + try: + msgs = self._close_internal() + finally: + self.lock.release() + for m in msgs: + if m is not None: + self.transport._send_user_message(m) + + def _feed(self, m): + if isinstance(m, bytes): + # passed from _feed_extended + s = m + else: + s = m.get_binary() + self.in_buffer.feed(s) + + def _feed_extended(self, m): + code = m.get_int() + s = m.get_binary() + if code != 1: + self._log( + ERROR, "unknown extended_data type {}; discarding".format(code) + ) + return + if self.combine_stderr: + self._feed(s) + else: + self.in_stderr_buffer.feed(s) + + def _window_adjust(self, m): + nbytes = m.get_int() + self.lock.acquire() + try: + if self.ultra_debug: + self._log(DEBUG, "window up {}".format(nbytes)) + self.out_window_size += nbytes + self.out_buffer_cv.notify_all() + finally: + self.lock.release() + + def _handle_request(self, m): + key = m.get_text() + want_reply = m.get_boolean() + server = self.transport.server_object + ok = False + if key == "exit-status": + self.exit_status = m.get_int() + self.status_event.set() + ok = True + elif key == "xon-xoff": + # ignore + ok = True + elif key == "pty-req": + term = m.get_string() + width = m.get_int() + height = m.get_int() + pixelwidth = m.get_int() + pixelheight = m.get_int() + modes = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_pty_request( + self, term, width, height, pixelwidth, pixelheight, modes + ) + elif key == "shell": + if server is None: + ok = False + else: + ok = server.check_channel_shell_request(self) + elif key == "env": + name = m.get_string() + value = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_env_request(self, name, value) + elif key == "exec": + cmd = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_exec_request(self, cmd) + elif key == "subsystem": + name = m.get_text() + if server is None: + ok = False + else: + ok = server.check_channel_subsystem_request(self, name) + elif key == "window-change": + width = m.get_int() + height = m.get_int() + pixelwidth = m.get_int() + pixelheight = m.get_int() + if server is None: + ok = False + else: + ok = server.check_channel_window_change_request( + self, width, height, pixelwidth, pixelheight + ) + elif key == "x11-req": + single_connection = m.get_boolean() + auth_proto = m.get_text() + auth_cookie = m.get_binary() + screen_number = m.get_int() + if server is None: + ok = False + else: + ok = server.check_channel_x11_request( + self, + single_connection, + auth_proto, + auth_cookie, + screen_number, + ) + elif key == "auth-agent-req@openssh.com": + if server is None: + ok = False + else: + ok = server.check_channel_forward_agent_request(self) + else: + self._log(DEBUG, 'Unhandled channel request "{}"'.format(key)) + ok = False + if want_reply: + m = Message() + if ok: + m.add_byte(cMSG_CHANNEL_SUCCESS) + else: + m.add_byte(cMSG_CHANNEL_FAILURE) + m.add_int(self.remote_chanid) + self.transport._send_user_message(m) + + def _handle_eof(self, m): + self.lock.acquire() + try: + if not self.eof_received: + self.eof_received = True + self.in_buffer.close() + self.in_stderr_buffer.close() + if self._pipe is not None: + self._pipe.set_forever() + finally: + self.lock.release() + self._log(DEBUG, "EOF received ({})".format(self._name)) + + def _handle_close(self, m): + self.lock.acquire() + try: + msgs = self._close_internal() + self.transport._unlink_channel(self.chanid) + finally: + self.lock.release() + for m in msgs: + if m is not None: + self.transport._send_user_message(m) + + # ...internals... + + def _send(self, s, m): + size = len(s) + self.lock.acquire() + try: + if self.closed: + # this doesn't seem useful, but it is the documented behavior + # of Socket + raise socket.error("Socket is closed") + size = self._wait_for_send_window(size) + if size == 0: + # eof or similar + return 0 + m.add_string(s[:size]) + finally: + self.lock.release() + # Note: We release self.lock before calling _send_user_message. + # Otherwise, we can deadlock during re-keying. + self.transport._send_user_message(m) + return size + + def _log(self, level, msg, *args): + self.logger.log(level, "[chan " + self._name + "] " + msg, *args) + + def _event_pending(self): + self.event.clear() + self.event_ready = False + + def _wait_for_event(self): + self.event.wait() + assert self.event.is_set() + if self.event_ready: + return + e = self.transport.get_exception() + if e is None: + e = SSHException("Channel closed.") + raise e + + def _set_closed(self): + # you are holding the lock. + self.closed = True + self.in_buffer.close() + self.in_stderr_buffer.close() + self.out_buffer_cv.notify_all() + # Notify any waiters that we are closed + self.event.set() + self.status_event.set() + if self._pipe is not None: + self._pipe.set_forever() + + def _send_eof(self): + # you are holding the lock. + if self.eof_sent: + return None + m = Message() + m.add_byte(cMSG_CHANNEL_EOF) + m.add_int(self.remote_chanid) + self.eof_sent = True + self._log(DEBUG, "EOF sent ({})".format(self._name)) + return m + + def _close_internal(self): + # you are holding the lock. + if not self.active or self.closed: + return None, None + m1 = self._send_eof() + m2 = Message() + m2.add_byte(cMSG_CHANNEL_CLOSE) + m2.add_int(self.remote_chanid) + self._set_closed() + # can't unlink from the Transport yet -- the remote side may still + # try to send meta-data (exit-status, etc) + return m1, m2 + + def _unlink(self): + # server connection could die before we become active: + # still signal the close! + if self.closed: + return + self.lock.acquire() + try: + self._set_closed() + self.transport._unlink_channel(self.chanid) + finally: + self.lock.release() + + def _check_add_window(self, n): + self.lock.acquire() + try: + if self.closed or self.eof_received or not self.active: + return 0 + if self.ultra_debug: + self._log(DEBUG, "addwindow {}".format(n)) + self.in_window_sofar += n + if self.in_window_sofar <= self.in_window_threshold: + return 0 + if self.ultra_debug: + self._log( + DEBUG, "addwindow send {}".format(self.in_window_sofar) + ) + out = self.in_window_sofar + self.in_window_sofar = 0 + return out + finally: + self.lock.release() + + def _wait_for_send_window(self, size): + """ + (You are already holding the lock.) + Wait for the send window to open up, and allocate up to ``size`` bytes + for transmission. If no space opens up before the timeout, a timeout + exception is raised. Returns the number of bytes available to send + (may be less than requested). + """ + # you are already holding the lock + if self.closed or self.eof_sent: + return 0 + if self.out_window_size == 0: + # should we block? + if self.timeout == 0.0: + raise socket.timeout() + # loop here in case we get woken up but a different thread has + # filled the buffer + timeout = self.timeout + while self.out_window_size == 0: + if self.closed or self.eof_sent: + return 0 + then = time.time() + self.out_buffer_cv.wait(timeout) + if timeout is not None: + timeout -= time.time() - then + if timeout <= 0.0: + raise socket.timeout() + # we have some window to squeeze into + if self.closed or self.eof_sent: + return 0 + if self.out_window_size < size: + size = self.out_window_size + if self.out_max_packet_size - 64 < size: + size = self.out_max_packet_size - 64 + self.out_window_size -= size + if self.ultra_debug: + self._log(DEBUG, "window down to {}".format(self.out_window_size)) + return size + + +class ChannelFile(BufferedFile): + """ + A file-like wrapper around `.Channel`. A ChannelFile is created by calling + `Channel.makefile`. + + .. warning:: + To correctly emulate the file object created from a socket's `makefile + ` method, a `.Channel` and its + `.ChannelFile` should be able to be closed or garbage-collected + independently. Currently, closing the `ChannelFile` does nothing but + flush the buffer. + """ + + def __init__(self, channel, mode="r", bufsize=-1): + self.channel = channel + BufferedFile.__init__(self) + self._set_mode(mode, bufsize) + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + """ + return "" + + def _read(self, size): + return self.channel.recv(size) + + def _write(self, data): + self.channel.sendall(data) + return len(data) + + +class ChannelStderrFile(ChannelFile): + """ + A file-like wrapper around `.Channel` stderr. + + See `Channel.makefile_stderr` for details. + """ + + def _read(self, size): + return self.channel.recv_stderr(size) + + def _write(self, data): + self.channel.sendall_stderr(data) + return len(data) + + +class ChannelStdinFile(ChannelFile): + """ + A file-like wrapper around `.Channel` stdin. + + See `Channel.makefile_stdin` for details. + """ + + def close(self): + super().close() + self.channel.shutdown_write() diff --git a/lib/paramiko/client.py b/lib/paramiko/client.py new file mode 100644 index 0000000..1f674a9 --- /dev/null +++ b/lib/paramiko/client.py @@ -0,0 +1,889 @@ +# Copyright (C) 2006-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +SSH client & key policies +""" + +from binascii import hexlify +import getpass +import inspect +import os +import socket +import warnings +from errno import ECONNREFUSED, EHOSTUNREACH + +from paramiko.agent import Agent +from paramiko.common import DEBUG +from paramiko.config import SSH_PORT +from paramiko.ecdsakey import ECDSAKey +from paramiko.ed25519key import Ed25519Key +from paramiko.hostkeys import HostKeys +from paramiko.rsakey import RSAKey +from paramiko.ssh_exception import ( + SSHException, + BadHostKeyException, + NoValidConnectionsError, +) +from paramiko.transport import Transport +from paramiko.util import ClosingContextManager + + +class SSHClient(ClosingContextManager): + """ + A high-level representation of a session with an SSH server. This class + wraps `.Transport`, `.Channel`, and `.SFTPClient` to take care of most + aspects of authenticating and opening channels. A typical use case is:: + + client = SSHClient() + client.load_system_host_keys() + client.connect('ssh.example.com') + stdin, stdout, stderr = client.exec_command('ls -l') + + You may pass in explicit overrides for authentication and server host key + checking. The default mechanism is to try to use local key files or an + SSH agent (if one is running). + + Instances of this class may be used as context managers. + + .. versionadded:: 1.6 + """ + + def __init__(self): + """ + Create a new SSHClient. + """ + self._system_host_keys = HostKeys() + self._host_keys = HostKeys() + self._host_keys_filename = None + self._log_channel = None + self._policy = RejectPolicy() + self._transport = None + self._agent = None + + def load_system_host_keys(self, filename=None): + """ + Load host keys from a system (read-only) file. Host keys read with + this method will not be saved back by `save_host_keys`. + + This method can be called multiple times. Each new set of host keys + will be merged with the existing set (new replacing old if there are + conflicts). + + If ``filename`` is left as ``None``, an attempt will be made to read + keys from the user's local "known hosts" file, as used by OpenSSH, + and no exception will be raised if the file can't be read. This is + probably only useful on posix. + + :param str filename: the filename to read, or ``None`` + + :raises: ``IOError`` -- + if a filename was provided and the file could not be read + """ + if filename is None: + # try the user's .ssh key file, and mask exceptions + filename = os.path.expanduser("~/.ssh/known_hosts") + try: + self._system_host_keys.load(filename) + except IOError: + pass + return + self._system_host_keys.load(filename) + + def load_host_keys(self, filename): + """ + Load host keys from a local host-key file. Host keys read with this + method will be checked after keys loaded via `load_system_host_keys`, + but will be saved back by `save_host_keys` (so they can be modified). + The missing host key policy `.AutoAddPolicy` adds keys to this set and + saves them, when connecting to a previously-unknown server. + + This method can be called multiple times. Each new set of host keys + will be merged with the existing set (new replacing old if there are + conflicts). When automatically saving, the last hostname is used. + + :param str filename: the filename to read + + :raises: ``IOError`` -- if the filename could not be read + """ + self._host_keys_filename = filename + self._host_keys.load(filename) + + def save_host_keys(self, filename): + """ + Save the host keys back to a file. Only the host keys loaded with + `load_host_keys` (plus any added directly) will be saved -- not any + host keys loaded with `load_system_host_keys`. + + :param str filename: the filename to save to + + :raises: ``IOError`` -- if the file could not be written + """ + + # update local host keys from file (in case other SSH clients + # have written to the known_hosts file meanwhile. + if self._host_keys_filename is not None: + self.load_host_keys(self._host_keys_filename) + + with open(filename, "w") as f: + for hostname, keys in self._host_keys.items(): + for keytype, key in keys.items(): + f.write( + "{} {} {}\n".format( + hostname, keytype, key.get_base64() + ) + ) + + def get_host_keys(self): + """ + Get the local `.HostKeys` object. This can be used to examine the + local host keys or change them. + + :return: the local host keys as a `.HostKeys` object. + """ + return self._host_keys + + def set_log_channel(self, name): + """ + Set the channel for logging. The default is ``"paramiko.transport"`` + but it can be set to anything you want. + + :param str name: new channel name for logging + """ + self._log_channel = name + + def set_missing_host_key_policy(self, policy): + """ + Set policy to use when connecting to servers without a known host key. + + Specifically: + + * A **policy** is a "policy class" (or instance thereof), namely some + subclass of `.MissingHostKeyPolicy` such as `.RejectPolicy` (the + default), `.AutoAddPolicy`, `.WarningPolicy`, or a user-created + subclass. + * A host key is **known** when it appears in the client object's cached + host keys structures (those manipulated by `load_system_host_keys` + and/or `load_host_keys`). + + :param .MissingHostKeyPolicy policy: + the policy to use when receiving a host key from a + previously-unknown server + """ + if inspect.isclass(policy): + policy = policy() + self._policy = policy + + def _families_and_addresses(self, hostname, port): + """ + Yield pairs of address families and addresses to try for connecting. + + :param str hostname: the server to connect to + :param int port: the server port to connect to + :returns: Yields an iterable of ``(family, address)`` tuples + """ + guess = True + addrinfos = socket.getaddrinfo( + hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM + ) + for (family, socktype, proto, canonname, sockaddr) in addrinfos: + if socktype == socket.SOCK_STREAM: + yield family, sockaddr + guess = False + + # some OS like AIX don't indicate SOCK_STREAM support, so just + # guess. :( We only do this if we did not get a single result marked + # as socktype == SOCK_STREAM. + if guess: + for family, _, _, _, sockaddr in addrinfos: + yield family, sockaddr + + def connect( + self, + hostname, + port=SSH_PORT, + username=None, + password=None, + pkey=None, + key_filename=None, + timeout=None, + allow_agent=True, + look_for_keys=True, + compress=False, + sock=None, + gss_auth=False, + gss_kex=False, + gss_deleg_creds=True, + gss_host=None, + banner_timeout=None, + auth_timeout=None, + channel_timeout=None, + gss_trust_dns=True, + passphrase=None, + disabled_algorithms=None, + transport_factory=None, + auth_strategy=None, + ): + """ + Connect to an SSH server and authenticate to it. The server's host key + is checked against the system host keys (see `load_system_host_keys`) + and any local host keys (`load_host_keys`). If the server's hostname + is not found in either set of host keys, the missing host key policy + is used (see `set_missing_host_key_policy`). The default policy is + to reject the key and raise an `.SSHException`. + + Authentication is attempted in the following order of priority: + + - The ``pkey`` or ``key_filename`` passed in (if any) + + - ``key_filename`` may contain OpenSSH public certificate paths + as well as regular private-key paths; when files ending in + ``-cert.pub`` are found, they are assumed to match a private + key, and both components will be loaded. (The private key + itself does *not* need to be listed in ``key_filename`` for + this to occur - *just* the certificate.) + + - Any key we can find through an SSH agent + - Any ``id_*`` keys discoverable in ``~/.ssh/`` + + - When OpenSSH-style public certificates exist that match an + existing such private key (so e.g. one has ``id_rsa`` and + ``id_rsa-cert.pub``) the certificate will be loaded alongside + the private key and used for authentication. + + - Plain username/password auth, if a password was given + + If a private key requires a password to unlock it, and a password is + passed in, that password will be used to attempt to unlock the key. + + :param str hostname: the server to connect to + :param int port: the server port to connect to + :param str username: + the username to authenticate as (defaults to the current local + username) + :param str password: + Used for password authentication; is also used for private key + decryption if ``passphrase`` is not given. + :param str passphrase: + Used for decrypting private keys. + :param .PKey pkey: an optional private key to use for authentication + :param str key_filename: + the filename, or list of filenames, of optional private key(s) + and/or certs to try for authentication + :param float timeout: + an optional timeout (in seconds) for the TCP connect + :param bool allow_agent: + set to False to disable connecting to the SSH agent + :param bool look_for_keys: + set to False to disable searching for discoverable private key + files in ``~/.ssh/`` + :param bool compress: set to True to turn on compression + :param socket sock: + an open socket or socket-like object (such as a `.Channel`) to use + for communication to the target host + :param bool gss_auth: + ``True`` if you want to use GSS-API authentication + :param bool gss_kex: + Perform GSS-API Key Exchange and user authentication + :param bool gss_deleg_creds: Delegate GSS-API client credentials or not + :param str gss_host: + The targets name in the kerberos database. default: hostname + :param bool gss_trust_dns: + Indicates whether or not the DNS is trusted to securely + canonicalize the name of the host being connected to (default + ``True``). + :param float banner_timeout: an optional timeout (in seconds) to wait + for the SSH banner to be presented. + :param float auth_timeout: an optional timeout (in seconds) to wait for + an authentication response. + :param float channel_timeout: an optional timeout (in seconds) to wait + for a channel open response. + :param dict disabled_algorithms: + an optional dict passed directly to `.Transport` and its keyword + argument of the same name. + :param transport_factory: + an optional callable which is handed a subset of the constructor + arguments (primarily those related to the socket, GSS + functionality, and algorithm selection) and generates a + `.Transport` instance to be used by this client. Defaults to + `.Transport.__init__`. + :param auth_strategy: + an optional instance of `.AuthStrategy`, triggering use of this + newer authentication mechanism instead of SSHClient's legacy auth + method. + + .. warning:: + This parameter is **incompatible** with all other + authentication-related parameters (such as, but not limited to, + ``password``, ``key_filename`` and ``allow_agent``) and will + trigger an exception if given alongside them. + + :returns: + `.AuthResult` if ``auth_strategy`` is non-``None``; otherwise, + returns ``None``. + + :raises BadHostKeyException: + if the server's host key could not be verified. + :raises AuthenticationException: + if authentication failed. + :raises UnableToAuthenticate: + if authentication failed (when ``auth_strategy`` is non-``None``; + and note that this is a subclass of ``AuthenticationException``). + :raises socket.error: + if a socket error (other than connection-refused or + host-unreachable) occurred while connecting. + :raises NoValidConnectionsError: + if all valid connection targets for the requested hostname (eg IPv4 + and IPv6) yielded connection-refused or host-unreachable socket + errors. + :raises SSHException: + if there was any other error connecting or establishing an SSH + session. + + .. versionchanged:: 1.15 + Added the ``banner_timeout``, ``gss_auth``, ``gss_kex``, + ``gss_deleg_creds`` and ``gss_host`` arguments. + .. versionchanged:: 2.3 + Added the ``gss_trust_dns`` argument. + .. versionchanged:: 2.4 + Added the ``passphrase`` argument. + .. versionchanged:: 2.6 + Added the ``disabled_algorithms`` argument. + .. versionchanged:: 2.12 + Added the ``transport_factory`` argument. + .. versionchanged:: 3.2 + Added the ``auth_strategy`` argument. + """ + if not sock: + errors = {} + # Try multiple possible address families (e.g. IPv4 vs IPv6) + to_try = list(self._families_and_addresses(hostname, port)) + for af, addr in to_try: + try: + sock = socket.socket(af, socket.SOCK_STREAM) + if timeout is not None: + try: + sock.settimeout(timeout) + except: + pass + sock.connect(addr) + # Break out of the loop on success + break + except socket.error as e: + # As mentioned in socket docs it is better + # to close sockets explicitly + if sock: + sock.close() + # Raise anything that isn't a straight up connection error + # (such as a resolution error) + if e.errno not in (ECONNREFUSED, EHOSTUNREACH): + raise + # Capture anything else so we know how the run looks once + # iteration is complete. Retain info about which attempt + # this was. + errors[addr] = e + + # Make sure we explode usefully if no address family attempts + # succeeded. We've no way of knowing which error is the "right" + # one, so we construct a hybrid exception containing all the real + # ones, of a subclass that client code should still be watching for + # (socket.error) + if len(errors) == len(to_try): + raise NoValidConnectionsError(errors) + + if transport_factory is None: + transport_factory = Transport + t = self._transport = transport_factory( + sock, + gss_kex=gss_kex, + gss_deleg_creds=gss_deleg_creds, + disabled_algorithms=disabled_algorithms, + ) + t.use_compression(compress=compress) + t.set_gss_host( + # t.hostname may be None, but GSS-API requires a target name. + # Therefore use hostname as fallback. + gss_host=gss_host or hostname, + trust_dns=gss_trust_dns, + gssapi_requested=gss_auth or gss_kex, + ) + if self._log_channel is not None: + t.set_log_channel(self._log_channel) + if banner_timeout is not None: + t.banner_timeout = banner_timeout + if auth_timeout is not None: + t.auth_timeout = auth_timeout + if channel_timeout is not None: + t.channel_timeout = channel_timeout + + if port == SSH_PORT: + server_hostkey_name = hostname + else: + server_hostkey_name = "[{}]:{}".format(hostname, port) + our_server_keys = None + + our_server_keys = self._system_host_keys.get(server_hostkey_name) + if our_server_keys is None: + our_server_keys = self._host_keys.get(server_hostkey_name) + if our_server_keys is not None: + keytype = our_server_keys.keys()[0] + sec_opts = t.get_security_options() + other_types = [x for x in sec_opts.key_types if x != keytype] + sec_opts.key_types = [keytype] + other_types + + t.start_client(timeout=timeout) + + # If GSS-API Key Exchange is performed we are not required to check the + # host key, because the host is authenticated via GSS-API / SSPI as + # well as our client. + if not self._transport.gss_kex_used: + server_key = t.get_remote_server_key() + if our_server_keys is None: + # will raise exception if the key is rejected + self._policy.missing_host_key( + self, server_hostkey_name, server_key + ) + else: + our_key = our_server_keys.get(server_key.get_name()) + if our_key != server_key: + if our_key is None: + our_key = list(our_server_keys.values())[0] + raise BadHostKeyException(hostname, server_key, our_key) + + if username is None: + username = getpass.getuser() + + # New auth flow! + if auth_strategy is not None: + return auth_strategy.authenticate(transport=t) + + # Old auth flow! + if key_filename is None: + key_filenames = [] + elif isinstance(key_filename, str): + key_filenames = [key_filename] + else: + key_filenames = key_filename + + self._auth( + username, + password, + pkey, + key_filenames, + allow_agent, + look_for_keys, + gss_auth, + gss_kex, + gss_deleg_creds, + t.gss_host, + passphrase, + ) + + def close(self): + """ + Close this SSHClient and its underlying `.Transport`. + + This should be called anytime you are done using the client object. + + .. warning:: + Paramiko registers garbage collection hooks that will try to + automatically close connections for you, but this is not presently + reliable. Failure to explicitly close your client after use may + lead to end-of-process hangs! + """ + if self._transport is None: + return + self._transport.close() + self._transport = None + + if self._agent is not None: + self._agent.close() + self._agent = None + + def exec_command( + self, + command, + bufsize=-1, + timeout=None, + get_pty=False, + environment=None, + ): + """ + Execute a command on the SSH server. A new `.Channel` is opened and + the requested command is executed. The command's input and output + streams are returned as Python ``file``-like objects representing + stdin, stdout, and stderr. + + :param str command: the command to execute + :param int bufsize: + interpreted the same way as by the built-in ``file()`` function in + Python + :param int timeout: + set command's channel timeout. See `.Channel.settimeout` + :param bool get_pty: + Request a pseudo-terminal from the server (default ``False``). + See `.Channel.get_pty` + :param dict environment: + a dict of shell environment variables, to be merged into the + default environment that the remote command executes within. + + .. warning:: + Servers may silently reject some environment variables; see the + warning in `.Channel.set_environment_variable` for details. + + :return: + the stdin, stdout, and stderr of the executing command, as a + 3-tuple + + :raises: `.SSHException` -- if the server fails to execute the command + + .. versionchanged:: 1.10 + Added the ``get_pty`` kwarg. + """ + chan = self._transport.open_session(timeout=timeout) + if get_pty: + chan.get_pty() + chan.settimeout(timeout) + if environment: + chan.update_environment(environment) + chan.exec_command(command) + stdin = chan.makefile_stdin("wb", bufsize) + stdout = chan.makefile("r", bufsize) + stderr = chan.makefile_stderr("r", bufsize) + return stdin, stdout, stderr + + def invoke_shell( + self, + term="vt100", + width=80, + height=24, + width_pixels=0, + height_pixels=0, + environment=None, + ): + """ + Start an interactive shell session on the SSH server. A new `.Channel` + is opened and connected to a pseudo-terminal using the requested + terminal type and size. + + :param str term: + the terminal type to emulate (for example, ``"vt100"``) + :param int width: the width (in characters) of the terminal window + :param int height: the height (in characters) of the terminal window + :param int width_pixels: the width (in pixels) of the terminal window + :param int height_pixels: the height (in pixels) of the terminal window + :param dict environment: the command's environment + :return: a new `.Channel` connected to the remote shell + + :raises: `.SSHException` -- if the server fails to invoke a shell + """ + chan = self._transport.open_session() + chan.get_pty(term, width, height, width_pixels, height_pixels) + chan.invoke_shell() + return chan + + def open_sftp(self): + """ + Open an SFTP session on the SSH server. + + :return: a new `.SFTPClient` session object + """ + return self._transport.open_sftp_client() + + def get_transport(self): + """ + Return the underlying `.Transport` object for this SSH connection. + This can be used to perform lower-level tasks, like opening specific + kinds of channels. + + :return: the `.Transport` for this connection + """ + return self._transport + + def _key_from_filepath(self, filename, klass, password): + """ + Attempt to derive a `.PKey` from given string path ``filename``: + + - If ``filename`` appears to be a cert, the matching private key is + loaded. + - Otherwise, the filename is assumed to be a private key, and the + matching public cert will be loaded if it exists. + """ + cert_suffix = "-cert.pub" + # Assume privkey, not cert, by default + if filename.endswith(cert_suffix): + key_path = filename[: -len(cert_suffix)] + cert_path = filename + else: + key_path = filename + cert_path = filename + cert_suffix + # Blindly try the key path; if no private key, nothing will work. + key = klass.from_private_key_file(key_path, password) + # TODO: change this to 'Loading' instead of 'Trying' sometime; probably + # when #387 is released, since this is a critical log message users are + # likely testing/filtering for (bah.) + msg = "Trying discovered key {} in {}".format( + hexlify(key.get_fingerprint()), key_path + ) + self._log(DEBUG, msg) + # Attempt to load cert if it exists. + if os.path.isfile(cert_path): + key.load_certificate(cert_path) + self._log(DEBUG, "Adding public certificate {}".format(cert_path)) + return key + + def _auth( + self, + username, + password, + pkey, + key_filenames, + allow_agent, + look_for_keys, + gss_auth, + gss_kex, + gss_deleg_creds, + gss_host, + passphrase, + ): + """ + Try, in order: + + - The key(s) passed in, if one was passed in. + - Any key we can find through an SSH agent (if allowed). + - Any id_* key discoverable in ~/.ssh/ (if allowed). + - Plain username/password auth, if a password was given. + + (The password might be needed to unlock a private key [if 'passphrase' + isn't also given], or for two-factor authentication [for which it is + required].) + """ + saved_exception = None + two_factor = False + allowed_types = set() + two_factor_types = {"keyboard-interactive", "password"} + if passphrase is None and password is not None: + passphrase = password + + # If GSS-API support and GSS-PI Key Exchange was performed, we attempt + # authentication with gssapi-keyex. + if gss_kex and self._transport.gss_kex_used: + try: + self._transport.auth_gssapi_keyex(username) + return + except Exception as e: + saved_exception = e + + # Try GSS-API authentication (gssapi-with-mic) only if GSS-API Key + # Exchange is not performed, because if we use GSS-API for the key + # exchange, there is already a fully established GSS-API context, so + # why should we do that again? + if gss_auth: + try: + return self._transport.auth_gssapi_with_mic( + username, gss_host, gss_deleg_creds + ) + except Exception as e: + saved_exception = e + + if pkey is not None: + try: + self._log( + DEBUG, + "Trying SSH key {}".format( + hexlify(pkey.get_fingerprint()) + ), + ) + allowed_types = set( + self._transport.auth_publickey(username, pkey) + ) + two_factor = allowed_types & two_factor_types + if not two_factor: + return + except SSHException as e: + saved_exception = e + + if not two_factor: + for key_filename in key_filenames: + # TODO 4.0: leverage PKey.from_path() if we don't end up just + # killing SSHClient entirely + for pkey_class in (RSAKey, ECDSAKey, Ed25519Key): + try: + key = self._key_from_filepath( + key_filename, pkey_class, passphrase + ) + allowed_types = set( + self._transport.auth_publickey(username, key) + ) + two_factor = allowed_types & two_factor_types + if not two_factor: + return + break + except SSHException as e: + saved_exception = e + + if not two_factor and allow_agent: + if self._agent is None: + self._agent = Agent() + + for key in self._agent.get_keys(): + try: + id_ = hexlify(key.get_fingerprint()) + self._log(DEBUG, "Trying SSH agent key {}".format(id_)) + # for 2-factor auth a successfully auth'd key password + # will return an allowed 2fac auth method + allowed_types = set( + self._transport.auth_publickey(username, key) + ) + two_factor = allowed_types & two_factor_types + if not two_factor: + return + break + except SSHException as e: + saved_exception = e + + if not two_factor: + keyfiles = [] + + for keytype, name in [ + (RSAKey, "rsa"), + (ECDSAKey, "ecdsa"), + (Ed25519Key, "ed25519"), + ]: + # ~/ssh/ is for windows + for directory in [".ssh", "ssh"]: + full_path = os.path.expanduser( + "~/{}/id_{}".format(directory, name) + ) + if os.path.isfile(full_path): + # TODO: only do this append if below did not run + keyfiles.append((keytype, full_path)) + if os.path.isfile(full_path + "-cert.pub"): + keyfiles.append((keytype, full_path + "-cert.pub")) + + if not look_for_keys: + keyfiles = [] + + for pkey_class, filename in keyfiles: + try: + key = self._key_from_filepath( + filename, pkey_class, passphrase + ) + # for 2-factor auth a successfully auth'd key will result + # in ['password'] + allowed_types = set( + self._transport.auth_publickey(username, key) + ) + two_factor = allowed_types & two_factor_types + if not two_factor: + return + break + except (SSHException, IOError) as e: + saved_exception = e + + if password is not None: + try: + self._transport.auth_password(username, password) + return + except SSHException as e: + saved_exception = e + elif two_factor: + try: + self._transport.auth_interactive_dumb(username) + return + except SSHException as e: + saved_exception = e + + # if we got an auth-failed exception earlier, re-raise it + if saved_exception is not None: + raise saved_exception + raise SSHException("No authentication methods available") + + def _log(self, level, msg): + self._transport._log(level, msg) + + +class MissingHostKeyPolicy: + """ + Interface for defining the policy that `.SSHClient` should use when the + SSH server's hostname is not in either the system host keys or the + application's keys. Pre-made classes implement policies for automatically + adding the key to the application's `.HostKeys` object (`.AutoAddPolicy`), + and for automatically rejecting the key (`.RejectPolicy`). + + This function may be used to ask the user to verify the key, for example. + """ + + def missing_host_key(self, client, hostname, key): + """ + Called when an `.SSHClient` receives a server key for a server that + isn't in either the system or local `.HostKeys` object. To accept + the key, simply return. To reject, raised an exception (which will + be passed to the calling application). + """ + pass + + +class AutoAddPolicy(MissingHostKeyPolicy): + """ + Policy for automatically adding the hostname and new host key to the + local `.HostKeys` object, and saving it. This is used by `.SSHClient`. + """ + + def missing_host_key(self, client, hostname, key): + client._host_keys.add(hostname, key.get_name(), key) + if client._host_keys_filename is not None: + client.save_host_keys(client._host_keys_filename) + client._log( + DEBUG, + "Adding {} host key for {}: {}".format( + key.get_name(), hostname, hexlify(key.get_fingerprint()) + ), + ) + + +class RejectPolicy(MissingHostKeyPolicy): + """ + Policy for automatically rejecting the unknown hostname & key. This is + used by `.SSHClient`. + """ + + def missing_host_key(self, client, hostname, key): + client._log( + DEBUG, + "Rejecting {} host key for {}: {}".format( + key.get_name(), hostname, hexlify(key.get_fingerprint()) + ), + ) + raise SSHException( + "Server {!r} not found in known_hosts".format(hostname) + ) + + +class WarningPolicy(MissingHostKeyPolicy): + """ + Policy for logging a Python-style warning for an unknown host key, but + accepting it. This is used by `.SSHClient`. + """ + + def missing_host_key(self, client, hostname, key): + warnings.warn( + "Unknown {} host key for {}: {}".format( + key.get_name(), hostname, hexlify(key.get_fingerprint()) + ) + ) diff --git a/lib/paramiko/common.py b/lib/paramiko/common.py new file mode 100644 index 0000000..b57149b --- /dev/null +++ b/lib/paramiko/common.py @@ -0,0 +1,245 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Common constants and global variables. +""" +import logging +import struct + +# +# Formerly of py3compat.py. May be fully delete'able with a deeper look? +# + + +def byte_chr(c): + assert isinstance(c, int) + return struct.pack("B", c) + + +def byte_mask(c, mask): + assert isinstance(c, int) + return struct.pack("B", c & mask) + + +def byte_ord(c): + # In case we're handed a string instead of an int. + if not isinstance(c, int): + c = ord(c) + return c + + +( + MSG_DISCONNECT, + MSG_IGNORE, + MSG_UNIMPLEMENTED, + MSG_DEBUG, + MSG_SERVICE_REQUEST, + MSG_SERVICE_ACCEPT, + MSG_EXT_INFO, +) = range(1, 8) +(MSG_KEXINIT, MSG_NEWKEYS) = range(20, 22) +( + MSG_USERAUTH_REQUEST, + MSG_USERAUTH_FAILURE, + MSG_USERAUTH_SUCCESS, + MSG_USERAUTH_BANNER, +) = range(50, 54) +MSG_USERAUTH_PK_OK = 60 +(MSG_USERAUTH_INFO_REQUEST, MSG_USERAUTH_INFO_RESPONSE) = range(60, 62) +(MSG_USERAUTH_GSSAPI_RESPONSE, MSG_USERAUTH_GSSAPI_TOKEN) = range(60, 62) +( + MSG_USERAUTH_GSSAPI_EXCHANGE_COMPLETE, + MSG_USERAUTH_GSSAPI_ERROR, + MSG_USERAUTH_GSSAPI_ERRTOK, + MSG_USERAUTH_GSSAPI_MIC, +) = range(63, 67) +HIGHEST_USERAUTH_MESSAGE_ID = 79 +(MSG_GLOBAL_REQUEST, MSG_REQUEST_SUCCESS, MSG_REQUEST_FAILURE) = range(80, 83) +( + MSG_CHANNEL_OPEN, + MSG_CHANNEL_OPEN_SUCCESS, + MSG_CHANNEL_OPEN_FAILURE, + MSG_CHANNEL_WINDOW_ADJUST, + MSG_CHANNEL_DATA, + MSG_CHANNEL_EXTENDED_DATA, + MSG_CHANNEL_EOF, + MSG_CHANNEL_CLOSE, + MSG_CHANNEL_REQUEST, + MSG_CHANNEL_SUCCESS, + MSG_CHANNEL_FAILURE, +) = range(90, 101) + +cMSG_DISCONNECT = byte_chr(MSG_DISCONNECT) +cMSG_IGNORE = byte_chr(MSG_IGNORE) +cMSG_UNIMPLEMENTED = byte_chr(MSG_UNIMPLEMENTED) +cMSG_DEBUG = byte_chr(MSG_DEBUG) +cMSG_SERVICE_REQUEST = byte_chr(MSG_SERVICE_REQUEST) +cMSG_SERVICE_ACCEPT = byte_chr(MSG_SERVICE_ACCEPT) +cMSG_EXT_INFO = byte_chr(MSG_EXT_INFO) +cMSG_KEXINIT = byte_chr(MSG_KEXINIT) +cMSG_NEWKEYS = byte_chr(MSG_NEWKEYS) +cMSG_USERAUTH_REQUEST = byte_chr(MSG_USERAUTH_REQUEST) +cMSG_USERAUTH_FAILURE = byte_chr(MSG_USERAUTH_FAILURE) +cMSG_USERAUTH_SUCCESS = byte_chr(MSG_USERAUTH_SUCCESS) +cMSG_USERAUTH_BANNER = byte_chr(MSG_USERAUTH_BANNER) +cMSG_USERAUTH_PK_OK = byte_chr(MSG_USERAUTH_PK_OK) +cMSG_USERAUTH_INFO_REQUEST = byte_chr(MSG_USERAUTH_INFO_REQUEST) +cMSG_USERAUTH_INFO_RESPONSE = byte_chr(MSG_USERAUTH_INFO_RESPONSE) +cMSG_USERAUTH_GSSAPI_RESPONSE = byte_chr(MSG_USERAUTH_GSSAPI_RESPONSE) +cMSG_USERAUTH_GSSAPI_TOKEN = byte_chr(MSG_USERAUTH_GSSAPI_TOKEN) +cMSG_USERAUTH_GSSAPI_EXCHANGE_COMPLETE = byte_chr( + MSG_USERAUTH_GSSAPI_EXCHANGE_COMPLETE +) +cMSG_USERAUTH_GSSAPI_ERROR = byte_chr(MSG_USERAUTH_GSSAPI_ERROR) +cMSG_USERAUTH_GSSAPI_ERRTOK = byte_chr(MSG_USERAUTH_GSSAPI_ERRTOK) +cMSG_USERAUTH_GSSAPI_MIC = byte_chr(MSG_USERAUTH_GSSAPI_MIC) +cMSG_GLOBAL_REQUEST = byte_chr(MSG_GLOBAL_REQUEST) +cMSG_REQUEST_SUCCESS = byte_chr(MSG_REQUEST_SUCCESS) +cMSG_REQUEST_FAILURE = byte_chr(MSG_REQUEST_FAILURE) +cMSG_CHANNEL_OPEN = byte_chr(MSG_CHANNEL_OPEN) +cMSG_CHANNEL_OPEN_SUCCESS = byte_chr(MSG_CHANNEL_OPEN_SUCCESS) +cMSG_CHANNEL_OPEN_FAILURE = byte_chr(MSG_CHANNEL_OPEN_FAILURE) +cMSG_CHANNEL_WINDOW_ADJUST = byte_chr(MSG_CHANNEL_WINDOW_ADJUST) +cMSG_CHANNEL_DATA = byte_chr(MSG_CHANNEL_DATA) +cMSG_CHANNEL_EXTENDED_DATA = byte_chr(MSG_CHANNEL_EXTENDED_DATA) +cMSG_CHANNEL_EOF = byte_chr(MSG_CHANNEL_EOF) +cMSG_CHANNEL_CLOSE = byte_chr(MSG_CHANNEL_CLOSE) +cMSG_CHANNEL_REQUEST = byte_chr(MSG_CHANNEL_REQUEST) +cMSG_CHANNEL_SUCCESS = byte_chr(MSG_CHANNEL_SUCCESS) +cMSG_CHANNEL_FAILURE = byte_chr(MSG_CHANNEL_FAILURE) + +# for debugging: +MSG_NAMES = { + MSG_DISCONNECT: "disconnect", + MSG_IGNORE: "ignore", + MSG_UNIMPLEMENTED: "unimplemented", + MSG_DEBUG: "debug", + MSG_SERVICE_REQUEST: "service-request", + MSG_SERVICE_ACCEPT: "service-accept", + MSG_KEXINIT: "kexinit", + MSG_EXT_INFO: "ext-info", + MSG_NEWKEYS: "newkeys", + 30: "kex30", + 31: "kex31", + 32: "kex32", + 33: "kex33", + 34: "kex34", + 40: "kex40", + 41: "kex41", + MSG_USERAUTH_REQUEST: "userauth-request", + MSG_USERAUTH_FAILURE: "userauth-failure", + MSG_USERAUTH_SUCCESS: "userauth-success", + MSG_USERAUTH_BANNER: "userauth--banner", + MSG_USERAUTH_PK_OK: "userauth-60(pk-ok/info-request)", + MSG_USERAUTH_INFO_RESPONSE: "userauth-info-response", + MSG_GLOBAL_REQUEST: "global-request", + MSG_REQUEST_SUCCESS: "request-success", + MSG_REQUEST_FAILURE: "request-failure", + MSG_CHANNEL_OPEN: "channel-open", + MSG_CHANNEL_OPEN_SUCCESS: "channel-open-success", + MSG_CHANNEL_OPEN_FAILURE: "channel-open-failure", + MSG_CHANNEL_WINDOW_ADJUST: "channel-window-adjust", + MSG_CHANNEL_DATA: "channel-data", + MSG_CHANNEL_EXTENDED_DATA: "channel-extended-data", + MSG_CHANNEL_EOF: "channel-eof", + MSG_CHANNEL_CLOSE: "channel-close", + MSG_CHANNEL_REQUEST: "channel-request", + MSG_CHANNEL_SUCCESS: "channel-success", + MSG_CHANNEL_FAILURE: "channel-failure", + MSG_USERAUTH_GSSAPI_RESPONSE: "userauth-gssapi-response", + MSG_USERAUTH_GSSAPI_TOKEN: "userauth-gssapi-token", + MSG_USERAUTH_GSSAPI_EXCHANGE_COMPLETE: "userauth-gssapi-exchange-complete", + MSG_USERAUTH_GSSAPI_ERROR: "userauth-gssapi-error", + MSG_USERAUTH_GSSAPI_ERRTOK: "userauth-gssapi-error-token", + MSG_USERAUTH_GSSAPI_MIC: "userauth-gssapi-mic", +} + + +# authentication request return codes: +AUTH_SUCCESSFUL, AUTH_PARTIALLY_SUCCESSFUL, AUTH_FAILED = range(3) + + +# channel request failed reasons: +( + OPEN_SUCCEEDED, + OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, + OPEN_FAILED_CONNECT_FAILED, + OPEN_FAILED_UNKNOWN_CHANNEL_TYPE, + OPEN_FAILED_RESOURCE_SHORTAGE, +) = range(0, 5) + + +CONNECTION_FAILED_CODE = { + 1: "Administratively prohibited", + 2: "Connect failed", + 3: "Unknown channel type", + 4: "Resource shortage", +} + + +( + DISCONNECT_SERVICE_NOT_AVAILABLE, + DISCONNECT_AUTH_CANCELLED_BY_USER, + DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, +) = (7, 13, 14) + +zero_byte = byte_chr(0) +one_byte = byte_chr(1) +four_byte = byte_chr(4) +max_byte = byte_chr(0xFF) +cr_byte = byte_chr(13) +linefeed_byte = byte_chr(10) +crlf = cr_byte + linefeed_byte +cr_byte_value = 13 +linefeed_byte_value = 10 + + +xffffffff = 0xFFFFFFFF +x80000000 = 0x80000000 +o666 = 438 +o660 = 432 +o644 = 420 +o600 = 384 +o777 = 511 +o700 = 448 +o70 = 56 + +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL + +# Common IO/select/etc sleep period, in seconds +io_sleep = 0.01 + +DEFAULT_WINDOW_SIZE = 64 * 2**15 +DEFAULT_MAX_PACKET_SIZE = 2**15 + +# lower bound on the max packet size we'll accept from the remote host +# Minimum packet size is 32768 bytes according to +# http://www.ietf.org/rfc/rfc4254.txt +MIN_WINDOW_SIZE = 2**15 + +# However, according to http://www.ietf.org/rfc/rfc4253.txt it is perfectly +# legal to accept a size much smaller, as OpenSSH client does as size 16384. +MIN_PACKET_SIZE = 2**12 + +# Max windows size according to http://www.ietf.org/rfc/rfc4254.txt +MAX_WINDOW_SIZE = 2**32 - 1 diff --git a/lib/paramiko/compress.py b/lib/paramiko/compress.py new file mode 100644 index 0000000..18ff484 --- /dev/null +++ b/lib/paramiko/compress.py @@ -0,0 +1,40 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Compression implementations for a Transport. +""" + +import zlib + + +class ZlibCompressor: + def __init__(self): + # Use the default level of zlib compression + self.z = zlib.compressobj() + + def __call__(self, data): + return self.z.compress(data) + self.z.flush(zlib.Z_FULL_FLUSH) + + +class ZlibDecompressor: + def __init__(self): + self.z = zlib.decompressobj() + + def __call__(self, data): + return self.z.decompress(data) diff --git a/lib/paramiko/config.py b/lib/paramiko/config.py new file mode 100644 index 0000000..8ab55c6 --- /dev/null +++ b/lib/paramiko/config.py @@ -0,0 +1,696 @@ +# Copyright (C) 2006-2007 Robey Pointer +# Copyright (C) 2012 Olle Lundberg +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Configuration file (aka ``ssh_config``) support. +""" + +import fnmatch +import getpass +import os +import re +import shlex +import socket +from hashlib import sha1 +from io import StringIO +from functools import partial + +invoke, invoke_import_error = None, None +try: + import invoke +except ImportError as e: + invoke_import_error = e + +from .ssh_exception import CouldNotCanonicalize, ConfigParseError + + +SSH_PORT = 22 + + +class SSHConfig: + """ + Representation of config information as stored in the format used by + OpenSSH. Queries can be made via `lookup`. The format is described in + OpenSSH's ``ssh_config`` man page. This class is provided primarily as a + convenience to posix users (since the OpenSSH format is a de-facto + standard on posix) but should work fine on Windows too. + + .. versionadded:: 1.6 + """ + + SETTINGS_REGEX = re.compile(r"(\w+)(?:\s*=\s*|\s+)(.+)") + + # TODO: do a full scan of ssh.c & friends to make sure we're fully + # compatible across the board, e.g. OpenSSH 8.1 added %n to ProxyCommand. + TOKENS_BY_CONFIG_KEY = { + "controlpath": ["%C", "%h", "%l", "%L", "%n", "%p", "%r", "%u"], + "hostname": ["%h"], + "identityfile": ["%C", "~", "%d", "%h", "%l", "%u", "%r"], + "proxycommand": ["~", "%h", "%p", "%r"], + "proxyjump": ["%h", "%p", "%r"], + # Doesn't seem worth making this 'special' for now, it will fit well + # enough (no actual match-exec config key to be confused with). + "match-exec": ["%C", "%d", "%h", "%L", "%l", "%n", "%p", "%r", "%u"], + } + + def __init__(self): + """ + Create a new OpenSSH config object. + + Note: the newer alternate constructors `from_path`, `from_file` and + `from_text` are simpler to use, as they parse on instantiation. For + example, instead of:: + + config = SSHConfig() + config.parse(open("some-path.config") + + you could:: + + config = SSHConfig.from_file(open("some-path.config")) + # Or more directly: + config = SSHConfig.from_path("some-path.config") + # Or if you have arbitrary ssh_config text from some other source: + config = SSHConfig.from_text("Host foo\\n\\tUser bar") + """ + self._config = [] + + @classmethod + def from_text(cls, text): + """ + Create a new, parsed `SSHConfig` from ``text`` string. + + .. versionadded:: 2.7 + """ + return cls.from_file(StringIO(text)) + + @classmethod + def from_path(cls, path): + """ + Create a new, parsed `SSHConfig` from the file found at ``path``. + + .. versionadded:: 2.7 + """ + with open(path) as flo: + return cls.from_file(flo) + + @classmethod + def from_file(cls, flo): + """ + Create a new, parsed `SSHConfig` from file-like object ``flo``. + + .. versionadded:: 2.7 + """ + obj = cls() + obj.parse(flo) + return obj + + def parse(self, file_obj): + """ + Read an OpenSSH config from the given file object. + + :param file_obj: a file-like object to read the config file from + """ + # Start out w/ implicit/anonymous global host-like block to hold + # anything not contained by an explicit one. + context = {"host": ["*"], "config": {}} + for line in file_obj: + # Strip any leading or trailing whitespace from the line. + # Refer to https://github.com/paramiko/paramiko/issues/499 + line = line.strip() + # Skip blanks, comments + if not line or line.startswith("#"): + continue + + # Parse line into key, value + match = re.match(self.SETTINGS_REGEX, line) + if not match: + raise ConfigParseError("Unparsable line {}".format(line)) + key = match.group(1).lower() + value = match.group(2) + + # Host keyword triggers switch to new block/context + if key in ("host", "match"): + self._config.append(context) + context = {"config": {}} + if key == "host": + # TODO 4.0: make these real objects or at least name this + # "hosts" to acknowledge it's an iterable. (Doing so prior + # to 3.0, despite it being a private API, feels bad - + # surely such an old codebase has folks actually relying on + # these keys.) + context["host"] = self._get_hosts(value) + else: + context["matches"] = self._get_matches(value) + # Special-case for noop ProxyCommands + elif key == "proxycommand" and value.lower() == "none": + # Store 'none' as None - not as a string implying that the + # proxycommand is the literal shell command "none"! + context["config"][key] = None + # All other keywords get stored, directly or via append + else: + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + + # identityfile, localforward, remoteforward keys are special + # cases, since they are allowed to be specified multiple times + # and they should be tried in order of specification. + if key in ["identityfile", "localforward", "remoteforward"]: + if key in context["config"]: + context["config"][key].append(value) + else: + context["config"][key] = [value] + elif key not in context["config"]: + context["config"][key] = value + # Store last 'open' block and we're done + self._config.append(context) + + def lookup(self, hostname): + """ + Return a dict (`SSHConfigDict`) of config options for a given hostname. + + The host-matching rules of OpenSSH's ``ssh_config`` man page are used: + For each parameter, the first obtained value will be used. The + configuration files contain sections separated by ``Host`` and/or + ``Match`` specifications, and that section is only applied for hosts + which match the given patterns or keywords + + Since the first obtained value for each parameter is used, more host- + specific declarations should be given near the beginning of the file, + and general defaults at the end. + + The keys in the returned dict are all normalized to lowercase (look for + ``"port"``, not ``"Port"``. The values are processed according to the + rules for substitution variable expansion in ``ssh_config``. + + Finally, please see the docs for `SSHConfigDict` for deeper info on + features such as optional type conversion methods, e.g.:: + + conf = my_config.lookup('myhost') + assert conf['passwordauthentication'] == 'yes' + assert conf.as_bool('passwordauthentication') is True + + .. note:: + If there is no explicitly configured ``HostName`` value, it will be + set to the being-looked-up hostname, which is as close as we can + get to OpenSSH's behavior around that particular option. + + :param str hostname: the hostname to lookup + + .. versionchanged:: 2.5 + Returns `SSHConfigDict` objects instead of dict literals. + .. versionchanged:: 2.7 + Added canonicalization support. + .. versionchanged:: 2.7 + Added ``Match`` support. + .. versionchanged:: 3.3 + Added ``Match final`` support. + """ + # First pass + options = self._lookup(hostname=hostname) + # Inject HostName if it was not set (this used to be done incidentally + # during tokenization, for some reason). + if "hostname" not in options: + options["hostname"] = hostname + # Handle canonicalization + canon = options.get("canonicalizehostname", None) in ("yes", "always") + maxdots = int(options.get("canonicalizemaxdots", 1)) + if canon and hostname.count(".") <= maxdots: + # NOTE: OpenSSH manpage does not explicitly state this, but its + # implementation for CanonicalDomains is 'split on any whitespace'. + domains = options["canonicaldomains"].split() + hostname = self.canonicalize(hostname, options, domains) + # Overwrite HostName again here (this is also what OpenSSH does) + options["hostname"] = hostname + options = self._lookup( + hostname, options, canonical=True, final=True + ) + else: + options = self._lookup( + hostname, options, canonical=False, final=True + ) + return options + + def _lookup(self, hostname, options=None, canonical=False, final=False): + # Init + if options is None: + options = SSHConfigDict() + # Iterate all stanzas, applying any that match, in turn (so that things + # like Match can reference currently understood state) + for context in self._config: + if not ( + self._pattern_matches(context.get("host", []), hostname) + or self._does_match( + context.get("matches", []), + hostname, + canonical, + final, + options, + ) + ): + continue + for key, value in context["config"].items(): + if key not in options: + # Create a copy of the original value, + # else it will reference the original list + # in self._config and update that value too + # when the extend() is being called. + options[key] = value[:] if value is not None else value + elif key == "identityfile": + options[key].extend( + x for x in value if x not in options[key] + ) + if final: + # Expand variables in resulting values + # (besides 'Match exec' which was already handled above) + options = self._expand_variables(options, hostname) + return options + + def canonicalize(self, hostname, options, domains): + """ + Return canonicalized version of ``hostname``. + + :param str hostname: Target hostname. + :param options: An `SSHConfigDict` from a previous lookup pass. + :param domains: List of domains (e.g. ``["paramiko.org"]``). + + :returns: A canonicalized hostname if one was found, else ``None``. + + .. versionadded:: 2.7 + """ + found = False + for domain in domains: + candidate = "{}.{}".format(hostname, domain) + family_specific = _addressfamily_host_lookup(candidate, options) + if family_specific is not None: + # TODO: would we want to dig deeper into other results? e.g. to + # find something that satisfies PermittedCNAMEs when that is + # implemented? + found = family_specific[0] + else: + # TODO: what does ssh use here and is there a reason to use + # that instead of gethostbyname? + try: + found = socket.gethostbyname(candidate) + except socket.gaierror: + pass + if found: + # TODO: follow CNAME (implied by found != candidate?) if + # CanonicalizePermittedCNAMEs allows it + return candidate + # If we got here, it means canonicalization failed. + # When CanonicalizeFallbackLocal is undefined or 'yes', we just spit + # back the original hostname. + if options.get("canonicalizefallbacklocal", "yes") == "yes": + return hostname + # And here, we failed AND fallback was set to a non-yes value, so we + # need to get mad. + raise CouldNotCanonicalize(hostname) + + def get_hostnames(self): + """ + Return the set of literal hostnames defined in the SSH config (both + explicit hostnames and wildcard entries). + """ + hosts = set() + for entry in self._config: + hosts.update(entry["host"]) + return hosts + + def _pattern_matches(self, patterns, target): + # Convenience auto-splitter if not already a list + if hasattr(patterns, "split"): + patterns = patterns.split(",") + match = False + for pattern in patterns: + # Short-circuit if target matches a negated pattern + if pattern.startswith("!") and fnmatch.fnmatch( + target, pattern[1:] + ): + return False + # Flag a match, but continue (in case of later negation) if regular + # match occurs + elif fnmatch.fnmatch(target, pattern): + match = True + return match + + def _does_match( + self, match_list, target_hostname, canonical, final, options + ): + matched = [] + candidates = match_list[:] + local_username = getpass.getuser() + while candidates: + candidate = candidates.pop(0) + passed = None + # Obtain latest host/user value every loop, so later Match may + # reference values assigned within a prior Match. + configured_host = options.get("hostname", None) + configured_user = options.get("user", None) + type_, param = candidate["type"], candidate["param"] + # Canonical is a hard pass/fail based on whether this is a + # canonicalized re-lookup. + if type_ == "canonical": + if self._should_fail(canonical, candidate): + return False + if type_ == "final": + passed = final + # The parse step ensures we only see this by itself or after + # canonical, so it's also an easy hard pass. (No negation here as + # that would be uh, pretty weird?) + elif type_ == "all": + return True + # From here, we are testing various non-hard criteria, + # short-circuiting only on fail + elif type_ == "host": + hostval = configured_host or target_hostname + passed = self._pattern_matches(param, hostval) + elif type_ == "originalhost": + passed = self._pattern_matches(param, target_hostname) + elif type_ == "user": + user = configured_user or local_username + passed = self._pattern_matches(param, user) + elif type_ == "localuser": + passed = self._pattern_matches(param, local_username) + elif type_ == "exec": + exec_cmd = self._tokenize( + options, target_hostname, "match-exec", param + ) + # This is the laziest spot in which we can get mad about an + # inability to import Invoke. + if invoke is None: + raise invoke_import_error + # Like OpenSSH, we 'redirect' stdout but let stderr bubble up + passed = invoke.run(exec_cmd, hide="stdout", warn=True).ok + # Tackle any 'passed, but was negated' results from above + if passed is not None and self._should_fail(passed, candidate): + return False + # Made it all the way here? Everything matched! + matched.append(candidate) + # Did anything match? (To be treated as bool, usually.) + return matched + + def _should_fail(self, would_pass, candidate): + return would_pass if candidate["negate"] else not would_pass + + def _tokenize(self, config, target_hostname, key, value): + """ + Tokenize a string based on current config/hostname data. + + :param config: Current config data. + :param target_hostname: Original target connection hostname. + :param key: Config key being tokenized (used to filter token list). + :param value: Config value being tokenized. + + :returns: The tokenized version of the input ``value`` string. + """ + allowed_tokens = self._allowed_tokens(key) + # Short-circuit if no tokenization possible + if not allowed_tokens: + return value + # Obtain potentially configured hostname, for use with %h. + # Special-case where we are tokenizing the hostname itself, to avoid + # replacing %h with a %h-bearing value, etc. + configured_hostname = target_hostname + if key != "hostname": + configured_hostname = config.get("hostname", configured_hostname) + # Ditto the rest of the source values + if "port" in config: + port = config["port"] + else: + port = SSH_PORT + user = getpass.getuser() + if "user" in config: + remoteuser = config["user"] + else: + remoteuser = user + local_hostname = socket.gethostname().split(".")[0] + local_fqdn = LazyFqdn(config, local_hostname) + homedir = os.path.expanduser("~") + tohash = local_hostname + target_hostname + repr(port) + remoteuser + # The actual tokens! + replacements = { + # TODO: %%??? + "%C": sha1(tohash.encode()).hexdigest(), + "%d": homedir, + "%h": configured_hostname, + # TODO: %i? + "%L": local_hostname, + "%l": local_fqdn, + # also this is pseudo buggy when not in Match exec mode so document + # that. also WHY is that the case?? don't we do all of this late? + "%n": target_hostname, + "%p": port, + "%r": remoteuser, + # TODO: %T? don't believe this is possible however + "%u": user, + "~": homedir, + } + # Do the thing with the stuff + tokenized = value + for find, replace in replacements.items(): + if find not in allowed_tokens: + continue + tokenized = tokenized.replace(find, str(replace)) + # TODO: log? eg that value -> tokenized + return tokenized + + def _allowed_tokens(self, key): + """ + Given config ``key``, return list of token strings to tokenize. + + .. note:: + This feels like it wants to eventually go away, but is used to + preserve as-strict-as-possible compatibility with OpenSSH, which + for whatever reason only applies some tokens to some config keys. + """ + return self.TOKENS_BY_CONFIG_KEY.get(key, []) + + def _expand_variables(self, config, target_hostname): + """ + Return a dict of config options with expanded substitutions + for a given original & current target hostname. + + Please refer to :doc:`/api/config` for details. + + :param dict config: the currently parsed config + :param str hostname: the hostname whose config is being looked up + """ + for k in config: + if config[k] is None: + continue + tokenizer = partial(self._tokenize, config, target_hostname, k) + if isinstance(config[k], list): + for i, value in enumerate(config[k]): + config[k][i] = tokenizer(value) + else: + config[k] = tokenizer(config[k]) + return config + + def _get_hosts(self, host): + """ + Return a list of host_names from host value. + """ + try: + return shlex.split(host) + except ValueError: + raise ConfigParseError("Unparsable host {}".format(host)) + + def _get_matches(self, match): + """ + Parse a specific Match config line into a list-of-dicts for its values. + + Performs some parse-time validation as well. + """ + matches = [] + tokens = shlex.split(match) + while tokens: + match = {"type": None, "param": None, "negate": False} + type_ = tokens.pop(0) + # Handle per-keyword negation + if type_.startswith("!"): + match["negate"] = True + type_ = type_[1:] + match["type"] = type_ + # all/canonical have no params (everything else does) + if type_ in ("all", "canonical", "final"): + matches.append(match) + continue + if not tokens: + raise ConfigParseError( + "Missing parameter to Match '{}' keyword".format(type_) + ) + match["param"] = tokens.pop(0) + matches.append(match) + # Perform some (easier to do now than in the middle) validation that is + # better handled here than at lookup time. + keywords = [x["type"] for x in matches] + if "all" in keywords: + allowable = ("all", "canonical") + ok, bad = ( + list(filter(lambda x: x in allowable, keywords)), + list(filter(lambda x: x not in allowable, keywords)), + ) + err = None + if any(bad): + err = "Match does not allow 'all' mixed with anything but 'canonical'" # noqa + elif "canonical" in ok and ok.index("canonical") > ok.index("all"): + err = "Match does not allow 'all' before 'canonical'" + if err is not None: + raise ConfigParseError(err) + return matches + + +def _addressfamily_host_lookup(hostname, options): + """ + Try looking up ``hostname`` in an IPv4 or IPv6 specific manner. + + This is an odd duck due to needing use in two divergent use cases. It looks + up ``AddressFamily`` in ``options`` and if it is ``inet`` or ``inet6``, + this function uses `socket.getaddrinfo` to perform a family-specific + lookup, returning the result if successful. + + In any other situation -- lookup failure, or ``AddressFamily`` being + unspecified or ``any`` -- ``None`` is returned instead and the caller is + expected to do something situation-appropriate like calling + `socket.gethostbyname`. + + :param str hostname: Hostname to look up. + :param options: `SSHConfigDict` instance w/ parsed options. + :returns: ``getaddrinfo``-style tuples, or ``None``, depending. + """ + address_family = options.get("addressfamily", "any").lower() + if address_family == "any": + return + try: + family = socket.AF_INET6 + if address_family == "inet": + family = socket.AF_INET + return socket.getaddrinfo( + hostname, + None, + family, + socket.SOCK_DGRAM, + socket.IPPROTO_IP, + socket.AI_CANONNAME, + ) + except socket.gaierror: + pass + + +class LazyFqdn: + """ + Returns the host's fqdn on request as string. + """ + + def __init__(self, config, host=None): + self.fqdn = None + self.config = config + self.host = host + + def __str__(self): + if self.fqdn is None: + # + # If the SSH config contains AddressFamily, use that when + # determining the local host's FQDN. Using socket.getfqdn() from + # the standard library is the most general solution, but can + # result in noticeable delays on some platforms when IPv6 is + # misconfigured or not available, as it calls getaddrinfo with no + # address family specified, so both IPv4 and IPv6 are checked. + # + + # Handle specific option + fqdn = None + results = _addressfamily_host_lookup(self.host, self.config) + if results is not None: + for res in results: + af, socktype, proto, canonname, sa = res + if canonname and "." in canonname: + fqdn = canonname + break + # Handle 'any' / unspecified / lookup failure + if fqdn is None: + fqdn = socket.getfqdn() + # Cache + self.fqdn = fqdn + return self.fqdn + + +class SSHConfigDict(dict): + """ + A dictionary wrapper/subclass for per-host configuration structures. + + This class introduces some usage niceties for consumers of `SSHConfig`, + specifically around the issue of variable type conversions: normal value + access yields strings, but there are now methods such as `as_bool` and + `as_int` that yield casted values instead. + + For example, given the following ``ssh_config`` file snippet:: + + Host foo.example.com + PasswordAuthentication no + Compression yes + ServerAliveInterval 60 + + the following code highlights how you can access the raw strings as well as + usefully Python type-casted versions (recalling that keys are all + normalized to lowercase first):: + + my_config = SSHConfig() + my_config.parse(open('~/.ssh/config')) + conf = my_config.lookup('foo.example.com') + + assert conf['passwordauthentication'] == 'no' + assert conf.as_bool('passwordauthentication') is False + assert conf['compression'] == 'yes' + assert conf.as_bool('compression') is True + assert conf['serveraliveinterval'] == '60' + assert conf.as_int('serveraliveinterval') == 60 + + .. versionadded:: 2.5 + """ + + def as_bool(self, key): + """ + Express given key's value as a boolean type. + + Typically, this is used for ``ssh_config``'s pseudo-boolean values + which are either ``"yes"`` or ``"no"``. In such cases, ``"yes"`` yields + ``True`` and any other value becomes ``False``. + + .. note:: + If (for whatever reason) the stored value is already boolean in + nature, it's simply returned. + + .. versionadded:: 2.5 + """ + val = self[key] + if isinstance(val, bool): + return val + return val.lower() == "yes" + + def as_int(self, key): + """ + Express given key's value as an integer, if possible. + + This method will raise ``ValueError`` or similar if the value is not + int-appropriate, same as the builtin `int` type. + + .. versionadded:: 2.5 + """ + return int(self[key]) diff --git a/lib/paramiko/ecdsakey.py b/lib/paramiko/ecdsakey.py new file mode 100644 index 0000000..6fd95fa --- /dev/null +++ b/lib/paramiko/ecdsakey.py @@ -0,0 +1,339 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +ECDSA keys +""" + +from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric.utils import ( + decode_dss_signature, + encode_dss_signature, +) + +from paramiko.common import four_byte +from paramiko.message import Message +from paramiko.pkey import PKey +from paramiko.ssh_exception import SSHException +from paramiko.util import deflate_long + + +class _ECDSACurve: + """ + Represents a specific ECDSA Curve (nistp256, nistp384, etc). + + Handles the generation of the key format identifier and the selection of + the proper hash function. Also grabs the proper curve from the 'ecdsa' + package. + """ + + def __init__(self, curve_class, nist_name): + self.nist_name = nist_name + self.key_length = curve_class.key_size + + # Defined in RFC 5656 6.2 + self.key_format_identifier = "ecdsa-sha2-" + self.nist_name + + # Defined in RFC 5656 6.2.1 + if self.key_length <= 256: + self.hash_object = hashes.SHA256 + elif self.key_length <= 384: + self.hash_object = hashes.SHA384 + else: + self.hash_object = hashes.SHA512 + + self.curve_class = curve_class + + +class _ECDSACurveSet: + """ + A collection to hold the ECDSA curves. Allows querying by oid and by key + format identifier. The two ways in which ECDSAKey needs to be able to look + up curves. + """ + + def __init__(self, ecdsa_curves): + self.ecdsa_curves = ecdsa_curves + + def get_key_format_identifier_list(self): + return [curve.key_format_identifier for curve in self.ecdsa_curves] + + def get_by_curve_class(self, curve_class): + for curve in self.ecdsa_curves: + if curve.curve_class == curve_class: + return curve + + def get_by_key_format_identifier(self, key_format_identifier): + for curve in self.ecdsa_curves: + if curve.key_format_identifier == key_format_identifier: + return curve + + def get_by_key_length(self, key_length): + for curve in self.ecdsa_curves: + if curve.key_length == key_length: + return curve + + +class ECDSAKey(PKey): + """ + Representation of an ECDSA key which can be used to sign and verify SSH2 + data. + """ + + _ECDSA_CURVES = _ECDSACurveSet( + [ + _ECDSACurve(ec.SECP256R1, "nistp256"), + _ECDSACurve(ec.SECP384R1, "nistp384"), + _ECDSACurve(ec.SECP521R1, "nistp521"), + ] + ) + + def __init__( + self, + msg=None, + data=None, + filename=None, + password=None, + vals=None, + file_obj=None, + # TODO 4.0: remove; it does nothing since porting to cryptography.io + validate_point=True, + ): + self.verifying_key = None + self.signing_key = None + self.public_blob = None + if file_obj is not None: + self._from_private_key(file_obj, password) + return + if filename is not None: + self._from_private_key_file(filename, password) + return + if (msg is None) and (data is not None): + msg = Message(data) + if vals is not None: + self.signing_key, self.verifying_key = vals + c_class = self.signing_key.curve.__class__ + self.ecdsa_curve = self._ECDSA_CURVES.get_by_curve_class(c_class) + else: + # Must set ecdsa_curve first; subroutines called herein may need to + # spit out our get_name(), which relies on this. + key_type = msg.get_text() + # But this also means we need to hand it a real key/curve + # identifier, so strip out any cert business. (NOTE: could push + # that into _ECDSACurveSet.get_by_key_format_identifier(), but it + # feels more correct to do it here?) + suffix = "-cert-v01@openssh.com" + if key_type.endswith(suffix): + key_type = key_type[: -len(suffix)] + self.ecdsa_curve = self._ECDSA_CURVES.get_by_key_format_identifier( + key_type + ) + key_types = self._ECDSA_CURVES.get_key_format_identifier_list() + cert_types = [ + "{}-cert-v01@openssh.com".format(x) for x in key_types + ] + self._check_type_and_load_cert( + msg=msg, key_type=key_types, cert_type=cert_types + ) + curvename = msg.get_text() + if curvename != self.ecdsa_curve.nist_name: + raise SSHException( + "Can't handle curve of type {}".format(curvename) + ) + + pointinfo = msg.get_binary() + try: + key = ec.EllipticCurvePublicKey.from_encoded_point( + self.ecdsa_curve.curve_class(), pointinfo + ) + self.verifying_key = key + except ValueError: + raise SSHException("Invalid public key") + + @classmethod + def identifiers(cls): + return cls._ECDSA_CURVES.get_key_format_identifier_list() + + # TODO 4.0: deprecate/remove + @classmethod + def supported_key_format_identifiers(cls): + return cls.identifiers() + + def asbytes(self): + key = self.verifying_key + m = Message() + m.add_string(self.ecdsa_curve.key_format_identifier) + m.add_string(self.ecdsa_curve.nist_name) + + numbers = key.public_numbers() + + key_size_bytes = (key.curve.key_size + 7) // 8 + + x_bytes = deflate_long(numbers.x, add_sign_padding=False) + x_bytes = b"\x00" * (key_size_bytes - len(x_bytes)) + x_bytes + + y_bytes = deflate_long(numbers.y, add_sign_padding=False) + y_bytes = b"\x00" * (key_size_bytes - len(y_bytes)) + y_bytes + + point_str = four_byte + x_bytes + y_bytes + m.add_string(point_str) + return m.asbytes() + + def __str__(self): + return self.asbytes() + + @property + def _fields(self): + return ( + self.get_name(), + self.verifying_key.public_numbers().x, + self.verifying_key.public_numbers().y, + ) + + def get_name(self): + return self.ecdsa_curve.key_format_identifier + + def get_bits(self): + return self.ecdsa_curve.key_length + + def can_sign(self): + return self.signing_key is not None + + def sign_ssh_data(self, data, algorithm=None): + ecdsa = ec.ECDSA(self.ecdsa_curve.hash_object()) + sig = self.signing_key.sign(data, ecdsa) + r, s = decode_dss_signature(sig) + + m = Message() + m.add_string(self.ecdsa_curve.key_format_identifier) + m.add_string(self._sigencode(r, s)) + return m + + def verify_ssh_sig(self, data, msg): + if msg.get_text() != self.ecdsa_curve.key_format_identifier: + return False + sig = msg.get_binary() + sigR, sigS = self._sigdecode(sig) + signature = encode_dss_signature(sigR, sigS) + + try: + self.verifying_key.verify( + signature, data, ec.ECDSA(self.ecdsa_curve.hash_object()) + ) + except InvalidSignature: + return False + else: + return True + + def write_private_key_file(self, filename, password=None): + self._write_private_key_file( + filename, + self.signing_key, + serialization.PrivateFormat.TraditionalOpenSSL, + password=password, + ) + + def write_private_key(self, file_obj, password=None): + self._write_private_key( + file_obj, + self.signing_key, + serialization.PrivateFormat.TraditionalOpenSSL, + password=password, + ) + + @classmethod + def generate(cls, curve=ec.SECP256R1(), progress_func=None, bits=None): + """ + Generate a new private ECDSA key. This factory function can be used to + generate a new host key or authentication key. + + :param progress_func: Not used for this type of key. + :returns: A new private key (`.ECDSAKey`) object + """ + if bits is not None: + curve = cls._ECDSA_CURVES.get_by_key_length(bits) + if curve is None: + raise ValueError("Unsupported key length: {:d}".format(bits)) + curve = curve.curve_class() + + private_key = ec.generate_private_key(curve, backend=default_backend()) + return ECDSAKey(vals=(private_key, private_key.public_key())) + + # ...internals... + + def _from_private_key_file(self, filename, password): + data = self._read_private_key_file("EC", filename, password) + self._decode_key(data) + + def _from_private_key(self, file_obj, password): + data = self._read_private_key("EC", file_obj, password) + self._decode_key(data) + + def _decode_key(self, data): + pkformat, data = data + if pkformat == self._PRIVATE_KEY_FORMAT_ORIGINAL: + try: + key = serialization.load_der_private_key( + data, password=None, backend=default_backend() + ) + except ( + ValueError, + AssertionError, + TypeError, + UnsupportedAlgorithm, + ) as e: + raise SSHException(str(e)) + elif pkformat == self._PRIVATE_KEY_FORMAT_OPENSSH: + try: + msg = Message(data) + curve_name = msg.get_text() + verkey = msg.get_binary() # noqa: F841 + sigkey = msg.get_mpint() + name = "ecdsa-sha2-" + curve_name + curve = self._ECDSA_CURVES.get_by_key_format_identifier(name) + if not curve: + raise SSHException("Invalid key curve identifier") + key = ec.derive_private_key( + sigkey, curve.curve_class(), default_backend() + ) + except Exception as e: + # PKey._read_private_key_openssh() should check or return + # keytype - parsing could fail for any reason due to wrong type + raise SSHException(str(e)) + else: + self._got_bad_key_format_id(pkformat) + + self.signing_key = key + self.verifying_key = key.public_key() + curve_class = key.curve.__class__ + self.ecdsa_curve = self._ECDSA_CURVES.get_by_curve_class(curve_class) + + def _sigencode(self, r, s): + msg = Message() + msg.add_mpint(r) + msg.add_mpint(s) + return msg.asbytes() + + def _sigdecode(self, sig): + msg = Message(sig) + r = msg.get_mpint() + s = msg.get_mpint() + return r, s diff --git a/lib/paramiko/ed25519key.py b/lib/paramiko/ed25519key.py new file mode 100644 index 0000000..e5e81ac --- /dev/null +++ b/lib/paramiko/ed25519key.py @@ -0,0 +1,212 @@ +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import bcrypt + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher + +import nacl.signing + +from paramiko.message import Message +from paramiko.pkey import PKey, OPENSSH_AUTH_MAGIC, _unpad_openssh +from paramiko.util import b +from paramiko.ssh_exception import SSHException, PasswordRequiredException + + +class Ed25519Key(PKey): + """ + Representation of an `Ed25519 `_ key. + + .. note:: + Ed25519 key support was added to OpenSSH in version 6.5. + + .. versionadded:: 2.2 + .. versionchanged:: 2.3 + Added a ``file_obj`` parameter to match other key classes. + """ + + name = "ssh-ed25519" + + def __init__( + self, msg=None, data=None, filename=None, password=None, file_obj=None + ): + self.public_blob = None + verifying_key = signing_key = None + if msg is None and data is not None: + msg = Message(data) + if msg is not None: + self._check_type_and_load_cert( + msg=msg, + key_type=self.name, + cert_type="ssh-ed25519-cert-v01@openssh.com", + ) + verifying_key = nacl.signing.VerifyKey(msg.get_binary()) + elif filename is not None: + with open(filename, "r") as f: + pkformat, data = self._read_private_key("OPENSSH", f) + elif file_obj is not None: + pkformat, data = self._read_private_key("OPENSSH", file_obj) + + if filename or file_obj: + signing_key = self._parse_signing_key_data(data, password) + + if signing_key is None and verifying_key is None: + raise ValueError("need a key") + + self._signing_key = signing_key + self._verifying_key = verifying_key + + def _parse_signing_key_data(self, data, password): + from paramiko.transport import Transport + + # We may eventually want this to be usable for other key types, as + # OpenSSH moves to it, but for now this is just for Ed25519 keys. + # This format is described here: + # https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key + # The description isn't totally complete, and I had to refer to the + # source for a full implementation. + message = Message(data) + if message.get_bytes(len(OPENSSH_AUTH_MAGIC)) != OPENSSH_AUTH_MAGIC: + raise SSHException("Invalid key") + + ciphername = message.get_text() + kdfname = message.get_text() + kdfoptions = message.get_binary() + num_keys = message.get_int() + + if kdfname == "none": + # kdfname of "none" must have an empty kdfoptions, the ciphername + # must be "none" + if kdfoptions or ciphername != "none": + raise SSHException("Invalid key") + elif kdfname == "bcrypt": + if not password: + raise PasswordRequiredException( + "Private key file is encrypted" + ) + kdf = Message(kdfoptions) + bcrypt_salt = kdf.get_binary() + bcrypt_rounds = kdf.get_int() + else: + raise SSHException("Invalid key") + + if ciphername != "none" and ciphername not in Transport._cipher_info: + raise SSHException("Invalid key") + + public_keys = [] + for _ in range(num_keys): + pubkey = Message(message.get_binary()) + if pubkey.get_text() != self.name: + raise SSHException("Invalid key") + public_keys.append(pubkey.get_binary()) + + private_ciphertext = message.get_binary() + if ciphername == "none": + private_data = private_ciphertext + else: + cipher = Transport._cipher_info[ciphername] + key = bcrypt.kdf( + password=b(password), + salt=bcrypt_salt, + desired_key_bytes=cipher["key-size"] + cipher["block-size"], + rounds=bcrypt_rounds, + # We can't control how many rounds are on disk, so no sense + # warning about it. + ignore_few_rounds=True, + ) + decryptor = Cipher( + cipher["class"](key[: cipher["key-size"]]), + cipher["mode"](key[cipher["key-size"] :]), + backend=default_backend(), + ).decryptor() + private_data = ( + decryptor.update(private_ciphertext) + decryptor.finalize() + ) + + message = Message(_unpad_openssh(private_data)) + if message.get_int() != message.get_int(): + raise SSHException("Invalid key") + + signing_keys = [] + for i in range(num_keys): + if message.get_text() != self.name: + raise SSHException("Invalid key") + # A copy of the public key, again, ignore. + public = message.get_binary() + key_data = message.get_binary() + # The second half of the key data is yet another copy of the public + # key... + signing_key = nacl.signing.SigningKey(key_data[:32]) + # Verify that all the public keys are the same... + assert ( + signing_key.verify_key.encode() + == public + == public_keys[i] + == key_data[32:] + ) + signing_keys.append(signing_key) + # Comment, ignore. + message.get_binary() + + if len(signing_keys) != 1: + raise SSHException("Invalid key") + return signing_keys[0] + + def asbytes(self): + if self.can_sign(): + v = self._signing_key.verify_key + else: + v = self._verifying_key + m = Message() + m.add_string(self.name) + m.add_string(v.encode()) + return m.asbytes() + + @property + def _fields(self): + if self.can_sign(): + v = self._signing_key.verify_key + else: + v = self._verifying_key + return (self.get_name(), v) + + # TODO 4.0: remove + def get_name(self): + return self.name + + def get_bits(self): + return 256 + + def can_sign(self): + return self._signing_key is not None + + def sign_ssh_data(self, data, algorithm=None): + m = Message() + m.add_string(self.name) + m.add_string(self._signing_key.sign(data).signature) + return m + + def verify_ssh_sig(self, data, msg): + if msg.get_text() != self.name: + return False + + try: + self._verifying_key.verify(data, msg.get_binary()) + except nacl.exceptions.BadSignatureError: + return False + else: + return True diff --git a/lib/paramiko/file.py b/lib/paramiko/file.py new file mode 100644 index 0000000..a36abb9 --- /dev/null +++ b/lib/paramiko/file.py @@ -0,0 +1,528 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +from io import BytesIO + +from paramiko.common import ( + linefeed_byte_value, + crlf, + cr_byte, + linefeed_byte, + cr_byte_value, +) + +from paramiko.util import ClosingContextManager, u + + +class BufferedFile(ClosingContextManager): + """ + Reusable base class to implement Python-style file buffering around a + simpler stream. + """ + + _DEFAULT_BUFSIZE = 8192 + + SEEK_SET = 0 + SEEK_CUR = 1 + SEEK_END = 2 + + FLAG_READ = 0x1 + FLAG_WRITE = 0x2 + FLAG_APPEND = 0x4 + FLAG_BINARY = 0x10 + FLAG_BUFFERED = 0x20 + FLAG_LINE_BUFFERED = 0x40 + FLAG_UNIVERSAL_NEWLINE = 0x80 + + def __init__(self): + self.newlines = None + self._flags = 0 + self._bufsize = self._DEFAULT_BUFSIZE + self._wbuffer = BytesIO() + self._rbuffer = bytes() + self._at_trailing_cr = False + self._closed = False + # pos - position within the file, according to the user + # realpos - position according the OS + # (these may be different because we buffer for line reading) + self._pos = self._realpos = 0 + # size only matters for seekable files + self._size = 0 + + def __del__(self): + self.close() + + def __iter__(self): + """ + Returns an iterator that can be used to iterate over the lines in this + file. This iterator happens to return the file itself, since a file is + its own iterator. + + :raises: ``ValueError`` -- if the file is closed. + """ + if self._closed: + raise ValueError("I/O operation on closed file") + return self + + def close(self): + """ + Close the file. Future read and write operations will fail. + """ + self.flush() + self._closed = True + + def flush(self): + """ + Write out any data in the write buffer. This may do nothing if write + buffering is not turned on. + """ + self._write_all(self._wbuffer.getvalue()) + self._wbuffer = BytesIO() + return + + def __next__(self): + """ + Returns the next line from the input, or raises ``StopIteration`` + when EOF is hit. Unlike python file objects, it's okay to mix + calls to `.next` and `.readline`. + + :raises: ``StopIteration`` -- when the end of the file is reached. + + :returns: + a line (`str`, or `bytes` if the file was opened in binary mode) + read from the file. + """ + line = self.readline() + if not line: + raise StopIteration + return line + + def readable(self): + """ + Check if the file can be read from. + + :returns: + `True` if the file can be read from. If `False`, `read` will raise + an exception. + """ + return (self._flags & self.FLAG_READ) == self.FLAG_READ + + def writable(self): + """ + Check if the file can be written to. + + :returns: + `True` if the file can be written to. If `False`, `write` will + raise an exception. + """ + return (self._flags & self.FLAG_WRITE) == self.FLAG_WRITE + + def seekable(self): + """ + Check if the file supports random access. + + :returns: + `True` if the file supports random access. If `False`, `seek` will + raise an exception. + """ + return False + + def readinto(self, buff): + """ + Read up to ``len(buff)`` bytes into ``bytearray`` *buff* and return the + number of bytes read. + + :returns: + The number of bytes read. + """ + data = self.read(len(buff)) + buff[: len(data)] = data + return len(data) + + def read(self, size=None): + """ + Read at most ``size`` bytes from the file (less if we hit the end of + the file first). If the ``size`` argument is negative or omitted, + read all the remaining data in the file. + + .. note:: + ``'b'`` mode flag is ignored (``self.FLAG_BINARY`` in + ``self._flags``), because SSH treats all files as binary, since we + have no idea what encoding the file is in, or even if the file is + text data. + + :param int size: maximum number of bytes to read + :returns: + data read from the file (as bytes), or an empty string if EOF was + encountered immediately + """ + if self._closed: + raise IOError("File is closed") + if not (self._flags & self.FLAG_READ): + raise IOError("File is not open for reading") + if (size is None) or (size < 0): + # go for broke + result = bytearray(self._rbuffer) + self._rbuffer = bytes() + self._pos += len(result) + while True: + try: + new_data = self._read(self._DEFAULT_BUFSIZE) + except EOFError: + new_data = None + if (new_data is None) or (len(new_data) == 0): + break + result.extend(new_data) + self._realpos += len(new_data) + self._pos += len(new_data) + return bytes(result) + if size <= len(self._rbuffer): + result = self._rbuffer[:size] + self._rbuffer = self._rbuffer[size:] + self._pos += len(result) + return result + while len(self._rbuffer) < size: + read_size = size - len(self._rbuffer) + if self._flags & self.FLAG_BUFFERED: + read_size = max(self._bufsize, read_size) + try: + new_data = self._read(read_size) + except EOFError: + new_data = None + if (new_data is None) or (len(new_data) == 0): + break + self._rbuffer += new_data + self._realpos += len(new_data) + result = self._rbuffer[:size] + self._rbuffer = self._rbuffer[size:] + self._pos += len(result) + return result + + def readline(self, size=None): + """ + Read one entire line from the file. A trailing newline character is + kept in the string (but may be absent when a file ends with an + incomplete line). If the size argument is present and non-negative, it + is a maximum byte count (including the trailing newline) and an + incomplete line may be returned. An empty string is returned only when + EOF is encountered immediately. + + .. note:: + Unlike stdio's ``fgets``, the returned string contains null + characters (``'\\0'``) if they occurred in the input. + + :param int size: maximum length of returned string. + :returns: + next line of the file, or an empty string if the end of the + file has been reached. + + If the file was opened in binary (``'b'``) mode: bytes are returned + Else: the encoding of the file is assumed to be UTF-8 and character + strings (`str`) are returned + """ + # it's almost silly how complex this function is. + if self._closed: + raise IOError("File is closed") + if not (self._flags & self.FLAG_READ): + raise IOError("File not open for reading") + line = self._rbuffer + truncated = False + while True: + if ( + self._at_trailing_cr + and self._flags & self.FLAG_UNIVERSAL_NEWLINE + and len(line) > 0 + ): + # edge case: the newline may be '\r\n' and we may have read + # only the first '\r' last time. + if line[0] == linefeed_byte_value: + line = line[1:] + self._record_newline(crlf) + else: + self._record_newline(cr_byte) + self._at_trailing_cr = False + # check size before looking for a linefeed, in case we already have + # enough. + if (size is not None) and (size >= 0): + if len(line) >= size: + # truncate line + self._rbuffer = line[size:] + line = line[:size] + truncated = True + break + n = size - len(line) + else: + n = self._bufsize + if linefeed_byte in line or ( + self._flags & self.FLAG_UNIVERSAL_NEWLINE and cr_byte in line + ): + break + try: + new_data = self._read(n) + except EOFError: + new_data = None + if (new_data is None) or (len(new_data) == 0): + self._rbuffer = bytes() + self._pos += len(line) + return line if self._flags & self.FLAG_BINARY else u(line) + line += new_data + self._realpos += len(new_data) + # find the newline + pos = line.find(linefeed_byte) + if self._flags & self.FLAG_UNIVERSAL_NEWLINE: + rpos = line.find(cr_byte) + if (rpos >= 0) and (rpos < pos or pos < 0): + pos = rpos + if pos == -1: + # we couldn't find a newline in the truncated string, return it + self._pos += len(line) + return line if self._flags & self.FLAG_BINARY else u(line) + xpos = pos + 1 + if ( + line[pos] == cr_byte_value + and xpos < len(line) + and line[xpos] == linefeed_byte_value + ): + xpos += 1 + # if the string was truncated, _rbuffer needs to have the string after + # the newline character plus the truncated part of the line we stored + # earlier in _rbuffer + if truncated: + self._rbuffer = line[xpos:] + self._rbuffer + else: + self._rbuffer = line[xpos:] + + lf = line[pos:xpos] + line = line[:pos] + linefeed_byte + if (len(self._rbuffer) == 0) and (lf == cr_byte): + # we could read the line up to a '\r' and there could still be a + # '\n' following that we read next time. note that and eat it. + self._at_trailing_cr = True + else: + self._record_newline(lf) + self._pos += len(line) + return line if self._flags & self.FLAG_BINARY else u(line) + + def readlines(self, sizehint=None): + """ + Read all remaining lines using `readline` and return them as a list. + If the optional ``sizehint`` argument is present, instead of reading up + to EOF, whole lines totalling approximately sizehint bytes (possibly + after rounding up to an internal buffer size) are read. + + :param int sizehint: desired maximum number of bytes to read. + :returns: list of lines read from the file. + """ + lines = [] + byte_count = 0 + while True: + line = self.readline() + if len(line) == 0: + break + lines.append(line) + byte_count += len(line) + if (sizehint is not None) and (byte_count >= sizehint): + break + return lines + + def seek(self, offset, whence=0): + """ + Set the file's current position, like stdio's ``fseek``. Not all file + objects support seeking. + + .. note:: + If a file is opened in append mode (``'a'`` or ``'a+'``), any seek + operations will be undone at the next write (as the file position + will move back to the end of the file). + + :param int offset: + position to move to within the file, relative to ``whence``. + :param int whence: + type of movement: 0 = absolute; 1 = relative to the current + position; 2 = relative to the end of the file. + + :raises: ``IOError`` -- if the file doesn't support random access. + """ + raise IOError("File does not support seeking.") + + def tell(self): + """ + Return the file's current position. This may not be accurate or + useful if the underlying file doesn't support random access, or was + opened in append mode. + + :returns: file position (`number ` of bytes). + """ + return self._pos + + def write(self, data): + """ + Write data to the file. If write buffering is on (``bufsize`` was + specified and non-zero), some or all of the data may not actually be + written yet. (Use `flush` or `close` to force buffered data to be + written out.) + + :param data: ``str``/``bytes`` data to write + """ + if isinstance(data, str): + # Accept text and encode as utf-8 for compatibility only. + data = data.encode("utf-8") + if self._closed: + raise IOError("File is closed") + if not (self._flags & self.FLAG_WRITE): + raise IOError("File not open for writing") + if not (self._flags & self.FLAG_BUFFERED): + self._write_all(data) + return + self._wbuffer.write(data) + if self._flags & self.FLAG_LINE_BUFFERED: + # only scan the new data for linefeed, to avoid wasting time. + last_newline_pos = data.rfind(linefeed_byte) + if last_newline_pos >= 0: + wbuf = self._wbuffer.getvalue() + last_newline_pos += len(wbuf) - len(data) + self._write_all(wbuf[: last_newline_pos + 1]) + self._wbuffer = BytesIO() + self._wbuffer.write(wbuf[last_newline_pos + 1 :]) + return + # even if we're line buffering, if the buffer has grown past the + # buffer size, force a flush. + if self._wbuffer.tell() >= self._bufsize: + self.flush() + return + + def writelines(self, sequence): + """ + Write a sequence of strings to the file. The sequence can be any + iterable object producing strings, typically a list of strings. (The + name is intended to match `readlines`; `writelines` does not add line + separators.) + + :param sequence: an iterable sequence of strings. + """ + for line in sequence: + self.write(line) + return + + def xreadlines(self): + """ + Identical to ``iter(f)``. This is a deprecated file interface that + predates Python iterator support. + """ + return self + + @property + def closed(self): + return self._closed + + # ...overrides... + + def _read(self, size): + """ + (subclass override) + Read data from the stream. Return ``None`` or raise ``EOFError`` to + indicate EOF. + """ + raise EOFError() + + def _write(self, data): + """ + (subclass override) + Write data into the stream. + """ + raise IOError("write not implemented") + + def _get_size(self): + """ + (subclass override) + Return the size of the file. This is called from within `_set_mode` + if the file is opened in append mode, so the file position can be + tracked and `seek` and `tell` will work correctly. If the file is + a stream that can't be randomly accessed, you don't need to override + this method, + """ + return 0 + + # ...internals... + + def _set_mode(self, mode="r", bufsize=-1): + """ + Subclasses call this method to initialize the BufferedFile. + """ + # set bufsize in any event, because it's used for readline(). + self._bufsize = self._DEFAULT_BUFSIZE + if bufsize < 0: + # do no buffering by default, because otherwise writes will get + # buffered in a way that will probably confuse people. + bufsize = 0 + if bufsize == 1: + # apparently, line buffering only affects writes. reads are only + # buffered if you call readline (directly or indirectly: iterating + # over a file will indirectly call readline). + self._flags |= self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED + elif bufsize > 1: + self._bufsize = bufsize + self._flags |= self.FLAG_BUFFERED + self._flags &= ~self.FLAG_LINE_BUFFERED + elif bufsize == 0: + # unbuffered + self._flags &= ~(self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED) + + if ("r" in mode) or ("+" in mode): + self._flags |= self.FLAG_READ + if ("w" in mode) or ("+" in mode): + self._flags |= self.FLAG_WRITE + if "a" in mode: + self._flags |= self.FLAG_WRITE | self.FLAG_APPEND + self._size = self._get_size() + self._pos = self._realpos = self._size + if "b" in mode: + self._flags |= self.FLAG_BINARY + if "U" in mode: + self._flags |= self.FLAG_UNIVERSAL_NEWLINE + # built-in file objects have this attribute to store which kinds of + # line terminations they've seen: + # + self.newlines = None + + def _write_all(self, raw_data): + # the underlying stream may be something that does partial writes (like + # a socket). + data = memoryview(raw_data) + while len(data) > 0: + count = self._write(data) + data = data[count:] + if self._flags & self.FLAG_APPEND: + self._size += count + self._pos = self._realpos = self._size + else: + self._pos += count + self._realpos += count + return None + + def _record_newline(self, newline): + # silliness about tracking what kinds of newlines we've seen. + # i don't understand why it can be None, a string, or a tuple, instead + # of just always being a tuple, but we'll emulate that behavior anyway. + if not (self._flags & self.FLAG_UNIVERSAL_NEWLINE): + return + if self.newlines is None: + self.newlines = newline + elif self.newlines != newline and isinstance(self.newlines, bytes): + self.newlines = (self.newlines, newline) + elif newline not in self.newlines: + self.newlines += (newline,) diff --git a/lib/paramiko/hostkeys.py b/lib/paramiko/hostkeys.py new file mode 100644 index 0000000..0bcf6c3 --- /dev/null +++ b/lib/paramiko/hostkeys.py @@ -0,0 +1,384 @@ +# Copyright (C) 2006-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from base64 import encodebytes, decodebytes +import binascii +import os +import re + +from collections.abc import MutableMapping +from hashlib import sha1 +from hmac import HMAC + + +from paramiko.pkey import PKey, UnknownKeyType +from paramiko.util import get_logger, constant_time_bytes_eq, b, u +from paramiko.ssh_exception import SSHException + + +class HostKeys(MutableMapping): + """ + Representation of an OpenSSH-style "known hosts" file. Host keys can be + read from one or more files, and then individual hosts can be looked up to + verify server keys during SSH negotiation. + + A `.HostKeys` object can be treated like a dict; any dict lookup is + equivalent to calling `lookup`. + + .. versionadded:: 1.5.3 + """ + + def __init__(self, filename=None): + """ + Create a new HostKeys object, optionally loading keys from an OpenSSH + style host-key file. + + :param str filename: filename to load host keys from, or ``None`` + """ + # emulate a dict of { hostname: { keytype: PKey } } + self._entries = [] + if filename is not None: + self.load(filename) + + def add(self, hostname, keytype, key): + """ + Add a host key entry to the table. Any existing entry for a + ``(hostname, keytype)`` pair will be replaced. + + :param str hostname: the hostname (or IP) to add + :param str keytype: key type (in ``"ssh-"`` format) + :param .PKey key: the key to add + """ + for e in self._entries: + if (hostname in e.hostnames) and (e.key.get_name() == keytype): + e.key = key + return + self._entries.append(HostKeyEntry([hostname], key)) + + def load(self, filename): + """ + Read a file of known SSH host keys, in the format used by OpenSSH. + This type of file unfortunately doesn't exist on Windows, but on + posix, it will usually be stored in + ``os.path.expanduser("~/.ssh/known_hosts")``. + + If this method is called multiple times, the host keys are merged, + not cleared. So multiple calls to `load` will just call `add`, + replacing any existing entries and adding new ones. + + :param str filename: name of the file to read host keys from + + :raises: ``IOError`` -- if there was an error reading the file + """ + with open(filename, "r") as f: + for lineno, line in enumerate(f, 1): + line = line.strip() + if (len(line) == 0) or (line[0] == "#"): + continue + try: + entry = HostKeyEntry.from_line(line, lineno) + except SSHException: + continue + if entry is not None: + _hostnames = entry.hostnames + for h in _hostnames: + if self.check(h, entry.key): + entry.hostnames.remove(h) + if len(entry.hostnames): + self._entries.append(entry) + + def save(self, filename): + """ + Save host keys into a file, in the format used by OpenSSH. The order + of keys in the file will be preserved when possible (if these keys were + loaded from a file originally). The single exception is that combined + lines will be split into individual key lines, which is arguably a bug. + + :param str filename: name of the file to write + + :raises: ``IOError`` -- if there was an error writing the file + + .. versionadded:: 1.6.1 + """ + with open(filename, "w") as f: + for e in self._entries: + line = e.to_line() + if line: + f.write(line) + + def lookup(self, hostname): + """ + Find a hostkey entry for a given hostname or IP. If no entry is found, + ``None`` is returned. Otherwise a dictionary of keytype to key is + returned. + + :param str hostname: the hostname (or IP) to lookup + :return: dict of `str` -> `.PKey` keys associated with this host + (or ``None``) + """ + + class SubDict(MutableMapping): + def __init__(self, hostname, entries, hostkeys): + self._hostname = hostname + self._entries = entries + self._hostkeys = hostkeys + + def __iter__(self): + for k in self.keys(): + yield k + + def __len__(self): + return len(self.keys()) + + def __delitem__(self, key): + for e in list(self._entries): + if e.key.get_name() == key: + self._entries.remove(e) + break + else: + raise KeyError(key) + + def __getitem__(self, key): + for e in self._entries: + if e.key.get_name() == key: + return e.key + raise KeyError(key) + + def __setitem__(self, key, val): + for e in self._entries: + if e.key is None: + continue + if e.key.get_name() == key: + # replace + e.key = val + break + else: + # add a new one + e = HostKeyEntry([hostname], val) + self._entries.append(e) + self._hostkeys._entries.append(e) + + def keys(self): + return [ + e.key.get_name() + for e in self._entries + if e.key is not None + ] + + entries = [] + for e in self._entries: + if self._hostname_matches(hostname, e): + entries.append(e) + if len(entries) == 0: + return None + return SubDict(hostname, entries, self) + + def _hostname_matches(self, hostname, entry): + """ + Tests whether ``hostname`` string matches given SubDict ``entry``. + + :returns bool: + """ + for h in entry.hostnames: + if ( + h == hostname + or h.startswith("|1|") + and not hostname.startswith("|1|") + and constant_time_bytes_eq(self.hash_host(hostname, h), h) + ): + return True + return False + + def check(self, hostname, key): + """ + Return True if the given key is associated with the given hostname + in this dictionary. + + :param str hostname: hostname (or IP) of the SSH server + :param .PKey key: the key to check + :return: + ``True`` if the key is associated with the hostname; else ``False`` + """ + k = self.lookup(hostname) + if k is None: + return False + host_key = k.get(key.get_name(), None) + if host_key is None: + return False + return host_key.asbytes() == key.asbytes() + + def clear(self): + """ + Remove all host keys from the dictionary. + """ + self._entries = [] + + def __iter__(self): + for k in self.keys(): + yield k + + def __len__(self): + return len(self.keys()) + + def __getitem__(self, key): + ret = self.lookup(key) + if ret is None: + raise KeyError(key) + return ret + + def __delitem__(self, key): + index = None + for i, entry in enumerate(self._entries): + if self._hostname_matches(key, entry): + index = i + break + if index is None: + raise KeyError(key) + self._entries.pop(index) + + def __setitem__(self, hostname, entry): + # don't use this please. + if len(entry) == 0: + self._entries.append(HostKeyEntry([hostname], None)) + return + for key_type in entry.keys(): + found = False + for e in self._entries: + if (hostname in e.hostnames) and e.key.get_name() == key_type: + # replace + e.key = entry[key_type] + found = True + if not found: + self._entries.append(HostKeyEntry([hostname], entry[key_type])) + + def keys(self): + ret = [] + for e in self._entries: + for h in e.hostnames: + if h not in ret: + ret.append(h) + return ret + + def values(self): + ret = [] + for k in self.keys(): + ret.append(self.lookup(k)) + return ret + + @staticmethod + def hash_host(hostname, salt=None): + """ + Return a "hashed" form of the hostname, as used by OpenSSH when storing + hashed hostnames in the known_hosts file. + + :param str hostname: the hostname to hash + :param str salt: optional salt to use when hashing + (must be 20 bytes long) + :return: the hashed hostname as a `str` + """ + if salt is None: + salt = os.urandom(sha1().digest_size) + else: + if salt.startswith("|1|"): + salt = salt.split("|")[2] + salt = decodebytes(b(salt)) + assert len(salt) == sha1().digest_size + hmac = HMAC(salt, b(hostname), sha1).digest() + hostkey = "|1|{}|{}".format(u(encodebytes(salt)), u(encodebytes(hmac))) + return hostkey.replace("\n", "") + + +class InvalidHostKey(Exception): + def __init__(self, line, exc): + self.line = line + self.exc = exc + self.args = (line, exc) + + +class HostKeyEntry: + """ + Representation of a line in an OpenSSH-style "known hosts" file. + """ + + def __init__(self, hostnames=None, key=None): + self.valid = (hostnames is not None) and (key is not None) + self.hostnames = hostnames + self.key = key + + @classmethod + def from_line(cls, line, lineno=None): + """ + Parses the given line of text to find the names for the host, + the type of key, and the key data. The line is expected to be in the + format used by the OpenSSH known_hosts file. Fields are separated by a + single space or tab. + + Lines are expected to not have leading or trailing whitespace. + We don't bother to check for comments or empty lines. All of + that should be taken care of before sending the line to us. + + :param str line: a line from an OpenSSH known_hosts file + """ + log = get_logger("paramiko.hostkeys") + fields = re.split(" |\t", line) + if len(fields) < 3: + # Bad number of fields + msg = "Not enough fields found in known_hosts in line {} ({!r})" + log.info(msg.format(lineno, line)) + return None + fields = fields[:3] + + names, key_type, key = fields + names = names.split(",") + + # Decide what kind of key we're looking at and create an object + # to hold it accordingly. + try: + # TODO: this grew organically and doesn't seem /wrong/ per se (file + # read -> unicode str -> bytes for base64 decode -> decoded bytes); + # but in Python 3 forever land, can we simply use + # `base64.b64decode(str-from-file)` here? + key_bytes = decodebytes(b(key)) + except binascii.Error as e: + raise InvalidHostKey(line, e) + + try: + return cls(names, PKey.from_type_string(key_type, key_bytes)) + except UnknownKeyType: + # TODO 4.0: consider changing HostKeys API so this just raises + # naturally and the exception is muted higher up in the stack? + log.info("Unable to handle key of type {}".format(key_type)) + return None + + def to_line(self): + """ + Returns a string in OpenSSH known_hosts file format, or None if + the object is not in a valid state. A trailing newline is + included. + """ + if self.valid: + return "{} {} {}\n".format( + ",".join(self.hostnames), + self.key.get_name(), + self.key.get_base64(), + ) + return None + + def __repr__(self): + return "".format(self.hostnames, self.key) diff --git a/lib/paramiko/kex_curve25519.py b/lib/paramiko/kex_curve25519.py new file mode 100644 index 0000000..20c23e4 --- /dev/null +++ b/lib/paramiko/kex_curve25519.py @@ -0,0 +1,131 @@ +import binascii +import hashlib + +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.primitives import constant_time, serialization +from cryptography.hazmat.primitives.asymmetric.x25519 import ( + X25519PrivateKey, + X25519PublicKey, +) + +from paramiko.message import Message +from paramiko.common import byte_chr +from paramiko.ssh_exception import SSHException + + +_MSG_KEXECDH_INIT, _MSG_KEXECDH_REPLY = range(30, 32) +c_MSG_KEXECDH_INIT, c_MSG_KEXECDH_REPLY = [byte_chr(c) for c in range(30, 32)] + + +class KexCurve25519: + hash_algo = hashlib.sha256 + + def __init__(self, transport): + self.transport = transport + self.key = None + + @classmethod + def is_available(cls): + try: + X25519PrivateKey.generate() + except UnsupportedAlgorithm: + return False + else: + return True + + def _perform_exchange(self, peer_key): + secret = self.key.exchange(peer_key) + if constant_time.bytes_eq(secret, b"\x00" * 32): + raise SSHException( + "peer's curve25519 public value has wrong order" + ) + return secret + + def start_kex(self): + self.key = X25519PrivateKey.generate() + if self.transport.server_mode: + self.transport._expect_packet(_MSG_KEXECDH_INIT) + return + + m = Message() + m.add_byte(c_MSG_KEXECDH_INIT) + m.add_string( + self.key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + ) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXECDH_REPLY) + + def parse_next(self, ptype, m): + if self.transport.server_mode and (ptype == _MSG_KEXECDH_INIT): + return self._parse_kexecdh_init(m) + elif not self.transport.server_mode and (ptype == _MSG_KEXECDH_REPLY): + return self._parse_kexecdh_reply(m) + raise SSHException( + "KexCurve25519 asked to handle packet type {:d}".format(ptype) + ) + + def _parse_kexecdh_init(self, m): + peer_key_bytes = m.get_string() + peer_key = X25519PublicKey.from_public_bytes(peer_key_bytes) + K = self._perform_exchange(peer_key) + K = int(binascii.hexlify(K), 16) + # compute exchange hash + hm = Message() + hm.add( + self.transport.remote_version, + self.transport.local_version, + self.transport.remote_kex_init, + self.transport.local_kex_init, + ) + server_key_bytes = self.transport.get_server_key().asbytes() + exchange_key_bytes = self.key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + hm.add_string(server_key_bytes) + hm.add_string(peer_key_bytes) + hm.add_string(exchange_key_bytes) + hm.add_mpint(K) + H = self.hash_algo(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + sig = self.transport.get_server_key().sign_ssh_data( + H, self.transport.host_key_type + ) + # construct reply + m = Message() + m.add_byte(c_MSG_KEXECDH_REPLY) + m.add_string(server_key_bytes) + m.add_string(exchange_key_bytes) + m.add_string(sig) + self.transport._send_message(m) + self.transport._activate_outbound() + + def _parse_kexecdh_reply(self, m): + peer_host_key_bytes = m.get_string() + peer_key_bytes = m.get_string() + sig = m.get_binary() + + peer_key = X25519PublicKey.from_public_bytes(peer_key_bytes) + + K = self._perform_exchange(peer_key) + K = int(binascii.hexlify(K), 16) + # compute exchange hash and verify signature + hm = Message() + hm.add( + self.transport.local_version, + self.transport.remote_version, + self.transport.local_kex_init, + self.transport.remote_kex_init, + ) + hm.add_string(peer_host_key_bytes) + hm.add_string( + self.key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + ) + hm.add_string(peer_key_bytes) + hm.add_mpint(K) + self.transport._set_K_H(K, self.hash_algo(hm.asbytes()).digest()) + self.transport._verify_key(peer_host_key_bytes, sig) + self.transport._activate_outbound() diff --git a/lib/paramiko/kex_ecdh_nist.py b/lib/paramiko/kex_ecdh_nist.py new file mode 100644 index 0000000..41fab46 --- /dev/null +++ b/lib/paramiko/kex_ecdh_nist.py @@ -0,0 +1,151 @@ +""" +Ephemeral Elliptic Curve Diffie-Hellman (ECDH) key exchange +RFC 5656, Section 4 +""" + +from hashlib import sha256, sha384, sha512 +from paramiko.common import byte_chr +from paramiko.message import Message +from paramiko.ssh_exception import SSHException +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives import serialization +from binascii import hexlify + +_MSG_KEXECDH_INIT, _MSG_KEXECDH_REPLY = range(30, 32) +c_MSG_KEXECDH_INIT, c_MSG_KEXECDH_REPLY = [byte_chr(c) for c in range(30, 32)] + + +class KexNistp256: + + name = "ecdh-sha2-nistp256" + hash_algo = sha256 + curve = ec.SECP256R1() + + def __init__(self, transport): + self.transport = transport + # private key, client public and server public keys + self.P = 0 + self.Q_C = None + self.Q_S = None + + def start_kex(self): + self._generate_key_pair() + if self.transport.server_mode: + self.transport._expect_packet(_MSG_KEXECDH_INIT) + return + m = Message() + m.add_byte(c_MSG_KEXECDH_INIT) + # SEC1: V2.0 2.3.3 Elliptic-Curve-Point-to-Octet-String Conversion + m.add_string( + self.Q_C.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint, + ) + ) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXECDH_REPLY) + + def parse_next(self, ptype, m): + if self.transport.server_mode and (ptype == _MSG_KEXECDH_INIT): + return self._parse_kexecdh_init(m) + elif not self.transport.server_mode and (ptype == _MSG_KEXECDH_REPLY): + return self._parse_kexecdh_reply(m) + raise SSHException( + "KexECDH asked to handle packet type {:d}".format(ptype) + ) + + def _generate_key_pair(self): + self.P = ec.generate_private_key(self.curve, default_backend()) + if self.transport.server_mode: + self.Q_S = self.P.public_key() + return + self.Q_C = self.P.public_key() + + def _parse_kexecdh_init(self, m): + Q_C_bytes = m.get_string() + self.Q_C = ec.EllipticCurvePublicKey.from_encoded_point( + self.curve, Q_C_bytes + ) + K_S = self.transport.get_server_key().asbytes() + K = self.P.exchange(ec.ECDH(), self.Q_C) + K = int(hexlify(K), 16) + # compute exchange hash + hm = Message() + hm.add( + self.transport.remote_version, + self.transport.local_version, + self.transport.remote_kex_init, + self.transport.local_kex_init, + ) + hm.add_string(K_S) + hm.add_string(Q_C_bytes) + # SEC1: V2.0 2.3.3 Elliptic-Curve-Point-to-Octet-String Conversion + hm.add_string( + self.Q_S.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint, + ) + ) + hm.add_mpint(int(K)) + H = self.hash_algo(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + sig = self.transport.get_server_key().sign_ssh_data( + H, self.transport.host_key_type + ) + # construct reply + m = Message() + m.add_byte(c_MSG_KEXECDH_REPLY) + m.add_string(K_S) + m.add_string( + self.Q_S.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint, + ) + ) + m.add_string(sig) + self.transport._send_message(m) + self.transport._activate_outbound() + + def _parse_kexecdh_reply(self, m): + K_S = m.get_string() + Q_S_bytes = m.get_string() + self.Q_S = ec.EllipticCurvePublicKey.from_encoded_point( + self.curve, Q_S_bytes + ) + sig = m.get_binary() + K = self.P.exchange(ec.ECDH(), self.Q_S) + K = int(hexlify(K), 16) + # compute exchange hash and verify signature + hm = Message() + hm.add( + self.transport.local_version, + self.transport.remote_version, + self.transport.local_kex_init, + self.transport.remote_kex_init, + ) + hm.add_string(K_S) + # SEC1: V2.0 2.3.3 Elliptic-Curve-Point-to-Octet-String Conversion + hm.add_string( + self.Q_C.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint, + ) + ) + hm.add_string(Q_S_bytes) + hm.add_mpint(K) + self.transport._set_K_H(K, self.hash_algo(hm.asbytes()).digest()) + self.transport._verify_key(K_S, sig) + self.transport._activate_outbound() + + +class KexNistp384(KexNistp256): + name = "ecdh-sha2-nistp384" + hash_algo = sha384 + curve = ec.SECP384R1() + + +class KexNistp521(KexNistp256): + name = "ecdh-sha2-nistp521" + hash_algo = sha512 + curve = ec.SECP521R1() diff --git a/lib/paramiko/kex_gex.py b/lib/paramiko/kex_gex.py new file mode 100644 index 0000000..baa0803 --- /dev/null +++ b/lib/paramiko/kex_gex.py @@ -0,0 +1,288 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Variant on `KexGroup1 ` where the prime "p" and +generator "g" are provided by the server. A bit more work is required on the +client side, and a **lot** more on the server side. +""" + +import os +from hashlib import sha1, sha256 + +from paramiko import util +from paramiko.common import DEBUG, byte_chr, byte_ord, byte_mask +from paramiko.message import Message +from paramiko.ssh_exception import SSHException + + +( + _MSG_KEXDH_GEX_REQUEST_OLD, + _MSG_KEXDH_GEX_GROUP, + _MSG_KEXDH_GEX_INIT, + _MSG_KEXDH_GEX_REPLY, + _MSG_KEXDH_GEX_REQUEST, +) = range(30, 35) + +( + c_MSG_KEXDH_GEX_REQUEST_OLD, + c_MSG_KEXDH_GEX_GROUP, + c_MSG_KEXDH_GEX_INIT, + c_MSG_KEXDH_GEX_REPLY, + c_MSG_KEXDH_GEX_REQUEST, +) = [byte_chr(c) for c in range(30, 35)] + + +class KexGex: + + name = "diffie-hellman-group-exchange-sha1" + min_bits = 1024 + max_bits = 8192 + preferred_bits = 2048 + hash_algo = sha1 + + def __init__(self, transport): + self.transport = transport + self.p = None + self.q = None + self.g = None + self.x = None + self.e = None + self.f = None + self.old_style = False + + def start_kex(self, _test_old_style=False): + if self.transport.server_mode: + self.transport._expect_packet( + _MSG_KEXDH_GEX_REQUEST, _MSG_KEXDH_GEX_REQUEST_OLD + ) + return + # request a bit range: we accept (min_bits) to (max_bits), but prefer + # (preferred_bits). according to the spec, we shouldn't pull the + # minimum up above 1024. + m = Message() + if _test_old_style: + # only used for unit tests: we shouldn't ever send this + m.add_byte(c_MSG_KEXDH_GEX_REQUEST_OLD) + m.add_int(self.preferred_bits) + self.old_style = True + else: + m.add_byte(c_MSG_KEXDH_GEX_REQUEST) + m.add_int(self.min_bits) + m.add_int(self.preferred_bits) + m.add_int(self.max_bits) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_GROUP) + + def parse_next(self, ptype, m): + if ptype == _MSG_KEXDH_GEX_REQUEST: + return self._parse_kexdh_gex_request(m) + elif ptype == _MSG_KEXDH_GEX_GROUP: + return self._parse_kexdh_gex_group(m) + elif ptype == _MSG_KEXDH_GEX_INIT: + return self._parse_kexdh_gex_init(m) + elif ptype == _MSG_KEXDH_GEX_REPLY: + return self._parse_kexdh_gex_reply(m) + elif ptype == _MSG_KEXDH_GEX_REQUEST_OLD: + return self._parse_kexdh_gex_request_old(m) + msg = "KexGex {} asked to handle packet type {:d}" + raise SSHException(msg.format(self.name, ptype)) + + # ...internals... + + def _generate_x(self): + # generate an "x" (1 < x < (p-1)/2). + q = (self.p - 1) // 2 + qnorm = util.deflate_long(q, 0) + qhbyte = byte_ord(qnorm[0]) + byte_count = len(qnorm) + qmask = 0xFF + while not (qhbyte & 0x80): + qhbyte <<= 1 + qmask >>= 1 + while True: + x_bytes = os.urandom(byte_count) + x_bytes = byte_mask(x_bytes[0], qmask) + x_bytes[1:] + x = util.inflate_long(x_bytes, 1) + if (x > 1) and (x < q): + break + self.x = x + + def _parse_kexdh_gex_request(self, m): + minbits = m.get_int() + preferredbits = m.get_int() + maxbits = m.get_int() + # smoosh the user's preferred size into our own limits + if preferredbits > self.max_bits: + preferredbits = self.max_bits + if preferredbits < self.min_bits: + preferredbits = self.min_bits + # fix min/max if they're inconsistent. technically, we could just pout + # and hang up, but there's no harm in giving them the benefit of the + # doubt and just picking a bitsize for them. + if minbits > preferredbits: + minbits = preferredbits + if maxbits < preferredbits: + maxbits = preferredbits + # now save a copy + self.min_bits = minbits + self.preferred_bits = preferredbits + self.max_bits = maxbits + # generate prime + pack = self.transport._get_modulus_pack() + if pack is None: + raise SSHException("Can't do server-side gex with no modulus pack") + self.transport._log( + DEBUG, + "Picking p ({} <= {} <= {} bits)".format( + minbits, preferredbits, maxbits + ), + ) + self.g, self.p = pack.get_modulus(minbits, preferredbits, maxbits) + m = Message() + m.add_byte(c_MSG_KEXDH_GEX_GROUP) + m.add_mpint(self.p) + m.add_mpint(self.g) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_INIT) + + def _parse_kexdh_gex_request_old(self, m): + # same as above, but without min_bits or max_bits (used by older + # clients like putty) + self.preferred_bits = m.get_int() + # smoosh the user's preferred size into our own limits + if self.preferred_bits > self.max_bits: + self.preferred_bits = self.max_bits + if self.preferred_bits < self.min_bits: + self.preferred_bits = self.min_bits + # generate prime + pack = self.transport._get_modulus_pack() + if pack is None: + raise SSHException("Can't do server-side gex with no modulus pack") + self.transport._log( + DEBUG, "Picking p (~ {} bits)".format(self.preferred_bits) + ) + self.g, self.p = pack.get_modulus( + self.min_bits, self.preferred_bits, self.max_bits + ) + m = Message() + m.add_byte(c_MSG_KEXDH_GEX_GROUP) + m.add_mpint(self.p) + m.add_mpint(self.g) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_INIT) + self.old_style = True + + def _parse_kexdh_gex_group(self, m): + self.p = m.get_mpint() + self.g = m.get_mpint() + # reject if p's bit length < 1024 or > 8192 + bitlen = util.bit_length(self.p) + if (bitlen < 1024) or (bitlen > 8192): + raise SSHException( + "Server-generated gex p (don't ask) is out of range " + "({} bits)".format(bitlen) + ) + self.transport._log(DEBUG, "Got server p ({} bits)".format(bitlen)) + self._generate_x() + # now compute e = g^x mod p + self.e = pow(self.g, self.x, self.p) + m = Message() + m.add_byte(c_MSG_KEXDH_GEX_INIT) + m.add_mpint(self.e) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_REPLY) + + def _parse_kexdh_gex_init(self, m): + self.e = m.get_mpint() + if (self.e < 1) or (self.e > self.p - 1): + raise SSHException('Client kex "e" is out of range') + self._generate_x() + self.f = pow(self.g, self.x, self.p) + K = pow(self.e, self.x, self.p) + key = self.transport.get_server_key().asbytes() + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) # noqa + hm = Message() + hm.add( + self.transport.remote_version, + self.transport.local_version, + self.transport.remote_kex_init, + self.transport.local_kex_init, + key, + ) + if not self.old_style: + hm.add_int(self.min_bits) + hm.add_int(self.preferred_bits) + if not self.old_style: + hm.add_int(self.max_bits) + hm.add_mpint(self.p) + hm.add_mpint(self.g) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = self.hash_algo(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + # sign it + sig = self.transport.get_server_key().sign_ssh_data( + H, self.transport.host_key_type + ) + # send reply + m = Message() + m.add_byte(c_MSG_KEXDH_GEX_REPLY) + m.add_string(key) + m.add_mpint(self.f) + m.add_string(sig) + self.transport._send_message(m) + self.transport._activate_outbound() + + def _parse_kexdh_gex_reply(self, m): + host_key = m.get_string() + self.f = m.get_mpint() + sig = m.get_string() + if (self.f < 1) or (self.f > self.p - 1): + raise SSHException('Server kex "f" is out of range') + K = pow(self.f, self.x, self.p) + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) # noqa + hm = Message() + hm.add( + self.transport.local_version, + self.transport.remote_version, + self.transport.local_kex_init, + self.transport.remote_kex_init, + host_key, + ) + if not self.old_style: + hm.add_int(self.min_bits) + hm.add_int(self.preferred_bits) + if not self.old_style: + hm.add_int(self.max_bits) + hm.add_mpint(self.p) + hm.add_mpint(self.g) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + self.transport._set_K_H(K, self.hash_algo(hm.asbytes()).digest()) + self.transport._verify_key(host_key, sig) + self.transport._activate_outbound() + + +class KexGexSHA256(KexGex): + name = "diffie-hellman-group-exchange-sha256" + hash_algo = sha256 diff --git a/lib/paramiko/kex_group1.py b/lib/paramiko/kex_group1.py new file mode 100644 index 0000000..f074256 --- /dev/null +++ b/lib/paramiko/kex_group1.py @@ -0,0 +1,155 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Standard SSH key exchange ("kex" if you wanna sound cool). Diffie-Hellman of +1024 bit key halves, using a known "p" prime and "g" generator. +""" + +import os +from hashlib import sha1 + +from paramiko import util +from paramiko.common import max_byte, zero_byte, byte_chr, byte_mask +from paramiko.message import Message +from paramiko.ssh_exception import SSHException + + +_MSG_KEXDH_INIT, _MSG_KEXDH_REPLY = range(30, 32) +c_MSG_KEXDH_INIT, c_MSG_KEXDH_REPLY = [byte_chr(c) for c in range(30, 32)] + +b7fffffffffffffff = byte_chr(0x7F) + max_byte * 7 +b0000000000000000 = zero_byte * 8 + + +class KexGroup1: + + # draft-ietf-secsh-transport-09.txt, page 17 + P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFF # noqa + G = 2 + + name = "diffie-hellman-group1-sha1" + hash_algo = sha1 + + def __init__(self, transport): + self.transport = transport + self.x = 0 + self.e = 0 + self.f = 0 + + def start_kex(self): + self._generate_x() + if self.transport.server_mode: + # compute f = g^x mod p, but don't send it yet + self.f = pow(self.G, self.x, self.P) + self.transport._expect_packet(_MSG_KEXDH_INIT) + return + # compute e = g^x mod p (where g=2), and send it + self.e = pow(self.G, self.x, self.P) + m = Message() + m.add_byte(c_MSG_KEXDH_INIT) + m.add_mpint(self.e) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_REPLY) + + def parse_next(self, ptype, m): + if self.transport.server_mode and (ptype == _MSG_KEXDH_INIT): + return self._parse_kexdh_init(m) + elif not self.transport.server_mode and (ptype == _MSG_KEXDH_REPLY): + return self._parse_kexdh_reply(m) + msg = "KexGroup1 asked to handle packet type {:d}" + raise SSHException(msg.format(ptype)) + + # ...internals... + + def _generate_x(self): + # generate an "x" (1 < x < q), where q is (p-1)/2. + # p is a 128-byte (1024-bit) number, where the first 64 bits are 1. + # therefore q can be approximated as a 2^1023. we drop the subset of + # potential x where the first 63 bits are 1, because some of those + # will be larger than q (but this is a tiny tiny subset of + # potential x). + while 1: + x_bytes = os.urandom(128) + x_bytes = byte_mask(x_bytes[0], 0x7F) + x_bytes[1:] + if ( + x_bytes[:8] != b7fffffffffffffff + and x_bytes[:8] != b0000000000000000 + ): + break + self.x = util.inflate_long(x_bytes) + + def _parse_kexdh_reply(self, m): + # client mode + host_key = m.get_string() + self.f = m.get_mpint() + if (self.f < 1) or (self.f > self.P - 1): + raise SSHException('Server kex "f" is out of range') + sig = m.get_binary() + K = pow(self.f, self.x, self.P) + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || e || f || K) + hm = Message() + hm.add( + self.transport.local_version, + self.transport.remote_version, + self.transport.local_kex_init, + self.transport.remote_kex_init, + ) + hm.add_string(host_key) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + self.transport._set_K_H(K, self.hash_algo(hm.asbytes()).digest()) + self.transport._verify_key(host_key, sig) + self.transport._activate_outbound() + + def _parse_kexdh_init(self, m): + # server mode + self.e = m.get_mpint() + if (self.e < 1) or (self.e > self.P - 1): + raise SSHException('Client kex "e" is out of range') + K = pow(self.e, self.x, self.P) + key = self.transport.get_server_key().asbytes() + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || e || f || K) + hm = Message() + hm.add( + self.transport.remote_version, + self.transport.local_version, + self.transport.remote_kex_init, + self.transport.local_kex_init, + ) + hm.add_string(key) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = self.hash_algo(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + # sign it + sig = self.transport.get_server_key().sign_ssh_data( + H, self.transport.host_key_type + ) + # send reply + m = Message() + m.add_byte(c_MSG_KEXDH_REPLY) + m.add_string(key) + m.add_mpint(self.f) + m.add_string(sig) + self.transport._send_message(m) + self.transport._activate_outbound() diff --git a/lib/paramiko/kex_group14.py b/lib/paramiko/kex_group14.py new file mode 100644 index 0000000..8dee551 --- /dev/null +++ b/lib/paramiko/kex_group14.py @@ -0,0 +1,40 @@ +# Copyright (C) 2013 Torsten Landschoff +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Standard SSH key exchange ("kex" if you wanna sound cool). Diffie-Hellman of +2048 bit key halves, using a known "p" prime and "g" generator. +""" + +from paramiko.kex_group1 import KexGroup1 +from hashlib import sha1, sha256 + + +class KexGroup14(KexGroup1): + + # http://tools.ietf.org/html/rfc3526#section-3 + P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF # noqa + G = 2 + + name = "diffie-hellman-group14-sha1" + hash_algo = sha1 + + +class KexGroup14SHA256(KexGroup14): + name = "diffie-hellman-group14-sha256" + hash_algo = sha256 diff --git a/lib/paramiko/kex_group16.py b/lib/paramiko/kex_group16.py new file mode 100644 index 0000000..c675f87 --- /dev/null +++ b/lib/paramiko/kex_group16.py @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Edgar Sousa +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Standard SSH key exchange ("kex" if you wanna sound cool). Diffie-Hellman of +4096 bit key halves, using a known "p" prime and "g" generator. +""" + +from paramiko.kex_group1 import KexGroup1 +from hashlib import sha512 + + +class KexGroup16SHA512(KexGroup1): + name = "diffie-hellman-group16-sha512" + # http://tools.ietf.org/html/rfc3526#section-5 + P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6BF12FFA06D98A0864D87602733EC86A64521F2B18177B200CBBE117577A615D6C770988C0BAD946E208E24FA074E5AB3143DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D788719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA993B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934063199FFFFFFFFFFFFFFFF # noqa + G = 2 + + name = "diffie-hellman-group16-sha512" + hash_algo = sha512 diff --git a/lib/paramiko/kex_gss.py b/lib/paramiko/kex_gss.py new file mode 100644 index 0000000..2a5f29e --- /dev/null +++ b/lib/paramiko/kex_gss.py @@ -0,0 +1,686 @@ +# Copyright (C) 2003-2007 Robey Pointer +# Copyright (C) 2013-2014 science + computing ag +# Author: Sebastian Deiss +# +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +""" +This module provides GSS-API / SSPI Key Exchange as defined in :rfc:`4462`. + +.. note:: Credential delegation is not supported in server mode. + +.. note:: + `RFC 4462 Section 2.2 + `_ says we are not + required to implement GSS-API error messages. Thus, in many methods within + this module, if an error occurs an exception will be thrown and the + connection will be terminated. + +.. seealso:: :doc:`/api/ssh_gss` + +.. versionadded:: 1.15 +""" + +import os +from hashlib import sha1 + +from paramiko.common import ( + DEBUG, + max_byte, + zero_byte, + byte_chr, + byte_mask, + byte_ord, +) +from paramiko import util +from paramiko.message import Message +from paramiko.ssh_exception import SSHException + + +( + MSG_KEXGSS_INIT, + MSG_KEXGSS_CONTINUE, + MSG_KEXGSS_COMPLETE, + MSG_KEXGSS_HOSTKEY, + MSG_KEXGSS_ERROR, +) = range(30, 35) +(MSG_KEXGSS_GROUPREQ, MSG_KEXGSS_GROUP) = range(40, 42) +( + c_MSG_KEXGSS_INIT, + c_MSG_KEXGSS_CONTINUE, + c_MSG_KEXGSS_COMPLETE, + c_MSG_KEXGSS_HOSTKEY, + c_MSG_KEXGSS_ERROR, +) = [byte_chr(c) for c in range(30, 35)] +(c_MSG_KEXGSS_GROUPREQ, c_MSG_KEXGSS_GROUP) = [ + byte_chr(c) for c in range(40, 42) +] + + +class KexGSSGroup1: + """ + GSS-API / SSPI Authenticated Diffie-Hellman Key Exchange as defined in `RFC + 4462 Section 2 `_ + """ + + # draft-ietf-secsh-transport-09.txt, page 17 + P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFF # noqa + G = 2 + b7fffffffffffffff = byte_chr(0x7F) + max_byte * 7 # noqa + b0000000000000000 = zero_byte * 8 # noqa + NAME = "gss-group1-sha1-toWM5Slw5Ew8Mqkay+al2g==" + + def __init__(self, transport): + self.transport = transport + self.kexgss = self.transport.kexgss_ctxt + self.gss_host = None + self.x = 0 + self.e = 0 + self.f = 0 + + def start_kex(self): + """ + Start the GSS-API / SSPI Authenticated Diffie-Hellman Key Exchange. + """ + self._generate_x() + if self.transport.server_mode: + # compute f = g^x mod p, but don't send it yet + self.f = pow(self.G, self.x, self.P) + self.transport._expect_packet(MSG_KEXGSS_INIT) + return + # compute e = g^x mod p (where g=2), and send it + self.e = pow(self.G, self.x, self.P) + # Initialize GSS-API Key Exchange + self.gss_host = self.transport.gss_host + m = Message() + m.add_byte(c_MSG_KEXGSS_INIT) + m.add_string(self.kexgss.ssh_init_sec_context(target=self.gss_host)) + m.add_mpint(self.e) + self.transport._send_message(m) + self.transport._expect_packet( + MSG_KEXGSS_HOSTKEY, + MSG_KEXGSS_CONTINUE, + MSG_KEXGSS_COMPLETE, + MSG_KEXGSS_ERROR, + ) + + def parse_next(self, ptype, m): + """ + Parse the next packet. + + :param ptype: The (string) type of the incoming packet + :param `.Message` m: The packet content + """ + if self.transport.server_mode and (ptype == MSG_KEXGSS_INIT): + return self._parse_kexgss_init(m) + elif not self.transport.server_mode and (ptype == MSG_KEXGSS_HOSTKEY): + return self._parse_kexgss_hostkey(m) + elif self.transport.server_mode and (ptype == MSG_KEXGSS_CONTINUE): + return self._parse_kexgss_continue(m) + elif not self.transport.server_mode and (ptype == MSG_KEXGSS_COMPLETE): + return self._parse_kexgss_complete(m) + elif ptype == MSG_KEXGSS_ERROR: + return self._parse_kexgss_error(m) + msg = "GSS KexGroup1 asked to handle packet type {:d}" + raise SSHException(msg.format(ptype)) + + # ## internals... + + def _generate_x(self): + """ + generate an "x" (1 < x < q), where q is (p-1)/2. + p is a 128-byte (1024-bit) number, where the first 64 bits are 1. + therefore q can be approximated as a 2^1023. we drop the subset of + potential x where the first 63 bits are 1, because some of those will + be larger than q (but this is a tiny tiny subset of potential x). + """ + while 1: + x_bytes = os.urandom(128) + x_bytes = byte_mask(x_bytes[0], 0x7F) + x_bytes[1:] + first = x_bytes[:8] + if first not in (self.b7fffffffffffffff, self.b0000000000000000): + break + self.x = util.inflate_long(x_bytes) + + def _parse_kexgss_hostkey(self, m): + """ + Parse the SSH2_MSG_KEXGSS_HOSTKEY message (client mode). + + :param `.Message` m: The content of the SSH2_MSG_KEXGSS_HOSTKEY message + """ + # client mode + host_key = m.get_string() + self.transport.host_key = host_key + sig = m.get_string() + self.transport._verify_key(host_key, sig) + self.transport._expect_packet(MSG_KEXGSS_CONTINUE, MSG_KEXGSS_COMPLETE) + + def _parse_kexgss_continue(self, m): + """ + Parse the SSH2_MSG_KEXGSS_CONTINUE message. + + :param `.Message` m: The content of the SSH2_MSG_KEXGSS_CONTINUE + message + """ + if not self.transport.server_mode: + srv_token = m.get_string() + m = Message() + m.add_byte(c_MSG_KEXGSS_CONTINUE) + m.add_string( + self.kexgss.ssh_init_sec_context( + target=self.gss_host, recv_token=srv_token + ) + ) + self.transport.send_message(m) + self.transport._expect_packet( + MSG_KEXGSS_CONTINUE, MSG_KEXGSS_COMPLETE, MSG_KEXGSS_ERROR + ) + else: + pass + + def _parse_kexgss_complete(self, m): + """ + Parse the SSH2_MSG_KEXGSS_COMPLETE message (client mode). + + :param `.Message` m: The content of the + SSH2_MSG_KEXGSS_COMPLETE message + """ + # client mode + if self.transport.host_key is None: + self.transport.host_key = NullHostKey() + self.f = m.get_mpint() + if (self.f < 1) or (self.f > self.P - 1): + raise SSHException('Server kex "f" is out of range') + mic_token = m.get_string() + # This must be TRUE, if there is a GSS-API token in this message. + bool = m.get_boolean() + srv_token = None + if bool: + srv_token = m.get_string() + K = pow(self.f, self.x, self.P) + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || e || f || K) + hm = Message() + hm.add( + self.transport.local_version, + self.transport.remote_version, + self.transport.local_kex_init, + self.transport.remote_kex_init, + ) + hm.add_string(self.transport.host_key.__str__()) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = sha1(str(hm)).digest() + self.transport._set_K_H(K, H) + if srv_token is not None: + self.kexgss.ssh_init_sec_context( + target=self.gss_host, recv_token=srv_token + ) + self.kexgss.ssh_check_mic(mic_token, H) + else: + self.kexgss.ssh_check_mic(mic_token, H) + self.transport.gss_kex_used = True + self.transport._activate_outbound() + + def _parse_kexgss_init(self, m): + """ + Parse the SSH2_MSG_KEXGSS_INIT message (server mode). + + :param `.Message` m: The content of the SSH2_MSG_KEXGSS_INIT message + """ + # server mode + client_token = m.get_string() + self.e = m.get_mpint() + if (self.e < 1) or (self.e > self.P - 1): + raise SSHException('Client kex "e" is out of range') + K = pow(self.e, self.x, self.P) + self.transport.host_key = NullHostKey() + key = self.transport.host_key.__str__() + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || e || f || K) + hm = Message() + hm.add( + self.transport.remote_version, + self.transport.local_version, + self.transport.remote_kex_init, + self.transport.local_kex_init, + ) + hm.add_string(key) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = sha1(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + srv_token = self.kexgss.ssh_accept_sec_context( + self.gss_host, client_token + ) + m = Message() + if self.kexgss._gss_srv_ctxt_status: + mic_token = self.kexgss.ssh_get_mic( + self.transport.session_id, gss_kex=True + ) + m.add_byte(c_MSG_KEXGSS_COMPLETE) + m.add_mpint(self.f) + m.add_string(mic_token) + if srv_token is not None: + m.add_boolean(True) + m.add_string(srv_token) + else: + m.add_boolean(False) + self.transport._send_message(m) + self.transport.gss_kex_used = True + self.transport._activate_outbound() + else: + m.add_byte(c_MSG_KEXGSS_CONTINUE) + m.add_string(srv_token) + self.transport._send_message(m) + self.transport._expect_packet( + MSG_KEXGSS_CONTINUE, MSG_KEXGSS_COMPLETE, MSG_KEXGSS_ERROR + ) + + def _parse_kexgss_error(self, m): + """ + Parse the SSH2_MSG_KEXGSS_ERROR message (client mode). + The server may send a GSS-API error message. if it does, we display + the error by throwing an exception (client mode). + + :param `.Message` m: The content of the SSH2_MSG_KEXGSS_ERROR message + :raise SSHException: Contains GSS-API major and minor status as well as + the error message and the language tag of the + message + """ + maj_status = m.get_int() + min_status = m.get_int() + err_msg = m.get_string() + m.get_string() # we don't care about the language! + raise SSHException( + """GSS-API Error: +Major Status: {} +Minor Status: {} +Error Message: {} +""".format( + maj_status, min_status, err_msg + ) + ) + + +class KexGSSGroup14(KexGSSGroup1): + """ + GSS-API / SSPI Authenticated Diffie-Hellman Group14 Key Exchange as defined + in `RFC 4462 Section 2 + `_ + """ + + P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF # noqa + G = 2 + NAME = "gss-group14-sha1-toWM5Slw5Ew8Mqkay+al2g==" + + +class KexGSSGex: + """ + GSS-API / SSPI Authenticated Diffie-Hellman Group Exchange as defined in + `RFC 4462 Section 2 `_ + """ + + NAME = "gss-gex-sha1-toWM5Slw5Ew8Mqkay+al2g==" + min_bits = 1024 + max_bits = 8192 + preferred_bits = 2048 + + def __init__(self, transport): + self.transport = transport + self.kexgss = self.transport.kexgss_ctxt + self.gss_host = None + self.p = None + self.q = None + self.g = None + self.x = None + self.e = None + self.f = None + self.old_style = False + + def start_kex(self): + """ + Start the GSS-API / SSPI Authenticated Diffie-Hellman Group Exchange + """ + if self.transport.server_mode: + self.transport._expect_packet(MSG_KEXGSS_GROUPREQ) + return + # request a bit range: we accept (min_bits) to (max_bits), but prefer + # (preferred_bits). according to the spec, we shouldn't pull the + # minimum up above 1024. + self.gss_host = self.transport.gss_host + m = Message() + m.add_byte(c_MSG_KEXGSS_GROUPREQ) + m.add_int(self.min_bits) + m.add_int(self.preferred_bits) + m.add_int(self.max_bits) + self.transport._send_message(m) + self.transport._expect_packet(MSG_KEXGSS_GROUP) + + def parse_next(self, ptype, m): + """ + Parse the next packet. + + :param ptype: The (string) type of the incoming packet + :param `.Message` m: The packet content + """ + if ptype == MSG_KEXGSS_GROUPREQ: + return self._parse_kexgss_groupreq(m) + elif ptype == MSG_KEXGSS_GROUP: + return self._parse_kexgss_group(m) + elif ptype == MSG_KEXGSS_INIT: + return self._parse_kexgss_gex_init(m) + elif ptype == MSG_KEXGSS_HOSTKEY: + return self._parse_kexgss_hostkey(m) + elif ptype == MSG_KEXGSS_CONTINUE: + return self._parse_kexgss_continue(m) + elif ptype == MSG_KEXGSS_COMPLETE: + return self._parse_kexgss_complete(m) + elif ptype == MSG_KEXGSS_ERROR: + return self._parse_kexgss_error(m) + msg = "KexGex asked to handle packet type {:d}" + raise SSHException(msg.format(ptype)) + + # ## internals... + + def _generate_x(self): + # generate an "x" (1 < x < (p-1)/2). + q = (self.p - 1) // 2 + qnorm = util.deflate_long(q, 0) + qhbyte = byte_ord(qnorm[0]) + byte_count = len(qnorm) + qmask = 0xFF + while not (qhbyte & 0x80): + qhbyte <<= 1 + qmask >>= 1 + while True: + x_bytes = os.urandom(byte_count) + x_bytes = byte_mask(x_bytes[0], qmask) + x_bytes[1:] + x = util.inflate_long(x_bytes, 1) + if (x > 1) and (x < q): + break + self.x = x + + def _parse_kexgss_groupreq(self, m): + """ + Parse the SSH2_MSG_KEXGSS_GROUPREQ message (server mode). + + :param `.Message` m: The content of the + SSH2_MSG_KEXGSS_GROUPREQ message + """ + minbits = m.get_int() + preferredbits = m.get_int() + maxbits = m.get_int() + # smoosh the user's preferred size into our own limits + if preferredbits > self.max_bits: + preferredbits = self.max_bits + if preferredbits < self.min_bits: + preferredbits = self.min_bits + # fix min/max if they're inconsistent. technically, we could just pout + # and hang up, but there's no harm in giving them the benefit of the + # doubt and just picking a bitsize for them. + if minbits > preferredbits: + minbits = preferredbits + if maxbits < preferredbits: + maxbits = preferredbits + # now save a copy + self.min_bits = minbits + self.preferred_bits = preferredbits + self.max_bits = maxbits + # generate prime + pack = self.transport._get_modulus_pack() + if pack is None: + raise SSHException("Can't do server-side gex with no modulus pack") + self.transport._log( + DEBUG, # noqa + "Picking p ({} <= {} <= {} bits)".format( + minbits, preferredbits, maxbits + ), + ) + self.g, self.p = pack.get_modulus(minbits, preferredbits, maxbits) + m = Message() + m.add_byte(c_MSG_KEXGSS_GROUP) + m.add_mpint(self.p) + m.add_mpint(self.g) + self.transport._send_message(m) + self.transport._expect_packet(MSG_KEXGSS_INIT) + + def _parse_kexgss_group(self, m): + """ + Parse the SSH2_MSG_KEXGSS_GROUP message (client mode). + + :param `Message` m: The content of the SSH2_MSG_KEXGSS_GROUP message + """ + self.p = m.get_mpint() + self.g = m.get_mpint() + # reject if p's bit length < 1024 or > 8192 + bitlen = util.bit_length(self.p) + if (bitlen < 1024) or (bitlen > 8192): + raise SSHException( + "Server-generated gex p (don't ask) is out of range " + "({} bits)".format(bitlen) + ) + self.transport._log( + DEBUG, "Got server p ({} bits)".format(bitlen) + ) # noqa + self._generate_x() + # now compute e = g^x mod p + self.e = pow(self.g, self.x, self.p) + m = Message() + m.add_byte(c_MSG_KEXGSS_INIT) + m.add_string(self.kexgss.ssh_init_sec_context(target=self.gss_host)) + m.add_mpint(self.e) + self.transport._send_message(m) + self.transport._expect_packet( + MSG_KEXGSS_HOSTKEY, + MSG_KEXGSS_CONTINUE, + MSG_KEXGSS_COMPLETE, + MSG_KEXGSS_ERROR, + ) + + def _parse_kexgss_gex_init(self, m): + """ + Parse the SSH2_MSG_KEXGSS_INIT message (server mode). + + :param `Message` m: The content of the SSH2_MSG_KEXGSS_INIT message + """ + client_token = m.get_string() + self.e = m.get_mpint() + if (self.e < 1) or (self.e > self.p - 1): + raise SSHException('Client kex "e" is out of range') + self._generate_x() + self.f = pow(self.g, self.x, self.p) + K = pow(self.e, self.x, self.p) + self.transport.host_key = NullHostKey() + key = self.transport.host_key.__str__() + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) # noqa + hm = Message() + hm.add( + self.transport.remote_version, + self.transport.local_version, + self.transport.remote_kex_init, + self.transport.local_kex_init, + key, + ) + hm.add_int(self.min_bits) + hm.add_int(self.preferred_bits) + hm.add_int(self.max_bits) + hm.add_mpint(self.p) + hm.add_mpint(self.g) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = sha1(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + srv_token = self.kexgss.ssh_accept_sec_context( + self.gss_host, client_token + ) + m = Message() + if self.kexgss._gss_srv_ctxt_status: + mic_token = self.kexgss.ssh_get_mic( + self.transport.session_id, gss_kex=True + ) + m.add_byte(c_MSG_KEXGSS_COMPLETE) + m.add_mpint(self.f) + m.add_string(mic_token) + if srv_token is not None: + m.add_boolean(True) + m.add_string(srv_token) + else: + m.add_boolean(False) + self.transport._send_message(m) + self.transport.gss_kex_used = True + self.transport._activate_outbound() + else: + m.add_byte(c_MSG_KEXGSS_CONTINUE) + m.add_string(srv_token) + self.transport._send_message(m) + self.transport._expect_packet( + MSG_KEXGSS_CONTINUE, MSG_KEXGSS_COMPLETE, MSG_KEXGSS_ERROR + ) + + def _parse_kexgss_hostkey(self, m): + """ + Parse the SSH2_MSG_KEXGSS_HOSTKEY message (client mode). + + :param `Message` m: The content of the SSH2_MSG_KEXGSS_HOSTKEY message + """ + # client mode + host_key = m.get_string() + self.transport.host_key = host_key + sig = m.get_string() + self.transport._verify_key(host_key, sig) + self.transport._expect_packet(MSG_KEXGSS_CONTINUE, MSG_KEXGSS_COMPLETE) + + def _parse_kexgss_continue(self, m): + """ + Parse the SSH2_MSG_KEXGSS_CONTINUE message. + + :param `Message` m: The content of the SSH2_MSG_KEXGSS_CONTINUE message + """ + if not self.transport.server_mode: + srv_token = m.get_string() + m = Message() + m.add_byte(c_MSG_KEXGSS_CONTINUE) + m.add_string( + self.kexgss.ssh_init_sec_context( + target=self.gss_host, recv_token=srv_token + ) + ) + self.transport.send_message(m) + self.transport._expect_packet( + MSG_KEXGSS_CONTINUE, MSG_KEXGSS_COMPLETE, MSG_KEXGSS_ERROR + ) + else: + pass + + def _parse_kexgss_complete(self, m): + """ + Parse the SSH2_MSG_KEXGSS_COMPLETE message (client mode). + + :param `Message` m: The content of the SSH2_MSG_KEXGSS_COMPLETE message + """ + if self.transport.host_key is None: + self.transport.host_key = NullHostKey() + self.f = m.get_mpint() + mic_token = m.get_string() + # This must be TRUE, if there is a GSS-API token in this message. + bool = m.get_boolean() + srv_token = None + if bool: + srv_token = m.get_string() + if (self.f < 1) or (self.f > self.p - 1): + raise SSHException('Server kex "f" is out of range') + K = pow(self.f, self.x, self.p) + # okay, build up the hash H of + # (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) # noqa + hm = Message() + hm.add( + self.transport.local_version, + self.transport.remote_version, + self.transport.local_kex_init, + self.transport.remote_kex_init, + self.transport.host_key.__str__(), + ) + if not self.old_style: + hm.add_int(self.min_bits) + hm.add_int(self.preferred_bits) + if not self.old_style: + hm.add_int(self.max_bits) + hm.add_mpint(self.p) + hm.add_mpint(self.g) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = sha1(hm.asbytes()).digest() + self.transport._set_K_H(K, H) + if srv_token is not None: + self.kexgss.ssh_init_sec_context( + target=self.gss_host, recv_token=srv_token + ) + self.kexgss.ssh_check_mic(mic_token, H) + else: + self.kexgss.ssh_check_mic(mic_token, H) + self.transport.gss_kex_used = True + self.transport._activate_outbound() + + def _parse_kexgss_error(self, m): + """ + Parse the SSH2_MSG_KEXGSS_ERROR message (client mode). + The server may send a GSS-API error message. if it does, we display + the error by throwing an exception (client mode). + + :param `Message` m: The content of the SSH2_MSG_KEXGSS_ERROR message + :raise SSHException: Contains GSS-API major and minor status as well as + the error message and the language tag of the + message + """ + maj_status = m.get_int() + min_status = m.get_int() + err_msg = m.get_string() + m.get_string() # we don't care about the language (lang_tag)! + raise SSHException( + """GSS-API Error: +Major Status: {} +Minor Status: {} +Error Message: {} +""".format( + maj_status, min_status, err_msg + ) + ) + + +class NullHostKey: + """ + This class represents the Null Host Key for GSS-API Key Exchange as defined + in `RFC 4462 Section 5 + `_ + """ + + def __init__(self): + self.key = "" + + def __str__(self): + return self.key + + def get_name(self): + return self.key diff --git a/lib/paramiko/message.py b/lib/paramiko/message.py new file mode 100644 index 0000000..8c2b3bd --- /dev/null +++ b/lib/paramiko/message.py @@ -0,0 +1,318 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Implementation of an SSH2 "message". +""" + +import struct +from io import BytesIO + +from paramiko import util +from paramiko.common import zero_byte, max_byte, one_byte +from paramiko.util import u + + +class Message: + """ + An SSH2 message is a stream of bytes that encodes some combination of + strings, integers, bools, and infinite-precision integers. This class + builds or breaks down such a byte stream. + + Normally you don't need to deal with anything this low-level, but it's + exposed for people implementing custom extensions, or features that + paramiko doesn't support yet. + """ + + big_int = 0xFF000000 + + def __init__(self, content=None): + """ + Create a new SSH2 message. + + :param bytes content: + the byte stream to use as the message content (passed in only when + decomposing a message). + """ + if content is not None: + self.packet = BytesIO(content) + else: + self.packet = BytesIO() + + def __bytes__(self): + return self.asbytes() + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + """ + return "paramiko.Message(" + repr(self.packet.getvalue()) + ")" + + # TODO 4.0: just merge into __bytes__ (everywhere) + def asbytes(self): + """ + Return the byte stream content of this Message, as a `bytes`. + """ + return self.packet.getvalue() + + def rewind(self): + """ + Rewind the message to the beginning as if no items had been parsed + out of it yet. + """ + self.packet.seek(0) + + def get_remainder(self): + """ + Return the `bytes` of this message that haven't already been parsed and + returned. + """ + position = self.packet.tell() + remainder = self.packet.read() + self.packet.seek(position) + return remainder + + def get_so_far(self): + """ + Returns the `bytes` of this message that have been parsed and + returned. The string passed into a message's constructor can be + regenerated by concatenating ``get_so_far`` and `get_remainder`. + """ + position = self.packet.tell() + self.rewind() + return self.packet.read(position) + + def get_bytes(self, n): + """ + Return the next ``n`` bytes of the message, without decomposing into an + int, decoded string, etc. Just the raw bytes are returned. Returns a + string of ``n`` zero bytes if there weren't ``n`` bytes remaining in + the message. + """ + b = self.packet.read(n) + max_pad_size = 1 << 20 # Limit padding to 1 MB + if len(b) < n < max_pad_size: + return b + zero_byte * (n - len(b)) + return b + + def get_byte(self): + """ + Return the next byte of the message, without decomposing it. This + is equivalent to `get_bytes(1) `. + + :return: + the next (`bytes`) byte of the message, or ``b'\000'`` if there + aren't any bytes remaining. + """ + return self.get_bytes(1) + + def get_boolean(self): + """ + Fetch a boolean from the stream. + """ + b = self.get_bytes(1) + return b != zero_byte + + def get_adaptive_int(self): + """ + Fetch an int from the stream. + + :return: a 32-bit unsigned `int`. + """ + byte = self.get_bytes(1) + if byte == max_byte: + return util.inflate_long(self.get_binary()) + byte += self.get_bytes(3) + return struct.unpack(">I", byte)[0] + + def get_int(self): + """ + Fetch an int from the stream. + """ + return struct.unpack(">I", self.get_bytes(4))[0] + + def get_int64(self): + """ + Fetch a 64-bit int from the stream. + + :return: a 64-bit unsigned integer (`int`). + """ + return struct.unpack(">Q", self.get_bytes(8))[0] + + def get_mpint(self): + """ + Fetch a long int (mpint) from the stream. + + :return: an arbitrary-length integer (`int`). + """ + return util.inflate_long(self.get_binary()) + + # TODO 4.0: depending on where this is used internally or downstream, force + # users to specify get_binary instead and delete this. + def get_string(self): + """ + Fetch a "string" from the stream. This will actually be a `bytes` + object, and may contain unprintable characters. (It's not unheard of + for a string to contain another byte-stream message.) + """ + return self.get_bytes(self.get_int()) + + # TODO 4.0: also consider having this take over the get_string name, and + # remove this name instead. + def get_text(self): + """ + Fetch a Unicode string from the stream. + + This currently operates by attempting to encode the next "string" as + ``utf-8``. + """ + return u(self.get_string()) + + def get_binary(self): + """ + Alias for `get_string` (obtains a bytestring). + """ + return self.get_bytes(self.get_int()) + + def get_list(self): + """ + Fetch a list of `strings ` from the stream. + + These are trivially encoded as comma-separated values in a string. + """ + return self.get_text().split(",") + + def add_bytes(self, b): + """ + Write bytes to the stream, without any formatting. + + :param bytes b: bytes to add + """ + self.packet.write(b) + return self + + def add_byte(self, b): + """ + Write a single byte to the stream, without any formatting. + + :param bytes b: byte to add + """ + self.packet.write(b) + return self + + def add_boolean(self, b): + """ + Add a boolean value to the stream. + + :param bool b: boolean value to add + """ + if b: + self.packet.write(one_byte) + else: + self.packet.write(zero_byte) + return self + + def add_int(self, n): + """ + Add an integer to the stream. + + :param int n: integer to add + """ + self.packet.write(struct.pack(">I", n)) + return self + + def add_adaptive_int(self, n): + """ + Add an integer to the stream. + + :param int n: integer to add + """ + if n >= Message.big_int: + self.packet.write(max_byte) + self.add_string(util.deflate_long(n)) + else: + self.packet.write(struct.pack(">I", n)) + return self + + def add_int64(self, n): + """ + Add a 64-bit int to the stream. + + :param int n: long int to add + """ + self.packet.write(struct.pack(">Q", n)) + return self + + def add_mpint(self, z): + """ + Add a long int to the stream, encoded as an infinite-precision + integer. This method only works on positive numbers. + + :param int z: long int to add + """ + self.add_string(util.deflate_long(z)) + return self + + # TODO: see the TODO for get_string/get_text/et al, this should change + # to match. + def add_string(self, s): + """ + Add a bytestring to the stream. + + :param byte s: bytestring to add + """ + s = util.asbytes(s) + self.add_int(len(s)) + self.packet.write(s) + return self + + def add_list(self, l): # noqa: E741 + """ + Add a list of strings to the stream. They are encoded identically to + a single string of values separated by commas. (Yes, really, that's + how SSH2 does it.) + + :param l: list of strings to add + """ + self.add_string(",".join(l)) + return self + + def _add(self, i): + if type(i) is bool: + return self.add_boolean(i) + elif isinstance(i, int): + return self.add_adaptive_int(i) + elif type(i) is list: + return self.add_list(i) + else: + return self.add_string(i) + + # TODO: this would never have worked for unicode strings under Python 3, + # guessing nobody/nothing ever used it for that purpose? + def add(self, *seq): + """ + Add a sequence of items to the stream. The values are encoded based + on their type: bytes, str, int, bool, or list. + + .. warning:: + Longs are encoded non-deterministically. Don't use this method. + + :param seq: the sequence of items + """ + for item in seq: + self._add(item) diff --git a/lib/paramiko/packet.py b/lib/paramiko/packet.py new file mode 100644 index 0000000..f1de4b0 --- /dev/null +++ b/lib/paramiko/packet.py @@ -0,0 +1,696 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Packet handling +""" + +import errno +import os +import socket +import struct +import threading +import time +from hmac import HMAC + +from paramiko import util +from paramiko.common import ( + linefeed_byte, + cr_byte_value, + MSG_NAMES, + DEBUG, + xffffffff, + zero_byte, + byte_ord, +) +from paramiko.util import u +from paramiko.ssh_exception import SSHException, ProxyCommandFailure +from paramiko.message import Message + + +def compute_hmac(key, message, digest_class): + return HMAC(key, message, digest_class).digest() + + +class NeedRekeyException(Exception): + """ + Exception indicating a rekey is needed. + """ + + pass + + +def first_arg(e): + arg = None + if type(e.args) is tuple and len(e.args) > 0: + arg = e.args[0] + return arg + + +class Packetizer: + """ + Implementation of the base SSH packet protocol. + """ + + # READ the secsh RFC's before raising these values. if anything, + # they should probably be lower. + REKEY_PACKETS = pow(2, 29) + REKEY_BYTES = pow(2, 29) + + # Allow receiving this many packets after a re-key request before + # terminating + REKEY_PACKETS_OVERFLOW_MAX = pow(2, 29) + # Allow receiving this many bytes after a re-key request before terminating + REKEY_BYTES_OVERFLOW_MAX = pow(2, 29) + + def __init__(self, socket): + self.__socket = socket + self.__logger = None + self.__closed = False + self.__dump_packets = False + self.__need_rekey = False + self.__init_count = 0 + self.__remainder = bytes() + self._initial_kex_done = False + + # used for noticing when to re-key: + self.__sent_bytes = 0 + self.__sent_packets = 0 + self.__received_bytes = 0 + self.__received_packets = 0 + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + + # current inbound/outbound ciphering: + self.__block_size_out = 8 + self.__block_size_in = 8 + self.__mac_size_out = 0 + self.__mac_size_in = 0 + self.__block_engine_out = None + self.__block_engine_in = None + self.__sdctr_out = False + self.__mac_engine_out = None + self.__mac_engine_in = None + self.__mac_key_out = bytes() + self.__mac_key_in = bytes() + self.__compress_engine_out = None + self.__compress_engine_in = None + self.__sequence_number_out = 0 + self.__sequence_number_in = 0 + self.__etm_out = False + self.__etm_in = False + + # AEAD (eg aes128-gcm/aes256-gcm) cipher use + self.__aead_out = False + self.__aead_in = False + self.__iv_out = None + self.__iv_in = None + + # lock around outbound writes (packet computation) + self.__write_lock = threading.RLock() + + # keepalives: + self.__keepalive_interval = 0 + self.__keepalive_last = time.time() + self.__keepalive_callback = None + + self.__timer = None + self.__handshake_complete = False + self.__timer_expired = False + + @property + def closed(self): + return self.__closed + + def reset_seqno_out(self): + self.__sequence_number_out = 0 + + def reset_seqno_in(self): + self.__sequence_number_in = 0 + + def set_log(self, log): + """ + Set the Python log object to use for logging. + """ + self.__logger = log + + def set_outbound_cipher( + self, + block_engine, + block_size, + mac_engine, + mac_size, + mac_key, + sdctr=False, + etm=False, + aead=False, + iv_out=None, + ): + """ + Switch outbound data cipher. + :param etm: Set encrypt-then-mac from OpenSSH + """ + self.__block_engine_out = block_engine + self.__sdctr_out = sdctr + self.__block_size_out = block_size + self.__mac_engine_out = mac_engine + self.__mac_size_out = mac_size + self.__mac_key_out = mac_key + self.__sent_bytes = 0 + self.__sent_packets = 0 + self.__etm_out = etm + self.__aead_out = aead + self.__iv_out = iv_out + # wait until the reset happens in both directions before clearing + # rekey flag + self.__init_count |= 1 + if self.__init_count == 3: + self.__init_count = 0 + self.__need_rekey = False + + def set_inbound_cipher( + self, + block_engine, + block_size, + mac_engine, + mac_size, + mac_key, + etm=False, + aead=False, + iv_in=None, + ): + """ + Switch inbound data cipher. + :param etm: Set encrypt-then-mac from OpenSSH + """ + self.__block_engine_in = block_engine + self.__block_size_in = block_size + self.__mac_engine_in = mac_engine + self.__mac_size_in = mac_size + self.__mac_key_in = mac_key + self.__received_bytes = 0 + self.__received_packets = 0 + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + self.__etm_in = etm + self.__aead_in = aead + self.__iv_in = iv_in + # wait until the reset happens in both directions before clearing + # rekey flag + self.__init_count |= 2 + if self.__init_count == 3: + self.__init_count = 0 + self.__need_rekey = False + + def set_outbound_compressor(self, compressor): + self.__compress_engine_out = compressor + + def set_inbound_compressor(self, compressor): + self.__compress_engine_in = compressor + + def close(self): + self.__closed = True + self.__socket.close() + + def set_hexdump(self, hexdump): + self.__dump_packets = hexdump + + def get_hexdump(self): + return self.__dump_packets + + def get_mac_size_in(self): + return self.__mac_size_in + + def get_mac_size_out(self): + return self.__mac_size_out + + def need_rekey(self): + """ + Returns ``True`` if a new set of keys needs to be negotiated. This + will be triggered during a packet read or write, so it should be + checked after every read or write, or at least after every few. + """ + return self.__need_rekey + + def set_keepalive(self, interval, callback): + """ + Turn on/off the callback keepalive. If ``interval`` seconds pass with + no data read from or written to the socket, the callback will be + executed and the timer will be reset. + """ + self.__keepalive_interval = interval + self.__keepalive_callback = callback + self.__keepalive_last = time.time() + + def read_timer(self): + self.__timer_expired = True + + def start_handshake(self, timeout): + """ + Tells `Packetizer` that the handshake process started. + Starts a book keeping timer that can signal a timeout in the + handshake process. + + :param float timeout: amount of seconds to wait before timing out + """ + if not self.__timer: + self.__timer = threading.Timer(float(timeout), self.read_timer) + self.__timer.start() + + def handshake_timed_out(self): + """ + Checks if the handshake has timed out. + + If `start_handshake` wasn't called before the call to this function, + the return value will always be `False`. If the handshake completed + before a timeout was reached, the return value will be `False` + + :return: handshake time out status, as a `bool` + """ + if not self.__timer: + return False + if self.__handshake_complete: + return False + return self.__timer_expired + + def complete_handshake(self): + """ + Tells `Packetizer` that the handshake has completed. + """ + if self.__timer: + self.__timer.cancel() + self.__timer_expired = False + self.__handshake_complete = True + + def read_all(self, n, check_rekey=False): + """ + Read as close to N bytes as possible, blocking as long as necessary. + + :param int n: number of bytes to read + :return: the data read, as a `str` + + :raises: + ``EOFError`` -- if the socket was closed before all the bytes could + be read + """ + out = bytes() + # handle over-reading from reading the banner line + if len(self.__remainder) > 0: + out = self.__remainder[:n] + self.__remainder = self.__remainder[n:] + n -= len(out) + while n > 0: + got_timeout = False + if self.handshake_timed_out(): + raise EOFError() + try: + x = self.__socket.recv(n) + if len(x) == 0: + raise EOFError() + out += x + n -= len(x) + except socket.timeout: + got_timeout = True + except socket.error as e: + # on Linux, sometimes instead of socket.timeout, we get + # EAGAIN. this is a bug in recent (> 2.6.9) kernels but + # we need to work around it. + arg = first_arg(e) + if arg == errno.EAGAIN: + got_timeout = True + elif self.__closed: + raise EOFError() + else: + raise + if got_timeout: + if self.__closed: + raise EOFError() + if check_rekey and (len(out) == 0) and self.__need_rekey: + raise NeedRekeyException() + self._check_keepalive() + return out + + def write_all(self, out): + self.__keepalive_last = time.time() + iteration_with_zero_as_return_value = 0 + while len(out) > 0: + retry_write = False + try: + n = self.__socket.send(out) + except socket.timeout: + retry_write = True + except socket.error as e: + arg = first_arg(e) + if arg == errno.EAGAIN: + retry_write = True + else: + n = -1 + except ProxyCommandFailure: + raise # so it doesn't get swallowed by the below catchall + except Exception: + # could be: (32, 'Broken pipe') + n = -1 + if retry_write: + n = 0 + if self.__closed: + n = -1 + else: + if n == 0 and iteration_with_zero_as_return_value > 10: + # We shouldn't retry the write, but we didn't + # manage to send anything over the socket. This might be an + # indication that we have lost contact with the remote + # side, but are yet to receive an EOFError or other socket + # errors. Let's give it some iteration to try and catch up. + n = -1 + iteration_with_zero_as_return_value += 1 + if n < 0: + raise EOFError() + if n == len(out): + break + out = out[n:] + return + + def readline(self, timeout): + """ + Read a line from the socket. We assume no data is pending after the + line, so it's okay to attempt large reads. + """ + buf = self.__remainder + while linefeed_byte not in buf: + buf += self._read_timeout(timeout) + n = buf.index(linefeed_byte) + self.__remainder = buf[n + 1 :] + buf = buf[:n] + if (len(buf) > 0) and (buf[-1] == cr_byte_value): + buf = buf[:-1] + return u(buf) + + def _inc_iv_counter(self, iv): + # Per https://www.rfc-editor.org/rfc/rfc5647.html#section-7.1 , + # we increment the last 8 bytes of the 12-byte IV... + iv_counter_b = iv[4:] + iv_counter = int.from_bytes(iv_counter_b, "big") + inc_iv_counter = iv_counter + 1 + inc_iv_counter_b = inc_iv_counter.to_bytes(8, "big") + # ...then re-concatenate it with the static first 4 bytes + new_iv = iv[0:4] + inc_iv_counter_b + return new_iv + + def send_message(self, data): + """ + Write a block of data using the current cipher, as an SSH block. + """ + # encrypt this sucka + data = data.asbytes() + cmd = byte_ord(data[0]) + if cmd in MSG_NAMES: + cmd_name = MSG_NAMES[cmd] + else: + cmd_name = "${:x}".format(cmd) + orig_len = len(data) + self.__write_lock.acquire() + try: + if self.__compress_engine_out is not None: + data = self.__compress_engine_out(data) + packet = self._build_packet(data) + if self.__dump_packets: + self._log( + DEBUG, + "Write packet <{}>, length {}".format(cmd_name, orig_len), + ) + self._log(DEBUG, util.format_binary(packet, "OUT: ")) + if self.__block_engine_out is not None: + if self.__etm_out: + # packet length is not encrypted in EtM + out = packet[0:4] + self.__block_engine_out.update( + packet[4:] + ) + elif self.__aead_out: + # Packet-length field is used as the 'associated data' + # under AES-GCM, so like EtM, it's not encrypted. See + # https://www.rfc-editor.org/rfc/rfc5647#section-7.3 + out = packet[0:4] + self.__block_engine_out.encrypt( + self.__iv_out, packet[4:], packet[0:4] + ) + self.__iv_out = self._inc_iv_counter(self.__iv_out) + else: + out = self.__block_engine_out.update(packet) + else: + out = packet + # Append an MAC when needed (eg, not under AES-GCM) + if self.__block_engine_out is not None and not self.__aead_out: + packed = struct.pack(">I", self.__sequence_number_out) + payload = packed + (out if self.__etm_out else packet) + out += compute_hmac( + self.__mac_key_out, payload, self.__mac_engine_out + )[: self.__mac_size_out] + next_seq = (self.__sequence_number_out + 1) & xffffffff + if next_seq == 0 and not self._initial_kex_done: + raise SSHException( + "Sequence number rolled over during initial kex!" + ) + self.__sequence_number_out = next_seq + self.write_all(out) + + self.__sent_bytes += len(out) + self.__sent_packets += 1 + sent_too_much = ( + self.__sent_packets >= self.REKEY_PACKETS + or self.__sent_bytes >= self.REKEY_BYTES + ) + if sent_too_much and not self.__need_rekey: + # only ask once for rekeying + msg = "Rekeying (hit {} packets, {} bytes sent)" + self._log( + DEBUG, msg.format(self.__sent_packets, self.__sent_bytes) + ) + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + self._trigger_rekey() + finally: + self.__write_lock.release() + + def read_message(self): + """ + Only one thread should ever be in this function (no other locking is + done). + + :raises: `.SSHException` -- if the packet is mangled + :raises: `.NeedRekeyException` -- if the transport should rekey + """ + header = self.read_all(self.__block_size_in, check_rekey=True) + if self.__etm_in: + packet_size = struct.unpack(">I", header[:4])[0] + remaining = packet_size - self.__block_size_in + 4 + packet = header[4:] + self.read_all(remaining, check_rekey=False) + mac = self.read_all(self.__mac_size_in, check_rekey=False) + mac_payload = ( + struct.pack(">II", self.__sequence_number_in, packet_size) + + packet + ) + my_mac = compute_hmac( + self.__mac_key_in, mac_payload, self.__mac_engine_in + )[: self.__mac_size_in] + if not util.constant_time_bytes_eq(my_mac, mac): + raise SSHException("Mismatched MAC") + header = packet + + if self.__aead_in: + # Grab unencrypted (considered 'additional data' under GCM) packet + # length. + packet_size = struct.unpack(">I", header[:4])[0] + aad = header[:4] + remaining = ( + packet_size - self.__block_size_in + 4 + self.__mac_size_in + ) + packet = header[4:] + self.read_all(remaining, check_rekey=False) + header = self.__block_engine_in.decrypt(self.__iv_in, packet, aad) + + self.__iv_in = self._inc_iv_counter(self.__iv_in) + + if self.__block_engine_in is not None and not self.__aead_in: + header = self.__block_engine_in.update(header) + if self.__dump_packets: + self._log(DEBUG, util.format_binary(header, "IN: ")) + + # When ETM or AEAD (GCM) are in use, we've already read the packet size + # & decrypted everything, so just set the packet back to the header we + # obtained. + if self.__etm_in or self.__aead_in: + packet = header + # Otherwise, use the older non-ETM logic + else: + packet_size = struct.unpack(">I", header[:4])[0] + + # leftover contains decrypted bytes from the first block (after the + # length field) + leftover = header[4:] + if (packet_size - len(leftover)) % self.__block_size_in != 0: + raise SSHException("Invalid packet blocking") + buf = self.read_all( + packet_size + self.__mac_size_in - len(leftover) + ) + packet = buf[: packet_size - len(leftover)] + post_packet = buf[packet_size - len(leftover) :] + + if self.__block_engine_in is not None: + packet = self.__block_engine_in.update(packet) + packet = leftover + packet + + if self.__dump_packets: + self._log(DEBUG, util.format_binary(packet, "IN: ")) + + if self.__mac_size_in > 0 and not self.__etm_in and not self.__aead_in: + mac = post_packet[: self.__mac_size_in] + mac_payload = ( + struct.pack(">II", self.__sequence_number_in, packet_size) + + packet + ) + my_mac = compute_hmac( + self.__mac_key_in, mac_payload, self.__mac_engine_in + )[: self.__mac_size_in] + if not util.constant_time_bytes_eq(my_mac, mac): + raise SSHException("Mismatched MAC") + padding = byte_ord(packet[0]) + payload = packet[1 : packet_size - padding] + + if self.__dump_packets: + self._log( + DEBUG, + "Got payload ({} bytes, {} padding)".format( + packet_size, padding + ), + ) + + if self.__compress_engine_in is not None: + payload = self.__compress_engine_in(payload) + + msg = Message(payload[1:]) + msg.seqno = self.__sequence_number_in + next_seq = (self.__sequence_number_in + 1) & xffffffff + if next_seq == 0 and not self._initial_kex_done: + raise SSHException( + "Sequence number rolled over during initial kex!" + ) + self.__sequence_number_in = next_seq + + # check for rekey + raw_packet_size = packet_size + self.__mac_size_in + 4 + self.__received_bytes += raw_packet_size + self.__received_packets += 1 + if self.__need_rekey: + # we've asked to rekey -- give them some packets to comply before + # dropping the connection + self.__received_bytes_overflow += raw_packet_size + self.__received_packets_overflow += 1 + if ( + self.__received_packets_overflow + >= self.REKEY_PACKETS_OVERFLOW_MAX + ) or ( + self.__received_bytes_overflow >= self.REKEY_BYTES_OVERFLOW_MAX + ): + raise SSHException( + "Remote transport is ignoring rekey requests" + ) + elif (self.__received_packets >= self.REKEY_PACKETS) or ( + self.__received_bytes >= self.REKEY_BYTES + ): + # only ask once for rekeying + err = "Rekeying (hit {} packets, {} bytes received)" + self._log( + DEBUG, + err.format(self.__received_packets, self.__received_bytes), + ) + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + self._trigger_rekey() + + cmd = byte_ord(payload[0]) + if cmd in MSG_NAMES: + cmd_name = MSG_NAMES[cmd] + else: + cmd_name = "${:x}".format(cmd) + if self.__dump_packets: + self._log( + DEBUG, + "Read packet <{}>, length {}".format(cmd_name, len(payload)), + ) + return cmd, msg + + # ...protected... + + def _log(self, level, msg): + if self.__logger is None: + return + if issubclass(type(msg), list): + for m in msg: + self.__logger.log(level, m) + else: + self.__logger.log(level, msg) + + def _check_keepalive(self): + if ( + not self.__keepalive_interval + or not self.__block_engine_out + or self.__need_rekey + ): + # wait till we're encrypting, and not in the middle of rekeying + return + now = time.time() + if now > self.__keepalive_last + self.__keepalive_interval: + self.__keepalive_callback() + self.__keepalive_last = now + + def _read_timeout(self, timeout): + start = time.time() + while True: + try: + x = self.__socket.recv(128) + if len(x) == 0: + raise EOFError() + break + except socket.timeout: + pass + if self.__closed: + raise EOFError() + now = time.time() + if now - start >= timeout: + raise socket.timeout() + return x + + def _build_packet(self, payload): + # pad up at least 4 bytes, to nearest block-size (usually 8) + bsize = self.__block_size_out + # do not include payload length in computations for padding in EtM mode + # (payload length won't be encrypted) + addlen = 4 if self.__etm_out or self.__aead_out else 8 + padding = 3 + bsize - ((len(payload) + addlen) % bsize) + packet = struct.pack(">IB", len(payload) + padding + 1, padding) + packet += payload + if self.__sdctr_out or self.__block_engine_out is None: + # cute trick i caught openssh doing: if we're not encrypting or + # SDCTR mode (RFC4344), + # don't waste random bytes for the padding + packet += zero_byte * padding + else: + packet += os.urandom(padding) + return packet + + def _trigger_rekey(self): + # outside code should check for this flag + self.__need_rekey = True diff --git a/lib/paramiko/pipe.py b/lib/paramiko/pipe.py new file mode 100644 index 0000000..65944fa --- /dev/null +++ b/lib/paramiko/pipe.py @@ -0,0 +1,148 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Abstraction of a one-way pipe where the read end can be used in +`select.select`. Normally this is trivial, but Windows makes it nearly +impossible. + +The pipe acts like an Event, which can be set or cleared. When set, the pipe +will trigger as readable in `select `. +""" + +import sys +import os +import socket + + +def make_pipe(): + if sys.platform[:3] != "win": + p = PosixPipe() + else: + p = WindowsPipe() + return p + + +class PosixPipe: + def __init__(self): + self._rfd, self._wfd = os.pipe() + self._set = False + self._forever = False + self._closed = False + + def close(self): + os.close(self._rfd) + os.close(self._wfd) + # used for unit tests: + self._closed = True + + def fileno(self): + return self._rfd + + def clear(self): + if not self._set or self._forever: + return + os.read(self._rfd, 1) + self._set = False + + def set(self): + if self._set or self._closed: + return + self._set = True + os.write(self._wfd, b"*") + + def set_forever(self): + self._forever = True + self.set() + + +class WindowsPipe: + """ + On Windows, only an OS-level "WinSock" may be used in select(), but reads + and writes must be to the actual socket object. + """ + + def __init__(self): + serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + serv.bind(("127.0.0.1", 0)) + serv.listen(1) + + # need to save sockets in _rsock/_wsock so they don't get closed + self._rsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._rsock.connect(("127.0.0.1", serv.getsockname()[1])) + + self._wsock, addr = serv.accept() + serv.close() + self._set = False + self._forever = False + self._closed = False + + def close(self): + self._rsock.close() + self._wsock.close() + # used for unit tests: + self._closed = True + + def fileno(self): + return self._rsock.fileno() + + def clear(self): + if not self._set or self._forever: + return + self._rsock.recv(1) + self._set = False + + def set(self): + if self._set or self._closed: + return + self._set = True + self._wsock.send(b"*") + + def set_forever(self): + self._forever = True + self.set() + + +class OrPipe: + def __init__(self, pipe): + self._set = False + self._partner = None + self._pipe = pipe + + def set(self): + self._set = True + if not self._partner._set: + self._pipe.set() + + def clear(self): + self._set = False + if not self._partner._set: + self._pipe.clear() + + +def make_or_pipe(pipe): + """ + wraps a pipe into two pipe-like objects which are "or"d together to + affect the real pipe. if either returned pipe is set, the wrapped pipe + is set. when both are cleared, the wrapped pipe is cleared. + """ + p1 = OrPipe(pipe) + p2 = OrPipe(pipe) + p1._partner = p2 + p2._partner = p1 + return p1, p2 diff --git a/lib/paramiko/pkey.py b/lib/paramiko/pkey.py new file mode 100644 index 0000000..50558cb --- /dev/null +++ b/lib/paramiko/pkey.py @@ -0,0 +1,955 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Common API for all public keys. +""" + +import base64 +from base64 import encodebytes, decodebytes +from binascii import unhexlify +import os +from pathlib import Path +from hashlib import md5, sha256 +import re +import struct + +import bcrypt + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import padding, serialization +from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher +from cryptography.hazmat.primitives import asymmetric + +from paramiko import util +from paramiko.util import u, b +from paramiko.common import o600 +from paramiko.ssh_exception import SSHException, PasswordRequiredException +from paramiko.message import Message + + +# TripleDES is moving from `cryptography.hazmat.primitives.ciphers.algorithms` +# in cryptography>=43.0.0 to `cryptography.hazmat.decrepit.ciphers.algorithms` +# It will be removed from `cryptography.hazmat.primitives.ciphers.algorithms` +# in cryptography==48.0.0. +# +# Source References: +# - https://github.com/pyca/cryptography/commit/722a6393e61b3ac +# - https://github.com/pyca/cryptography/pull/11407/files +try: + from cryptography.hazmat.decrepit.ciphers.algorithms import TripleDES +except ImportError: + from cryptography.hazmat.primitives.ciphers.algorithms import TripleDES + + +OPENSSH_AUTH_MAGIC = b"openssh-key-v1\x00" + + +def _unpad_openssh(data): + # At the moment, this is only used for unpadding private keys on disk. This + # really ought to be made constant time (possibly by upstreaming this logic + # into pyca/cryptography). + padding_length = data[-1] + if 0x20 <= padding_length < 0x7F: + return data # no padding, last byte part comment (printable ascii) + if padding_length > 15: + raise SSHException("Invalid key") + for i in range(padding_length): + if data[i - padding_length] != i + 1: + raise SSHException("Invalid key") + return data[:-padding_length] + + +class UnknownKeyType(Exception): + """ + An unknown public/private key algorithm was attempted to be read. + """ + + def __init__(self, key_type=None, key_bytes=None): + self.key_type = key_type + self.key_bytes = key_bytes + + def __str__(self): + return f"UnknownKeyType(type={self.key_type!r}, bytes=<{len(self.key_bytes)}>)" # noqa + + +class PKey: + """ + Base class for public keys. + + Also includes some "meta" level convenience constructors such as + `.from_type_string`. + """ + + # known encryption types for private key files: + _CIPHER_TABLE = { + "AES-128-CBC": { + "cipher": algorithms.AES, + "keysize": 16, + "blocksize": 16, + "mode": modes.CBC, + }, + "AES-256-CBC": { + "cipher": algorithms.AES, + "keysize": 32, + "blocksize": 16, + "mode": modes.CBC, + }, + "DES-EDE3-CBC": { + "cipher": TripleDES, + "keysize": 24, + "blocksize": 8, + "mode": modes.CBC, + }, + } + _PRIVATE_KEY_FORMAT_ORIGINAL = 1 + _PRIVATE_KEY_FORMAT_OPENSSH = 2 + BEGIN_TAG = re.compile(r"^-{5}BEGIN (RSA|EC|OPENSSH) PRIVATE KEY-{5}\s*$") + END_TAG = re.compile(r"^-{5}END (RSA|EC|OPENSSH) PRIVATE KEY-{5}\s*$") + + @staticmethod + def from_path(path, passphrase=None): + """ + Attempt to instantiate appropriate key subclass from given file path. + + :param Path path: The path to load (may also be a `str`). + + :returns: + A `PKey` subclass instance. + + :raises: + `UnknownKeyType`, if our crypto backend doesn't know this key type. + + .. versionadded:: 3.2 + """ + # TODO: make sure sphinx is reading Path right in param list... + + # Lazy import to avoid circular import issues + from paramiko import RSAKey, Ed25519Key, ECDSAKey + + # Normalize to string, as cert suffix isn't quite an extension, so + # pathlib isn't useful for this. + path = str(path) + + # Sort out cert vs key, i.e. it is 'legal' to hand this kind of API + # /either/ the key /or/ the cert, when there is a key/cert pair. + cert_suffix = "-cert.pub" + if str(path).endswith(cert_suffix): + key_path = path[: -len(cert_suffix)] + cert_path = path + else: + key_path = path + cert_path = path + cert_suffix + + key_path = Path(key_path).expanduser() + cert_path = Path(cert_path).expanduser() + + data = key_path.read_bytes() + # Like OpenSSH, try modern/OpenSSH-specific key load first + try: + loaded = serialization.load_ssh_private_key( + data=data, password=passphrase + ) + # Then fall back to assuming legacy PEM type + except ValueError: + loaded = serialization.load_pem_private_key( + data=data, password=passphrase + ) + # TODO Python 3.10: match statement? (NOTE: we cannot use a dict + # because the results from the loader are literal backend, eg openssl, + # private classes, so isinstance tests work but exact 'x class is y' + # tests will not work) + # TODO: leverage already-parsed/math'd obj to avoid duplicate cpu + # cycles? seemingly requires most of our key subclasses to be rewritten + # to be cryptography-object-forward. this is still likely faster than + # the old SSHClient code that just tried instantiating every class! + key_class = None + if isinstance(loaded, asymmetric.rsa.RSAPrivateKey): + key_class = RSAKey + elif isinstance(loaded, asymmetric.ed25519.Ed25519PrivateKey): + key_class = Ed25519Key + elif isinstance(loaded, asymmetric.ec.EllipticCurvePrivateKey): + key_class = ECDSAKey + else: + raise UnknownKeyType(key_bytes=data, key_type=loaded.__class__) + with key_path.open() as fd: + key = key_class.from_private_key(fd, password=passphrase) + if cert_path.exists(): + # load_certificate can take Message, path-str, or value-str + key.load_certificate(str(cert_path)) + return key + + @staticmethod + def from_type_string(key_type, key_bytes): + """ + Given type `str` & raw `bytes`, return a `PKey` subclass instance. + + For example, ``PKey.from_type_string("ssh-ed25519", )`` + will (if successful) return a new `.Ed25519Key`. + + :param str key_type: + The key type, eg ``"ssh-ed25519"``. + :param bytes key_bytes: + The raw byte data forming the key material, as expected by + subclasses' ``data`` parameter. + + :returns: + A `PKey` subclass instance. + + :raises: + `UnknownKeyType`, if no registered classes knew about this type. + + .. versionadded:: 3.2 + """ + from paramiko import key_classes + + for key_class in key_classes: + if key_type in key_class.identifiers(): + # TODO: needs to passthru things like passphrase + return key_class(data=key_bytes) + raise UnknownKeyType(key_type=key_type, key_bytes=key_bytes) + + @classmethod + def identifiers(cls): + """ + returns an iterable of key format/name strings this class can handle. + + Most classes only have a single identifier, and thus this default + implementation suffices; see `.ECDSAKey` for one example of an + override. + """ + return [cls.name] + + # TODO 4.0: make this and subclasses consistent, some of our own + # classmethods even assume kwargs we don't define! + # TODO 4.0: prob also raise NotImplementedError instead of pass'ing; the + # contract is pretty obviously that you need to handle msg/data/filename + # appropriately. (If 'pass' is a concession to testing, see about doing the + # work to fix the tests instead) + def __init__(self, msg=None, data=None): + """ + Create a new instance of this public key type. If ``msg`` is given, + the key's public part(s) will be filled in from the message. If + ``data`` is given, the key's public part(s) will be filled in from + the string. + + :param .Message msg: + an optional SSH `.Message` containing a public key of this type. + :param bytes data: + optional, the bytes of a public key of this type + + :raises: `.SSHException` -- + if a key cannot be created from the ``data`` or ``msg`` given, or + no key was passed in. + """ + pass + + # TODO: arguably this might want to be __str__ instead? ehh + # TODO: ditto the interplay between showing class name (currently we just + # say PKey writ large) and algorithm (usually == class name, but not + # always, also sometimes shows certificate-ness) + # TODO: if we do change it, we also want to tweak eg AgentKey, as it + # currently displays agent-ness with a suffix + def __repr__(self): + comment = "" + # Works for AgentKey, may work for others? + if hasattr(self, "comment") and self.comment: + comment = f", comment={self.comment!r}" + return f"PKey(alg={self.algorithm_name}, bits={self.get_bits()}, fp={self.fingerprint}{comment})" # noqa + + # TODO 4.0: just merge into __bytes__ (everywhere) + def asbytes(self): + """ + Return a string of an SSH `.Message` made up of the public part(s) of + this key. This string is suitable for passing to `__init__` to + re-create the key object later. + """ + return bytes() + + def __bytes__(self): + return self.asbytes() + + def __eq__(self, other): + return isinstance(other, PKey) and self._fields == other._fields + + def __hash__(self): + return hash(self._fields) + + @property + def _fields(self): + raise NotImplementedError + + def get_name(self): + """ + Return the name of this private key implementation. + + :return: + name of this private key type, in SSH terminology, as a `str` (for + example, ``"ssh-rsa"``). + """ + return "" + + @property + def algorithm_name(self): + """ + Return the key algorithm identifier for this key. + + Similar to `get_name`, but aimed at pure algorithm name instead of SSH + protocol field value. + """ + # Nuke the leading 'ssh-' + # TODO in Python 3.9: use .removeprefix() + name = self.get_name().replace("ssh-", "") + # Trim any cert suffix (but leave the -cert, as OpenSSH does) + cert_tail = "-cert-v01@openssh.com" + if cert_tail in name: + name = name.replace(cert_tail, "-cert") + # Nuke any eg ECDSA suffix, OpenSSH does basically this too. + else: + name = name.split("-")[0] + return name.upper() + + def get_bits(self): + """ + Return the number of significant bits in this key. This is useful + for judging the relative security of a key. + + :return: bits in the key (as an `int`) + """ + # TODO 4.0: raise NotImplementedError, 0 is unlikely to ever be + # _correct_ and nothing in the critical path seems to use this. + return 0 + + def can_sign(self): + """ + Return ``True`` if this key has the private part necessary for signing + data. + """ + return False + + def get_fingerprint(self): + """ + Return an MD5 fingerprint of the public part of this key. Nothing + secret is revealed. + + :return: + a 16-byte `string ` (binary) of the MD5 fingerprint, in SSH + format. + """ + return md5(self.asbytes()).digest() + + @property + def fingerprint(self): + """ + Modern fingerprint property designed to be comparable to OpenSSH. + + Currently only does SHA256 (the OpenSSH default). + + .. versionadded:: 3.2 + """ + hashy = sha256(bytes(self)) + hash_name = hashy.name.upper() + b64ed = encodebytes(hashy.digest()) + cleaned = u(b64ed).strip().rstrip("=") # yes, OpenSSH does this too! + return f"{hash_name}:{cleaned}" + + def get_base64(self): + """ + Return a base64 string containing the public part of this key. Nothing + secret is revealed. This format is compatible with that used to store + public key files or recognized host keys. + + :return: a base64 `string ` containing the public part of the key. + """ + return u(encodebytes(self.asbytes())).replace("\n", "") + + def sign_ssh_data(self, data, algorithm=None): + """ + Sign a blob of data with this private key, and return a `.Message` + representing an SSH signature message. + + :param bytes data: + the data to sign. + :param str algorithm: + the signature algorithm to use, if different from the key's + internal name. Default: ``None``. + :return: an SSH signature `message <.Message>`. + + .. versionchanged:: 2.9 + Added the ``algorithm`` kwarg. + """ + return bytes() + + def verify_ssh_sig(self, data, msg): + """ + Given a blob of data, and an SSH message representing a signature of + that data, verify that it was signed with this key. + + :param bytes data: the data that was signed. + :param .Message msg: an SSH signature message + :return: + ``True`` if the signature verifies correctly; ``False`` otherwise. + """ + return False + + @classmethod + def from_private_key_file(cls, filename, password=None): + """ + Create a key object by reading a private key file. If the private + key is encrypted and ``password`` is not ``None``, the given password + will be used to decrypt the key (otherwise `.PasswordRequiredException` + is thrown). Through the magic of Python, this factory method will + exist in all subclasses of PKey (such as `.RSAKey`), but + is useless on the abstract PKey class. + + :param str filename: name of the file to read + :param str password: + an optional password to use to decrypt the key file, if it's + encrypted + :return: a new `.PKey` based on the given private key + + :raises: ``IOError`` -- if there was an error reading the file + :raises: `.PasswordRequiredException` -- if the private key file is + encrypted, and ``password`` is ``None`` + :raises: `.SSHException` -- if the key file is invalid + """ + key = cls(filename=filename, password=password) + return key + + @classmethod + def from_private_key(cls, file_obj, password=None): + """ + Create a key object by reading a private key from a file (or file-like) + object. If the private key is encrypted and ``password`` is not + ``None``, the given password will be used to decrypt the key (otherwise + `.PasswordRequiredException` is thrown). + + :param file_obj: the file-like object to read from + :param str password: + an optional password to use to decrypt the key, if it's encrypted + :return: a new `.PKey` based on the given private key + + :raises: ``IOError`` -- if there was an error reading the key + :raises: `.PasswordRequiredException` -- + if the private key file is encrypted, and ``password`` is ``None`` + :raises: `.SSHException` -- if the key file is invalid + """ + key = cls(file_obj=file_obj, password=password) + return key + + def write_private_key_file(self, filename, password=None): + """ + Write private key contents into a file. If the password is not + ``None``, the key is encrypted before writing. + + :param str filename: name of the file to write + :param str password: + an optional password to use to encrypt the key file + + :raises: ``IOError`` -- if there was an error writing the file + :raises: `.SSHException` -- if the key is invalid + """ + raise Exception("Not implemented in PKey") + + def write_private_key(self, file_obj, password=None): + """ + Write private key contents into a file (or file-like) object. If the + password is not ``None``, the key is encrypted before writing. + + :param file_obj: the file-like object to write into + :param str password: an optional password to use to encrypt the key + + :raises: ``IOError`` -- if there was an error writing to the file + :raises: `.SSHException` -- if the key is invalid + """ + # TODO 4.0: NotImplementedError (plus everywhere else in here) + raise Exception("Not implemented in PKey") + + def _read_private_key_file(self, tag, filename, password=None): + """ + Read an SSH2-format private key file, looking for a string of the type + ``"BEGIN xxx PRIVATE KEY"`` for some ``xxx``, base64-decode the text we + find, and return it as a string. If the private key is encrypted and + ``password`` is not ``None``, the given password will be used to + decrypt the key (otherwise `.PasswordRequiredException` is thrown). + + :param str tag: + ``"RSA"`` (or etc), the tag used to mark the data block. + :param str filename: + name of the file to read. + :param str password: + an optional password to use to decrypt the key file, if it's + encrypted. + :return: + the `bytes` that make up the private key. + + :raises: ``IOError`` -- if there was an error reading the file. + :raises: `.PasswordRequiredException` -- if the private key file is + encrypted, and ``password`` is ``None``. + :raises: `.SSHException` -- if the key file is invalid. + """ + with open(filename, "r") as f: + data = self._read_private_key(tag, f, password) + return data + + def _read_private_key(self, tag, f, password=None): + lines = f.readlines() + if not lines: + raise SSHException("no lines in {} private key file".format(tag)) + + # find the BEGIN tag + start = 0 + m = self.BEGIN_TAG.match(lines[start]) + line_range = len(lines) - 1 + while start < line_range and not m: + start += 1 + m = self.BEGIN_TAG.match(lines[start]) + start += 1 + keytype = m.group(1) if m else None + if start >= len(lines) or keytype is None: + raise SSHException("not a valid {} private key file".format(tag)) + + # find the END tag + end = start + m = self.END_TAG.match(lines[end]) + while end < line_range and not m: + end += 1 + m = self.END_TAG.match(lines[end]) + + if keytype == tag: + data = self._read_private_key_pem(lines, end, password) + pkformat = self._PRIVATE_KEY_FORMAT_ORIGINAL + elif keytype == "OPENSSH": + data = self._read_private_key_openssh(lines[start:end], password) + pkformat = self._PRIVATE_KEY_FORMAT_OPENSSH + else: + raise SSHException( + "encountered {} key, expected {} key".format(keytype, tag) + ) + + return pkformat, data + + def _got_bad_key_format_id(self, id_): + err = "{}._read_private_key() spat out an unknown key format id '{}'" + raise SSHException(err.format(self.__class__.__name__, id_)) + + def _read_private_key_pem(self, lines, end, password): + start = 0 + # parse any headers first + headers = {} + start += 1 + while start < len(lines): + line = lines[start].split(": ") + if len(line) == 1: + break + headers[line[0].lower()] = line[1].strip() + start += 1 + # if we trudged to the end of the file, just try to cope. + try: + data = decodebytes(b("".join(lines[start:end]))) + except base64.binascii.Error as e: + raise SSHException("base64 decoding error: {}".format(e)) + if "proc-type" not in headers: + # unencryped: done + return data + # encrypted keyfile: will need a password + proc_type = headers["proc-type"] + if proc_type != "4,ENCRYPTED": + raise SSHException( + 'Unknown private key structure "{}"'.format(proc_type) + ) + try: + encryption_type, saltstr = headers["dek-info"].split(",") + except: + raise SSHException("Can't parse DEK-info in private key file") + if encryption_type not in self._CIPHER_TABLE: + raise SSHException( + 'Unknown private key cipher "{}"'.format(encryption_type) + ) + # if no password was passed in, + # raise an exception pointing out that we need one + if password is None: + raise PasswordRequiredException("Private key file is encrypted") + cipher = self._CIPHER_TABLE[encryption_type]["cipher"] + keysize = self._CIPHER_TABLE[encryption_type]["keysize"] + mode = self._CIPHER_TABLE[encryption_type]["mode"] + salt = unhexlify(b(saltstr)) + key = util.generate_key_bytes(md5, salt, password, keysize) + decryptor = Cipher( + cipher(key), mode(salt), backend=default_backend() + ).decryptor() + decrypted_data = decryptor.update(data) + decryptor.finalize() + unpadder = padding.PKCS7(cipher.block_size).unpadder() + try: + return unpadder.update(decrypted_data) + unpadder.finalize() + except ValueError: + raise SSHException("Bad password or corrupt private key file") + + def _read_private_key_openssh(self, lines, password): + """ + Read the new OpenSSH SSH2 private key format available + since OpenSSH version 6.5 + Reference: + https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key + """ + try: + data = decodebytes(b("".join(lines))) + except base64.binascii.Error as e: + raise SSHException("base64 decoding error: {}".format(e)) + + # read data struct + auth_magic = data[:15] + if auth_magic != OPENSSH_AUTH_MAGIC: + raise SSHException("unexpected OpenSSH key header encountered") + + cstruct = self._uint32_cstruct_unpack(data[15:], "sssur") + cipher, kdfname, kdf_options, num_pubkeys, remainder = cstruct + # For now, just support 1 key. + if num_pubkeys > 1: + raise SSHException( + "unsupported: private keyfile has multiple keys" + ) + pubkey, privkey_blob = self._uint32_cstruct_unpack(remainder, "ss") + + if kdfname == b("bcrypt"): + if cipher == b("aes256-cbc"): + mode = modes.CBC + elif cipher == b("aes256-ctr"): + mode = modes.CTR + else: + raise SSHException( + "unknown cipher `{}` used in private key file".format( + cipher.decode("utf-8") + ) + ) + # Encrypted private key. + # If no password was passed in, raise an exception pointing + # out that we need one + if password is None: + raise PasswordRequiredException( + "private key file is encrypted" + ) + + # Unpack salt and rounds from kdfoptions + salt, rounds = self._uint32_cstruct_unpack(kdf_options, "su") + + # run bcrypt kdf to derive key and iv/nonce (32 + 16 bytes) + key_iv = bcrypt.kdf( + b(password), + b(salt), + 48, + rounds, + # We can't control how many rounds are on disk, so no sense + # warning about it. + ignore_few_rounds=True, + ) + key = key_iv[:32] + iv = key_iv[32:] + + # decrypt private key blob + decryptor = Cipher( + algorithms.AES(key), mode(iv), default_backend() + ).decryptor() + decrypted_privkey = decryptor.update(privkey_blob) + decrypted_privkey += decryptor.finalize() + elif cipher == b("none") and kdfname == b("none"): + # Unencrypted private key + decrypted_privkey = privkey_blob + else: + raise SSHException( + "unknown cipher or kdf used in private key file" + ) + + # Unpack private key and verify checkints + cstruct = self._uint32_cstruct_unpack(decrypted_privkey, "uusr") + checkint1, checkint2, keytype, keydata = cstruct + + if checkint1 != checkint2: + raise SSHException( + "OpenSSH private key file checkints do not match" + ) + + return _unpad_openssh(keydata) + + def _uint32_cstruct_unpack(self, data, strformat): + """ + Used to read new OpenSSH private key format. + Unpacks a c data structure containing a mix of 32-bit uints and + variable length strings prefixed by 32-bit uint size field, + according to the specified format. Returns the unpacked vars + in a tuple. + Format strings: + s - denotes a string + i - denotes a long integer, encoded as a byte string + u - denotes a 32-bit unsigned integer + r - the remainder of the input string, returned as a string + """ + arr = [] + idx = 0 + try: + for f in strformat: + if f == "s": + # string + s_size = struct.unpack(">L", data[idx : idx + 4])[0] + idx += 4 + s = data[idx : idx + s_size] + idx += s_size + arr.append(s) + if f == "i": + # long integer + s_size = struct.unpack(">L", data[idx : idx + 4])[0] + idx += 4 + s = data[idx : idx + s_size] + idx += s_size + i = util.inflate_long(s, True) + arr.append(i) + elif f == "u": + # 32-bit unsigned int + u = struct.unpack(">L", data[idx : idx + 4])[0] + idx += 4 + arr.append(u) + elif f == "r": + # remainder as string + s = data[idx:] + arr.append(s) + break + except Exception as e: + # PKey-consuming code frequently wants to save-and-skip-over issues + # with loading keys, and uses SSHException as the (really friggin + # awful) signal for this. So for now...we do this. + raise SSHException(str(e)) + return tuple(arr) + + def _write_private_key_file(self, filename, key, format, password=None): + """ + Write an SSH2-format private key file in a form that can be read by + paramiko or openssh. If no password is given, the key is written in + a trivially-encoded format (base64) which is completely insecure. If + a password is given, DES-EDE3-CBC is used. + + :param str tag: + ``"RSA"`` or etc, the tag used to mark the data block. + :param filename: name of the file to write. + :param bytes data: data blob that makes up the private key. + :param str password: an optional password to use to encrypt the file. + + :raises: ``IOError`` -- if there was an error writing the file. + """ + # Ensure that we create new key files directly with a user-only mode, + # instead of opening, writing, then chmodding, which leaves us open to + # CVE-2022-24302. + with os.fdopen( + os.open( + filename, + # NOTE: O_TRUNC is a noop on new files, and O_CREAT is a noop + # on existing files, so using all 3 in both cases is fine. + flags=os.O_WRONLY | os.O_TRUNC | os.O_CREAT, + # Ditto the use of the 'mode' argument; it should be safe to + # give even for existing files (though it will not act like a + # chmod in that case). + mode=o600, + ), + # Yea, you still gotta inform the FLO that it is in "write" mode. + "w", + ) as f: + self._write_private_key(f, key, format, password=password) + + def _write_private_key(self, f, key, format, password=None): + if password is None: + encryption = serialization.NoEncryption() + else: + encryption = serialization.BestAvailableEncryption(b(password)) + + f.write( + key.private_bytes( + serialization.Encoding.PEM, format, encryption + ).decode() + ) + + def _check_type_and_load_cert(self, msg, key_type, cert_type): + """ + Perform message type-checking & optional certificate loading. + + This includes fast-forwarding cert ``msg`` objects past the nonce, so + that the subsequent fields are the key numbers; thus the caller may + expect to treat the message as key material afterwards either way. + + The obtained key type is returned for classes which need to know what + it was (e.g. ECDSA.) + """ + # Normalization; most classes have a single key type and give a string, + # but eg ECDSA is a 1:N mapping. + key_types = key_type + cert_types = cert_type + if isinstance(key_type, str): + key_types = [key_types] + if isinstance(cert_types, str): + cert_types = [cert_types] + # Can't do much with no message, that should've been handled elsewhere + if msg is None: + raise SSHException("Key object may not be empty") + # First field is always key type, in either kind of object. (make sure + # we rewind before grabbing it - sometimes caller had to do their own + # introspection first!) + msg.rewind() + type_ = msg.get_text() + # Regular public key - nothing special to do besides the implicit + # type check. + if type_ in key_types: + pass + # OpenSSH-compatible certificate - store full copy as .public_blob + # (so signing works correctly) and then fast-forward past the + # nonce. + elif type_ in cert_types: + # This seems the cleanest way to 'clone' an already-being-read + # message; they're *IO objects at heart and their .getvalue() + # always returns the full value regardless of pointer position. + self.load_certificate(Message(msg.asbytes())) + # Read out nonce as it comes before the public numbers - our caller + # is likely going to use the (only borrowed by us, not owned) + # 'msg' object for loading those numbers right after this. + # TODO: usefully interpret it & other non-public-number fields + # (requires going back into per-type subclasses.) + msg.get_string() + else: + err = "Invalid key (class: {}, data type: {}" + raise SSHException(err.format(self.__class__.__name__, type_)) + + def load_certificate(self, value): + """ + Supplement the private key contents with data loaded from an OpenSSH + public key (``.pub``) or certificate (``-cert.pub``) file, a string + containing such a file, or a `.Message` object. + + The .pub contents adds no real value, since the private key + file includes sufficient information to derive the public + key info. For certificates, however, this can be used on + the client side to offer authentication requests to the server + based on certificate instead of raw public key. + + See: + https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.certkeys + + Note: very little effort is made to validate the certificate contents, + that is for the server to decide if it is good enough to authenticate + successfully. + """ + if isinstance(value, Message): + constructor = "from_message" + elif os.path.isfile(value): + constructor = "from_file" + else: + constructor = "from_string" + blob = getattr(PublicBlob, constructor)(value) + if not blob.key_type.startswith(self.get_name()): + err = "PublicBlob type {} incompatible with key type {}" + raise ValueError(err.format(blob.key_type, self.get_name())) + self.public_blob = blob + + +# General construct for an OpenSSH style Public Key blob +# readable from a one-line file of the format: +# [] +# Of little value in the case of standard public keys +# {ssh-rsa, ssh-ecdsa, ssh-ed25519}, but should +# provide rudimentary support for {*-cert.v01} +class PublicBlob: + """ + OpenSSH plain public key or OpenSSH signed public key (certificate). + + Tries to be as dumb as possible and barely cares about specific + per-key-type data. + + .. note:: + + Most of the time you'll want to call `from_file`, `from_string` or + `from_message` for useful instantiation, the main constructor is + basically "I should be using ``attrs`` for this." + """ + + def __init__(self, type_, blob, comment=None): + """ + Create a new public blob of given type and contents. + + :param str type_: Type indicator, eg ``ssh-rsa``. + :param bytes blob: The blob bytes themselves. + :param str comment: A comment, if one was given (e.g. file-based.) + """ + self.key_type = type_ + self.key_blob = blob + self.comment = comment + + @classmethod + def from_file(cls, filename): + """ + Create a public blob from a ``-cert.pub``-style file on disk. + """ + with open(filename) as f: + string = f.read() + return cls.from_string(string) + + @classmethod + def from_string(cls, string): + """ + Create a public blob from a ``-cert.pub``-style string. + """ + fields = string.split(None, 2) + if len(fields) < 2: + msg = "Not enough fields for public blob: {}" + raise ValueError(msg.format(fields)) + key_type = fields[0] + key_blob = decodebytes(b(fields[1])) + try: + comment = fields[2].strip() + except IndexError: + comment = None + # Verify that the blob message first (string) field matches the + # key_type + m = Message(key_blob) + blob_type = m.get_text() + if blob_type != key_type: + deets = "key type={!r}, but blob type={!r}".format( + key_type, blob_type + ) + raise ValueError("Invalid PublicBlob contents: {}".format(deets)) + # All good? All good. + return cls(type_=key_type, blob=key_blob, comment=comment) + + @classmethod + def from_message(cls, message): + """ + Create a public blob from a network `.Message`. + + Specifically, a cert-bearing pubkey auth packet, because by definition + OpenSSH-style certificates 'are' their own network representation." + """ + type_ = message.get_text() + return cls(type_=type_, blob=message.asbytes()) + + def __str__(self): + ret = "{} public key/certificate".format(self.key_type) + if self.comment: + ret += "- {}".format(self.comment) + return ret + + def __eq__(self, other): + # Just piggyback on Message/BytesIO, since both of these should be one. + return self and other and self.key_blob == other.key_blob + + def __ne__(self, other): + return not self == other diff --git a/lib/paramiko/primes.py b/lib/paramiko/primes.py new file mode 100644 index 0000000..663c58e --- /dev/null +++ b/lib/paramiko/primes.py @@ -0,0 +1,148 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Utility functions for dealing with primes. +""" + +import os + +from paramiko import util +from paramiko.common import byte_mask +from paramiko.ssh_exception import SSHException + + +def _roll_random(n): + """returns a random # from 0 to N-1""" + bits = util.bit_length(n - 1) + byte_count = (bits + 7) // 8 + hbyte_mask = pow(2, bits % 8) - 1 + + # so here's the plan: + # we fetch as many random bits as we'd need to fit N-1, and if the + # generated number is >= N, we try again. in the worst case (N-1 is a + # power of 2), we have slightly better than 50% odds of getting one that + # fits, so i can't guarantee that this loop will ever finish, but the odds + # of it looping forever should be infinitesimal. + while True: + x = os.urandom(byte_count) + if hbyte_mask > 0: + x = byte_mask(x[0], hbyte_mask) + x[1:] + num = util.inflate_long(x, 1) + if num < n: + break + return num + + +class ModulusPack: + """ + convenience object for holding the contents of the /etc/ssh/moduli file, + on systems that have such a file. + """ + + def __init__(self): + # pack is a hash of: bits -> [ (generator, modulus) ... ] + self.pack = {} + self.discarded = [] + + def _parse_modulus(self, line): + ( + timestamp, + mod_type, + tests, + tries, + size, + generator, + modulus, + ) = line.split() + mod_type = int(mod_type) + tests = int(tests) + tries = int(tries) + size = int(size) + generator = int(generator) + modulus = int(modulus, 16) + + # weed out primes that aren't at least: + # type 2 (meets basic structural requirements) + # test 4 (more than just a small-prime sieve) + # tries < 100 if test & 4 (at least 100 tries of miller-rabin) + if ( + mod_type < 2 + or tests < 4 + or (tests & 4 and tests < 8 and tries < 100) + ): + self.discarded.append( + (modulus, "does not meet basic requirements") + ) + return + if generator == 0: + generator = 2 + + # there's a bug in the ssh "moduli" file (yeah, i know: shock! dismay! + # call cnn!) where it understates the bit lengths of these primes by 1. + # this is okay. + bl = util.bit_length(modulus) + if (bl != size) and (bl != size + 1): + self.discarded.append( + (modulus, "incorrectly reported bit length {}".format(size)) + ) + return + if bl not in self.pack: + self.pack[bl] = [] + self.pack[bl].append((generator, modulus)) + + def read_file(self, filename): + """ + :raises IOError: passed from any file operations that fail. + """ + self.pack = {} + with open(filename, "r") as f: + for line in f: + line = line.strip() + if (len(line) == 0) or (line[0] == "#"): + continue + try: + self._parse_modulus(line) + except: + continue + + def get_modulus(self, min, prefer, max): + bitsizes = sorted(self.pack.keys()) + if len(bitsizes) == 0: + raise SSHException("no moduli available") + good = -1 + # find nearest bitsize >= preferred + for b in bitsizes: + if (b >= prefer) and (b <= max) and (b < good or good == -1): + good = b + # if that failed, find greatest bitsize >= min + if good == -1: + for b in bitsizes: + if (b >= min) and (b <= max) and (b > good): + good = b + if good == -1: + # their entire (min, max) range has no intersection with our range. + # if their range is below ours, pick the smallest. otherwise pick + # the largest. it'll be out of their range requirement either way, + # but we'll be sending them the closest one we have. + good = bitsizes[0] + if min > good: + good = bitsizes[-1] + # now pick a random modulus of this bitsize + n = _roll_random(len(self.pack[good])) + return self.pack[good][n] diff --git a/lib/paramiko/proxy.py b/lib/paramiko/proxy.py new file mode 100644 index 0000000..f7609c9 --- /dev/null +++ b/lib/paramiko/proxy.py @@ -0,0 +1,134 @@ +# Copyright (C) 2012 Yipit, Inc +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import shlex +import signal +from select import select +import socket +import time + +# Try-and-ignore import so platforms w/o subprocess (eg Google App Engine) can +# still import paramiko. +subprocess, subprocess_import_error = None, None +try: + import subprocess +except ImportError as e: + subprocess_import_error = e + +from paramiko.ssh_exception import ProxyCommandFailure +from paramiko.util import ClosingContextManager + + +class ProxyCommand(ClosingContextManager): + """ + Wraps a subprocess running ProxyCommand-driven programs. + + This class implements a the socket-like interface needed by the + `.Transport` and `.Packetizer` classes. Using this class instead of a + regular socket makes it possible to talk with a Popen'd command that will + proxy traffic between the client and a server hosted in another machine. + + Instances of this class may be used as context managers. + """ + + def __init__(self, command_line): + """ + Create a new CommandProxy instance. The instance created by this + class can be passed as an argument to the `.Transport` class. + + :param str command_line: + the command that should be executed and used as the proxy. + """ + if subprocess is None: + raise subprocess_import_error + self.cmd = shlex.split(command_line) + self.process = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + ) + self.timeout = None + + def send(self, content): + """ + Write the content received from the SSH client to the standard + input of the forked command. + + :param str content: string to be sent to the forked command + """ + try: + self.process.stdin.write(content) + except IOError as e: + # There was a problem with the child process. It probably + # died and we can't proceed. The best option here is to + # raise an exception informing the user that the informed + # ProxyCommand is not working. + raise ProxyCommandFailure(" ".join(self.cmd), e.strerror) + return len(content) + + def recv(self, size): + """ + Read from the standard output of the forked program. + + :param int size: how many chars should be read + + :return: the string of bytes read, which may be shorter than requested + """ + try: + buffer = b"" + start = time.time() + while len(buffer) < size: + select_timeout = None + if self.timeout is not None: + elapsed = time.time() - start + if elapsed >= self.timeout: + raise socket.timeout() + select_timeout = self.timeout - elapsed + + r, w, x = select([self.process.stdout], [], [], select_timeout) + if r and r[0] == self.process.stdout: + buffer += os.read( + self.process.stdout.fileno(), size - len(buffer) + ) + return buffer + except socket.timeout: + if buffer: + # Don't raise socket.timeout, return partial result instead + return buffer + raise # socket.timeout is a subclass of IOError + except IOError as e: + raise ProxyCommandFailure(" ".join(self.cmd), e.strerror) + + def close(self): + os.kill(self.process.pid, signal.SIGTERM) + + @property + def closed(self): + return self.process.returncode is not None + + @property + def _closed(self): + # Concession to Python 3 socket-like API + return self.closed + + def settimeout(self, timeout): + self.timeout = timeout diff --git a/lib/paramiko/rsakey.py b/lib/paramiko/rsakey.py new file mode 100644 index 0000000..b7ad3ce --- /dev/null +++ b/lib/paramiko/rsakey.py @@ -0,0 +1,227 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +RSA keys. +""" + +from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa, padding + +from paramiko.message import Message +from paramiko.pkey import PKey +from paramiko.ssh_exception import SSHException + + +class RSAKey(PKey): + """ + Representation of an RSA key which can be used to sign and verify SSH2 + data. + """ + + name = "ssh-rsa" + HASHES = { + "ssh-rsa": hashes.SHA1, + "ssh-rsa-cert-v01@openssh.com": hashes.SHA1, + "rsa-sha2-256": hashes.SHA256, + "rsa-sha2-256-cert-v01@openssh.com": hashes.SHA256, + "rsa-sha2-512": hashes.SHA512, + "rsa-sha2-512-cert-v01@openssh.com": hashes.SHA512, + } + + def __init__( + self, + msg=None, + data=None, + filename=None, + password=None, + key=None, + file_obj=None, + ): + self.key = None + self.public_blob = None + if file_obj is not None: + self._from_private_key(file_obj, password) + return + if filename is not None: + self._from_private_key_file(filename, password) + return + if (msg is None) and (data is not None): + msg = Message(data) + if key is not None: + self.key = key + else: + self._check_type_and_load_cert( + msg=msg, + # NOTE: this does NOT change when using rsa2 signatures; it's + # purely about key loading, not exchange or verification + key_type=self.name, + cert_type="ssh-rsa-cert-v01@openssh.com", + ) + self.key = rsa.RSAPublicNumbers( + e=msg.get_mpint(), n=msg.get_mpint() + ).public_key(default_backend()) + + @classmethod + def identifiers(cls): + return list(cls.HASHES.keys()) + + @property + def size(self): + return self.key.key_size + + @property + def public_numbers(self): + if isinstance(self.key, rsa.RSAPrivateKey): + return self.key.private_numbers().public_numbers + else: + return self.key.public_numbers() + + def asbytes(self): + m = Message() + m.add_string(self.name) + m.add_mpint(self.public_numbers.e) + m.add_mpint(self.public_numbers.n) + return m.asbytes() + + def __str__(self): + # NOTE: see #853 to explain some legacy behavior. + # TODO 4.0: replace with a nice clean fingerprint display or something + return self.asbytes().decode("utf8", errors="ignore") + + @property + def _fields(self): + return (self.get_name(), self.public_numbers.e, self.public_numbers.n) + + def get_name(self): + return self.name + + def get_bits(self): + return self.size + + def can_sign(self): + return isinstance(self.key, rsa.RSAPrivateKey) + + def sign_ssh_data(self, data, algorithm=None): + if algorithm is None: + algorithm = self.name + sig = self.key.sign( + data, + padding=padding.PKCS1v15(), + # HASHES being just a map from long identifier to either SHA1 or + # SHA256 - cert'ness is not truly relevant. + algorithm=self.HASHES[algorithm](), + ) + m = Message() + # And here again, cert'ness is irrelevant, so it is stripped out. + m.add_string(algorithm.replace("-cert-v01@openssh.com", "")) + m.add_string(sig) + return m + + def verify_ssh_sig(self, data, msg): + sig_algorithm = msg.get_text() + if sig_algorithm not in self.HASHES: + return False + key = self.key + if isinstance(key, rsa.RSAPrivateKey): + key = key.public_key() + + # NOTE: pad received signature with leading zeros, key.verify() + # expects a signature of key size (e.g. PuTTY doesn't pad) + sign = msg.get_binary() + diff = key.key_size - len(sign) * 8 + if diff > 0: + sign = b"\x00" * ((diff + 7) // 8) + sign + + try: + key.verify( + sign, data, padding.PKCS1v15(), self.HASHES[sig_algorithm]() + ) + except InvalidSignature: + return False + else: + return True + + def write_private_key_file(self, filename, password=None): + self._write_private_key_file( + filename, + self.key, + serialization.PrivateFormat.TraditionalOpenSSL, + password=password, + ) + + def write_private_key(self, file_obj, password=None): + self._write_private_key( + file_obj, + self.key, + serialization.PrivateFormat.TraditionalOpenSSL, + password=password, + ) + + @staticmethod + def generate(bits, progress_func=None): + """ + Generate a new private RSA key. This factory function can be used to + generate a new host key or authentication key. + + :param int bits: number of bits the generated key should be. + :param progress_func: Unused + :return: new `.RSAKey` private key + """ + key = rsa.generate_private_key( + public_exponent=65537, key_size=bits, backend=default_backend() + ) + return RSAKey(key=key) + + # ...internals... + + def _from_private_key_file(self, filename, password): + data = self._read_private_key_file("RSA", filename, password) + self._decode_key(data) + + def _from_private_key(self, file_obj, password): + data = self._read_private_key("RSA", file_obj, password) + self._decode_key(data) + + def _decode_key(self, data): + pkformat, data = data + if pkformat == self._PRIVATE_KEY_FORMAT_ORIGINAL: + try: + key = serialization.load_der_private_key( + data, password=None, backend=default_backend() + ) + except (ValueError, TypeError, UnsupportedAlgorithm) as e: + raise SSHException(str(e)) + elif pkformat == self._PRIVATE_KEY_FORMAT_OPENSSH: + n, e, d, iqmp, p, q = self._uint32_cstruct_unpack(data, "iiiiii") + public_numbers = rsa.RSAPublicNumbers(e=e, n=n) + key = rsa.RSAPrivateNumbers( + p=p, + q=q, + d=d, + dmp1=d % (p - 1), + dmq1=d % (q - 1), + iqmp=iqmp, + public_numbers=public_numbers, + ).private_key(default_backend()) + else: + self._got_bad_key_format_id(pkformat) + assert isinstance(key, rsa.RSAPrivateKey) + self.key = key diff --git a/lib/paramiko/server.py b/lib/paramiko/server.py new file mode 100644 index 0000000..6923bdf --- /dev/null +++ b/lib/paramiko/server.py @@ -0,0 +1,732 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +`.ServerInterface` is an interface to override for server support. +""" + +import threading +from paramiko import util +from paramiko.common import ( + DEBUG, + ERROR, + OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, + AUTH_FAILED, + AUTH_SUCCESSFUL, +) + + +class ServerInterface: + """ + This class defines an interface for controlling the behavior of Paramiko + in server mode. + + Methods on this class are called from Paramiko's primary thread, so you + shouldn't do too much work in them. (Certainly nothing that blocks or + sleeps.) + """ + + def check_channel_request(self, kind, chanid): + """ + Determine if a channel request of a given type will be granted, and + return ``OPEN_SUCCEEDED`` or an error code. This method is + called in server mode when the client requests a channel, after + authentication is complete. + + If you allow channel requests (and an ssh server that didn't would be + useless), you should also override some of the channel request methods + below, which are used to determine which services will be allowed on + a given channel: + + - `check_channel_pty_request` + - `check_channel_shell_request` + - `check_channel_subsystem_request` + - `check_channel_window_change_request` + - `check_channel_x11_request` + - `check_channel_forward_agent_request` + + The ``chanid`` parameter is a small number that uniquely identifies the + channel within a `.Transport`. A `.Channel` object is not created + unless this method returns ``OPEN_SUCCEEDED`` -- once a + `.Channel` object is created, you can call `.Channel.get_id` to + retrieve the channel ID. + + The return value should either be ``OPEN_SUCCEEDED`` (or + ``0``) to allow the channel request, or one of the following error + codes to reject it: + + - ``OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED`` + - ``OPEN_FAILED_CONNECT_FAILED`` + - ``OPEN_FAILED_UNKNOWN_CHANNEL_TYPE`` + - ``OPEN_FAILED_RESOURCE_SHORTAGE`` + + The default implementation always returns + ``OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED``. + + :param str kind: + the kind of channel the client would like to open (usually + ``"session"``). + :param int chanid: ID of the channel + :return: an `int` success or failure code (listed above) + """ + return OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def get_allowed_auths(self, username): + """ + Return a list of authentication methods supported by the server. + This list is sent to clients attempting to authenticate, to inform them + of authentication methods that might be successful. + + The "list" is actually a string of comma-separated names of types of + authentication. Possible values are ``"password"``, ``"publickey"``, + and ``"none"``. + + The default implementation always returns ``"password"``. + + :param str username: the username requesting authentication. + :return: a comma-separated `str` of authentication types + """ + return "password" + + def check_auth_none(self, username): + """ + Determine if a client may open channels with no (further) + authentication. + + Return ``AUTH_FAILED`` if the client must authenticate, or + ``AUTH_SUCCESSFUL`` if it's okay for the client to not + authenticate. + + The default implementation always returns ``AUTH_FAILED``. + + :param str username: the username of the client. + :return: + ``AUTH_FAILED`` if the authentication fails; ``AUTH_SUCCESSFUL`` if + it succeeds. + :rtype: int + """ + return AUTH_FAILED + + def check_auth_password(self, username, password): + """ + Determine if a given username and password supplied by the client is + acceptable for use in authentication. + + Return ``AUTH_FAILED`` if the password is not accepted, + ``AUTH_SUCCESSFUL`` if the password is accepted and completes + the authentication, or ``AUTH_PARTIALLY_SUCCESSFUL`` if your + authentication is stateful, and this key is accepted for + authentication, but more authentication is required. (In this latter + case, `get_allowed_auths` will be called to report to the client what + options it has for continuing the authentication.) + + The default implementation always returns ``AUTH_FAILED``. + + :param str username: the username of the authenticating client. + :param str password: the password given by the client. + :return: + ``AUTH_FAILED`` if the authentication fails; ``AUTH_SUCCESSFUL`` if + it succeeds; ``AUTH_PARTIALLY_SUCCESSFUL`` if the password auth is + successful, but authentication must continue. + :rtype: int + """ + return AUTH_FAILED + + def check_auth_publickey(self, username, key): + """ + Determine if a given key supplied by the client is acceptable for use + in authentication. You should override this method in server mode to + check the username and key and decide if you would accept a signature + made using this key. + + Return ``AUTH_FAILED`` if the key is not accepted, + ``AUTH_SUCCESSFUL`` if the key is accepted and completes the + authentication, or ``AUTH_PARTIALLY_SUCCESSFUL`` if your + authentication is stateful, and this password is accepted for + authentication, but more authentication is required. (In this latter + case, `get_allowed_auths` will be called to report to the client what + options it has for continuing the authentication.) + + Note that you don't have to actually verify any key signtature here. + If you're willing to accept the key, Paramiko will do the work of + verifying the client's signature. + + The default implementation always returns ``AUTH_FAILED``. + + :param str username: the username of the authenticating client + :param .PKey key: the key object provided by the client + :return: + ``AUTH_FAILED`` if the client can't authenticate with this key; + ``AUTH_SUCCESSFUL`` if it can; ``AUTH_PARTIALLY_SUCCESSFUL`` if it + can authenticate with this key but must continue with + authentication + :rtype: int + """ + return AUTH_FAILED + + def check_auth_interactive(self, username, submethods): + """ + Begin an interactive authentication challenge, if supported. You + should override this method in server mode if you want to support the + ``"keyboard-interactive"`` auth type, which requires you to send a + series of questions for the client to answer. + + Return ``AUTH_FAILED`` if this auth method isn't supported. Otherwise, + you should return an `.InteractiveQuery` object containing the prompts + and instructions for the user. The response will be sent via a call + to `check_auth_interactive_response`. + + The default implementation always returns ``AUTH_FAILED``. + + :param str username: the username of the authenticating client + :param str submethods: + a comma-separated list of methods preferred by the client (usually + empty) + :return: + ``AUTH_FAILED`` if this auth method isn't supported; otherwise an + object containing queries for the user + :rtype: int or `.InteractiveQuery` + """ + return AUTH_FAILED + + def check_auth_interactive_response(self, responses): + """ + Continue or finish an interactive authentication challenge, if + supported. You should override this method in server mode if you want + to support the ``"keyboard-interactive"`` auth type. + + Return ``AUTH_FAILED`` if the responses are not accepted, + ``AUTH_SUCCESSFUL`` if the responses are accepted and complete + the authentication, or ``AUTH_PARTIALLY_SUCCESSFUL`` if your + authentication is stateful, and this set of responses is accepted for + authentication, but more authentication is required. (In this latter + case, `get_allowed_auths` will be called to report to the client what + options it has for continuing the authentication.) + + If you wish to continue interactive authentication with more questions, + you may return an `.InteractiveQuery` object, which should cause the + client to respond with more answers, calling this method again. This + cycle can continue indefinitely. + + The default implementation always returns ``AUTH_FAILED``. + + :param responses: list of `str` responses from the client + :return: + ``AUTH_FAILED`` if the authentication fails; ``AUTH_SUCCESSFUL`` if + it succeeds; ``AUTH_PARTIALLY_SUCCESSFUL`` if the interactive auth + is successful, but authentication must continue; otherwise an + object containing queries for the user + :rtype: int or `.InteractiveQuery` + """ + return AUTH_FAILED + + def check_auth_gssapi_with_mic( + self, username, gss_authenticated=AUTH_FAILED, cc_file=None + ): + """ + Authenticate the given user to the server if he is a valid krb5 + principal. + + :param str username: The username of the authenticating client + :param int gss_authenticated: The result of the krb5 authentication + :param str cc_filename: The krb5 client credentials cache filename + :return: ``AUTH_FAILED`` if the user is not authenticated otherwise + ``AUTH_SUCCESSFUL`` + :rtype: int + :note: Kerberos credential delegation is not supported. + :see: `.ssh_gss` + :note: : We are just checking in L{AuthHandler} that the given user is + a valid krb5 principal! + We don't check if the krb5 principal is allowed to log in on + the server, because there is no way to do that in python. So + if you develop your own SSH server with paramiko for a certain + platform like Linux, you should call C{krb5_kuserok()} in + your local kerberos library to make sure that the + krb5_principal has an account on the server and is allowed to + log in as a user. + :see: http://www.unix.com/man-page/all/3/krb5_kuserok/ + """ + if gss_authenticated == AUTH_SUCCESSFUL: + return AUTH_SUCCESSFUL + return AUTH_FAILED + + def check_auth_gssapi_keyex( + self, username, gss_authenticated=AUTH_FAILED, cc_file=None + ): + """ + Authenticate the given user to the server if he is a valid krb5 + principal and GSS-API Key Exchange was performed. + If GSS-API Key Exchange was not performed, this authentication method + won't be available. + + :param str username: The username of the authenticating client + :param int gss_authenticated: The result of the krb5 authentication + :param str cc_filename: The krb5 client credentials cache filename + :return: ``AUTH_FAILED`` if the user is not authenticated otherwise + ``AUTH_SUCCESSFUL`` + :rtype: int + :note: Kerberos credential delegation is not supported. + :see: `.ssh_gss` `.kex_gss` + :note: : We are just checking in L{AuthHandler} that the given user is + a valid krb5 principal! + We don't check if the krb5 principal is allowed to log in on + the server, because there is no way to do that in python. So + if you develop your own SSH server with paramiko for a certain + platform like Linux, you should call C{krb5_kuserok()} in + your local kerberos library to make sure that the + krb5_principal has an account on the server and is allowed + to log in as a user. + :see: http://www.unix.com/man-page/all/3/krb5_kuserok/ + """ + if gss_authenticated == AUTH_SUCCESSFUL: + return AUTH_SUCCESSFUL + return AUTH_FAILED + + def enable_auth_gssapi(self): + """ + Overwrite this function in your SSH server to enable GSSAPI + authentication. + The default implementation always returns false. + + :returns bool: Whether GSSAPI authentication is enabled. + :see: `.ssh_gss` + """ + UseGSSAPI = False + return UseGSSAPI + + def check_port_forward_request(self, address, port): + """ + Handle a request for port forwarding. The client is asking that + connections to the given address and port be forwarded back across + this ssh connection. An address of ``"0.0.0.0"`` indicates a global + address (any address associated with this server) and a port of ``0`` + indicates that no specific port is requested (usually the OS will pick + a port). + + The default implementation always returns ``False``, rejecting the + port forwarding request. If the request is accepted, you should return + the port opened for listening. + + :param str address: the requested address + :param int port: the requested port + :return: + the port number (`int`) that was opened for listening, or ``False`` + to reject + """ + return False + + def cancel_port_forward_request(self, address, port): + """ + The client would like to cancel a previous port-forwarding request. + If the given address and port is being forwarded across this ssh + connection, the port should be closed. + + :param str address: the forwarded address + :param int port: the forwarded port + """ + pass + + def check_global_request(self, kind, msg): + """ + Handle a global request of the given ``kind``. This method is called + in server mode and client mode, whenever the remote host makes a global + request. If there are any arguments to the request, they will be in + ``msg``. + + There aren't any useful global requests defined, aside from port + forwarding, so usually this type of request is an extension to the + protocol. + + If the request was successful and you would like to return contextual + data to the remote host, return a tuple. Items in the tuple will be + sent back with the successful result. (Note that the items in the + tuple can only be strings, ints, or bools.) + + The default implementation always returns ``False``, indicating that it + does not support any global requests. + + .. note:: Port forwarding requests are handled separately, in + `check_port_forward_request`. + + :param str kind: the kind of global request being made. + :param .Message msg: any extra arguments to the request. + :return: + ``True`` or a `tuple` of data if the request was granted; ``False`` + otherwise. + """ + return False + + # ...Channel requests... + + def check_channel_pty_request( + self, channel, term, width, height, pixelwidth, pixelheight, modes + ): + """ + Determine if a pseudo-terminal of the given dimensions (usually + requested for shell access) can be provided on the given channel. + + The default implementation always returns ``False``. + + :param .Channel channel: the `.Channel` the pty request arrived on. + :param str term: type of terminal requested (for example, ``"vt100"``). + :param int width: width of screen in characters. + :param int height: height of screen in characters. + :param int pixelwidth: + width of screen in pixels, if known (may be ``0`` if unknown). + :param int pixelheight: + height of screen in pixels, if known (may be ``0`` if unknown). + :return: + ``True`` if the pseudo-terminal has been allocated; ``False`` + otherwise. + """ + return False + + def check_channel_shell_request(self, channel): + """ + Determine if a shell will be provided to the client on the given + channel. If this method returns ``True``, the channel should be + connected to the stdin/stdout of a shell (or something that acts like + a shell). + + The default implementation always returns ``False``. + + :param .Channel channel: the `.Channel` the request arrived on. + :return: + ``True`` if this channel is now hooked up to a shell; ``False`` if + a shell can't or won't be provided. + """ + return False + + def check_channel_exec_request(self, channel, command): + """ + Determine if a shell command will be executed for the client. If this + method returns ``True``, the channel should be connected to the stdin, + stdout, and stderr of the shell command. + + The default implementation always returns ``False``. + + :param .Channel channel: the `.Channel` the request arrived on. + :param str command: the command to execute. + :return: + ``True`` if this channel is now hooked up to the stdin, stdout, and + stderr of the executing command; ``False`` if the command will not + be executed. + + .. versionadded:: 1.1 + """ + return False + + def check_channel_subsystem_request(self, channel, name): + """ + Determine if a requested subsystem will be provided to the client on + the given channel. If this method returns ``True``, all future I/O + through this channel will be assumed to be connected to the requested + subsystem. An example of a subsystem is ``sftp``. + + The default implementation checks for a subsystem handler assigned via + `.Transport.set_subsystem_handler`. + If one has been set, the handler is invoked and this method returns + ``True``. Otherwise it returns ``False``. + + .. note:: Because the default implementation uses the `.Transport` to + identify valid subsystems, you probably won't need to override this + method. + + :param .Channel channel: the `.Channel` the pty request arrived on. + :param str name: name of the requested subsystem. + :return: + ``True`` if this channel is now hooked up to the requested + subsystem; ``False`` if that subsystem can't or won't be provided. + """ + transport = channel.get_transport() + handler_class, args, kwargs = transport._get_subsystem_handler(name) + if handler_class is None: + return False + handler = handler_class(channel, name, self, *args, **kwargs) + handler.start() + return True + + def check_channel_window_change_request( + self, channel, width, height, pixelwidth, pixelheight + ): + """ + Determine if the pseudo-terminal on the given channel can be resized. + This only makes sense if a pty was previously allocated on it. + + The default implementation always returns ``False``. + + :param .Channel channel: the `.Channel` the pty request arrived on. + :param int width: width of screen in characters. + :param int height: height of screen in characters. + :param int pixelwidth: + width of screen in pixels, if known (may be ``0`` if unknown). + :param int pixelheight: + height of screen in pixels, if known (may be ``0`` if unknown). + :return: ``True`` if the terminal was resized; ``False`` if not. + """ + return False + + def check_channel_x11_request( + self, + channel, + single_connection, + auth_protocol, + auth_cookie, + screen_number, + ): + """ + Determine if the client will be provided with an X11 session. If this + method returns ``True``, X11 applications should be routed through new + SSH channels, using `.Transport.open_x11_channel`. + + The default implementation always returns ``False``. + + :param .Channel channel: the `.Channel` the X11 request arrived on + :param bool single_connection: + ``True`` if only a single X11 channel should be opened, else + ``False``. + :param str auth_protocol: the protocol used for X11 authentication + :param str auth_cookie: the cookie used to authenticate to X11 + :param int screen_number: the number of the X11 screen to connect to + :return: ``True`` if the X11 session was opened; ``False`` if not + """ + return False + + def check_channel_forward_agent_request(self, channel): + """ + Determine if the client will be provided with an forward agent session. + If this method returns ``True``, the server will allow SSH Agent + forwarding. + + The default implementation always returns ``False``. + + :param .Channel channel: the `.Channel` the request arrived on + :return: ``True`` if the AgentForward was loaded; ``False`` if not + + If ``True`` is returned, the server should create an + :class:`AgentServerProxy` to access the agent. + """ + return False + + def check_channel_direct_tcpip_request(self, chanid, origin, destination): + """ + Determine if a local port forwarding channel will be granted, and + return ``OPEN_SUCCEEDED`` or an error code. This method is + called in server mode when the client requests a channel, after + authentication is complete. + + The ``chanid`` parameter is a small number that uniquely identifies the + channel within a `.Transport`. A `.Channel` object is not created + unless this method returns ``OPEN_SUCCEEDED`` -- once a + `.Channel` object is created, you can call `.Channel.get_id` to + retrieve the channel ID. + + The origin and destination parameters are (ip_address, port) tuples + that correspond to both ends of the TCP connection in the forwarding + tunnel. + + The return value should either be ``OPEN_SUCCEEDED`` (or + ``0``) to allow the channel request, or one of the following error + codes to reject it: + + - ``OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED`` + - ``OPEN_FAILED_CONNECT_FAILED`` + - ``OPEN_FAILED_UNKNOWN_CHANNEL_TYPE`` + - ``OPEN_FAILED_RESOURCE_SHORTAGE`` + + The default implementation always returns + ``OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED``. + + :param int chanid: ID of the channel + :param tuple origin: + 2-tuple containing the IP address and port of the originator + (client side) + :param tuple destination: + 2-tuple containing the IP address and port of the destination + (server side) + :return: an `int` success or failure code (listed above) + """ + return OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_channel_env_request(self, channel, name, value): + """ + Check whether a given environment variable can be specified for the + given channel. This method should return ``True`` if the server + is willing to set the specified environment variable. Note that + some environment variables (e.g., PATH) can be exceedingly + dangerous, so blindly allowing the client to set the environment + is almost certainly not a good idea. + + The default implementation always returns ``False``. + + :param channel: the `.Channel` the env request arrived on + :param str name: name + :param str value: Channel value + :returns: A boolean + """ + return False + + def get_banner(self): + """ + A pre-login banner to display to the user. The message may span + multiple lines separated by crlf pairs. The language should be in + rfc3066 style, for example: en-US + + The default implementation always returns ``(None, None)``. + + :returns: A tuple containing the banner and language code. + + .. versionadded:: 2.3 + """ + return (None, None) + + +class InteractiveQuery: + """ + A query (set of prompts) for a user during interactive authentication. + """ + + def __init__(self, name="", instructions="", *prompts): + """ + Create a new interactive query to send to the client. The name and + instructions are optional, but are generally displayed to the end + user. A list of prompts may be included, or they may be added via + the `add_prompt` method. + + :param str name: name of this query + :param str instructions: + user instructions (usually short) about this query + :param str prompts: one or more authentication prompts + """ + self.name = name + self.instructions = instructions + self.prompts = [] + for x in prompts: + if isinstance(x, str): + self.add_prompt(x) + else: + self.add_prompt(x[0], x[1]) + + def add_prompt(self, prompt, echo=True): + """ + Add a prompt to this query. The prompt should be a (reasonably short) + string. Multiple prompts can be added to the same query. + + :param str prompt: the user prompt + :param bool echo: + ``True`` (default) if the user's response should be echoed; + ``False`` if not (for a password or similar) + """ + self.prompts.append((prompt, echo)) + + +class SubsystemHandler(threading.Thread): + """ + Handler for a subsystem in server mode. If you create a subclass of this + class and pass it to `.Transport.set_subsystem_handler`, an object of this + class will be created for each request for this subsystem. Each new object + will be executed within its own new thread by calling `start_subsystem`. + When that method completes, the channel is closed. + + For example, if you made a subclass ``MP3Handler`` and registered it as the + handler for subsystem ``"mp3"``, then whenever a client has successfully + authenticated and requests subsystem ``"mp3"``, an object of class + ``MP3Handler`` will be created, and `start_subsystem` will be called on + it from a new thread. + """ + + def __init__(self, channel, name, server): + """ + Create a new handler for a channel. This is used by `.ServerInterface` + to start up a new handler when a channel requests this subsystem. You + don't need to override this method, but if you do, be sure to pass the + ``channel`` and ``name`` parameters through to the original + ``__init__`` method here. + + :param .Channel channel: the channel associated with this + subsystem request. + :param str name: name of the requested subsystem. + :param .ServerInterface server: + the server object for the session that started this subsystem + """ + threading.Thread.__init__(self, target=self._run) + self.__channel = channel + self.__transport = channel.get_transport() + self.__name = name + self.__server = server + + def get_server(self): + """ + Return the `.ServerInterface` object associated with this channel and + subsystem. + """ + return self.__server + + def _run(self): + try: + self.__transport._log( + DEBUG, "Starting handler for subsystem {}".format(self.__name) + ) + self.start_subsystem(self.__name, self.__transport, self.__channel) + except Exception as e: + self.__transport._log( + ERROR, + 'Exception in subsystem handler for "{}": {}'.format( + self.__name, e + ), + ) + self.__transport._log(ERROR, util.tb_strings()) + try: + self.finish_subsystem() + except: + pass + + def start_subsystem(self, name, transport, channel): + """ + Process an ssh subsystem in server mode. This method is called on a + new object (and in a new thread) for each subsystem request. It is + assumed that all subsystem logic will take place here, and when the + subsystem is finished, this method will return. After this method + returns, the channel is closed. + + The combination of ``transport`` and ``channel`` are unique; this + handler corresponds to exactly one `.Channel` on one `.Transport`. + + .. note:: + It is the responsibility of this method to exit if the underlying + `.Transport` is closed. This can be done by checking + `.Transport.is_active` or noticing an EOF on the `.Channel`. If + this method loops forever without checking for this case, your + Python interpreter may refuse to exit because this thread will + still be running. + + :param str name: name of the requested subsystem. + :param .Transport transport: the server-mode `.Transport`. + :param .Channel channel: the channel associated with this subsystem + request. + """ + pass + + def finish_subsystem(self): + """ + Perform any cleanup at the end of a subsystem. The default + implementation just closes the channel. + + .. versionadded:: 1.1 + """ + self.__channel.close() diff --git a/lib/paramiko/sftp.py b/lib/paramiko/sftp.py new file mode 100644 index 0000000..b3528d4 --- /dev/null +++ b/lib/paramiko/sftp.py @@ -0,0 +1,224 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import select +import socket +import struct + +from paramiko import util +from paramiko.common import DEBUG, byte_chr, byte_ord +from paramiko.message import Message + + +( + CMD_INIT, + CMD_VERSION, + CMD_OPEN, + CMD_CLOSE, + CMD_READ, + CMD_WRITE, + CMD_LSTAT, + CMD_FSTAT, + CMD_SETSTAT, + CMD_FSETSTAT, + CMD_OPENDIR, + CMD_READDIR, + CMD_REMOVE, + CMD_MKDIR, + CMD_RMDIR, + CMD_REALPATH, + CMD_STAT, + CMD_RENAME, + CMD_READLINK, + CMD_SYMLINK, +) = range(1, 21) +(CMD_STATUS, CMD_HANDLE, CMD_DATA, CMD_NAME, CMD_ATTRS) = range(101, 106) +(CMD_EXTENDED, CMD_EXTENDED_REPLY) = range(200, 202) + +SFTP_OK = 0 +( + SFTP_EOF, + SFTP_NO_SUCH_FILE, + SFTP_PERMISSION_DENIED, + SFTP_FAILURE, + SFTP_BAD_MESSAGE, + SFTP_NO_CONNECTION, + SFTP_CONNECTION_LOST, + SFTP_OP_UNSUPPORTED, +) = range(1, 9) + +SFTP_DESC = [ + "Success", + "End of file", + "No such file", + "Permission denied", + "Failure", + "Bad message", + "No connection", + "Connection lost", + "Operation unsupported", +] + +SFTP_FLAG_READ = 0x1 +SFTP_FLAG_WRITE = 0x2 +SFTP_FLAG_APPEND = 0x4 +SFTP_FLAG_CREATE = 0x8 +SFTP_FLAG_TRUNC = 0x10 +SFTP_FLAG_EXCL = 0x20 + +_VERSION = 3 + + +# for debugging +CMD_NAMES = { + CMD_INIT: "init", + CMD_VERSION: "version", + CMD_OPEN: "open", + CMD_CLOSE: "close", + CMD_READ: "read", + CMD_WRITE: "write", + CMD_LSTAT: "lstat", + CMD_FSTAT: "fstat", + CMD_SETSTAT: "setstat", + CMD_FSETSTAT: "fsetstat", + CMD_OPENDIR: "opendir", + CMD_READDIR: "readdir", + CMD_REMOVE: "remove", + CMD_MKDIR: "mkdir", + CMD_RMDIR: "rmdir", + CMD_REALPATH: "realpath", + CMD_STAT: "stat", + CMD_RENAME: "rename", + CMD_READLINK: "readlink", + CMD_SYMLINK: "symlink", + CMD_STATUS: "status", + CMD_HANDLE: "handle", + CMD_DATA: "data", + CMD_NAME: "name", + CMD_ATTRS: "attrs", + CMD_EXTENDED: "extended", + CMD_EXTENDED_REPLY: "extended_reply", +} + + +# TODO: rewrite SFTP file/server modules' overly-flexible "make a request with +# xyz components" so we don't need this very silly method of signaling whether +# a given Python integer should be 32- or 64-bit. +# NOTE: this only became an issue when dropping Python 2 support; prior to +# doing so, we had to support actual-longs, which served as that signal. This +# is simply recreating that structure in a more tightly scoped fashion. +class int64(int): + pass + + +class SFTPError(Exception): + pass + + +class BaseSFTP: + def __init__(self): + self.logger = util.get_logger("paramiko.sftp") + self.sock = None + self.ultra_debug = False + + # ...internals... + + def _send_version(self): + m = Message() + m.add_int(_VERSION) + self._send_packet(CMD_INIT, m) + t, data = self._read_packet() + if t != CMD_VERSION: + raise SFTPError("Incompatible sftp protocol") + version = struct.unpack(">I", data[:4])[0] + # if version != _VERSION: + # raise SFTPError('Incompatible sftp protocol') + return version + + def _send_server_version(self): + # winscp will freak out if the server sends version info before the + # client finishes sending INIT. + t, data = self._read_packet() + if t != CMD_INIT: + raise SFTPError("Incompatible sftp protocol") + version = struct.unpack(">I", data[:4])[0] + # advertise that we support "check-file" + extension_pairs = ["check-file", "md5,sha1"] + msg = Message() + msg.add_int(_VERSION) + msg.add(*extension_pairs) + self._send_packet(CMD_VERSION, msg) + return version + + def _log(self, level, msg, *args): + self.logger.log(level, msg, *args) + + def _write_all(self, out): + while len(out) > 0: + n = self.sock.send(out) + if n <= 0: + raise EOFError() + if n == len(out): + return + out = out[n:] + return + + def _read_all(self, n): + out = bytes() + while n > 0: + if isinstance(self.sock, socket.socket): + # sometimes sftp is used directly over a socket instead of + # through a paramiko channel. in this case, check periodically + # if the socket is closed. (for some reason, recv() won't ever + # return or raise an exception, but calling select on a closed + # socket will.) + while True: + read, write, err = select.select([self.sock], [], [], 0.1) + if len(read) > 0: + x = self.sock.recv(n) + break + else: + x = self.sock.recv(n) + + if len(x) == 0: + raise EOFError() + out += x + n -= len(x) + return out + + def _send_packet(self, t, packet): + packet = packet.asbytes() + out = struct.pack(">I", len(packet) + 1) + byte_chr(t) + packet + if self.ultra_debug: + self._log(DEBUG, util.format_binary(out, "OUT: ")) + self._write_all(out) + + def _read_packet(self): + x = self._read_all(4) + # most sftp servers won't accept packets larger than about 32k, so + # anything with the high byte set (> 16MB) is just garbage. + if byte_ord(x[0]): + raise SFTPError("Garbage packet received") + size = struct.unpack(">I", x)[0] + data = self._read_all(size) + if self.ultra_debug: + self._log(DEBUG, util.format_binary(data, "IN: ")) + if size > 0: + t = byte_ord(data[0]) + return t, data[1:] + return 0, bytes() diff --git a/lib/paramiko/sftp_attr.py b/lib/paramiko/sftp_attr.py new file mode 100644 index 0000000..18ffbf8 --- /dev/null +++ b/lib/paramiko/sftp_attr.py @@ -0,0 +1,239 @@ +# Copyright (C) 2003-2006 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import stat +import time +from paramiko.common import x80000000, o700, o70, xffffffff + + +class SFTPAttributes: + """ + Representation of the attributes of a file (or proxied file) for SFTP in + client or server mode. It attempts to mirror the object returned by + `os.stat` as closely as possible, so it may have the following fields, + with the same meanings as those returned by an `os.stat` object: + + - ``st_size`` + - ``st_uid`` + - ``st_gid`` + - ``st_mode`` + - ``st_atime`` + - ``st_mtime`` + + Because SFTP allows flags to have other arbitrary named attributes, these + are stored in a dict named ``attr``. Occasionally, the filename is also + stored, in ``filename``. + """ + + FLAG_SIZE = 1 + FLAG_UIDGID = 2 + FLAG_PERMISSIONS = 4 + FLAG_AMTIME = 8 + FLAG_EXTENDED = x80000000 + + def __init__(self): + """ + Create a new (empty) SFTPAttributes object. All fields will be empty. + """ + self._flags = 0 + self.st_size = None + self.st_uid = None + self.st_gid = None + self.st_mode = None + self.st_atime = None + self.st_mtime = None + self.attr = {} + + @classmethod + def from_stat(cls, obj, filename=None): + """ + Create an `.SFTPAttributes` object from an existing ``stat`` object (an + object returned by `os.stat`). + + :param object obj: an object returned by `os.stat` (or equivalent). + :param str filename: the filename associated with this file. + :return: new `.SFTPAttributes` object with the same attribute fields. + """ + attr = cls() + attr.st_size = obj.st_size + attr.st_uid = obj.st_uid + attr.st_gid = obj.st_gid + attr.st_mode = obj.st_mode + attr.st_atime = obj.st_atime + attr.st_mtime = obj.st_mtime + if filename is not None: + attr.filename = filename + return attr + + def __repr__(self): + return "".format(self._debug_str()) + + # ...internals... + @classmethod + def _from_msg(cls, msg, filename=None, longname=None): + attr = cls() + attr._unpack(msg) + if filename is not None: + attr.filename = filename + if longname is not None: + attr.longname = longname + return attr + + def _unpack(self, msg): + self._flags = msg.get_int() + if self._flags & self.FLAG_SIZE: + self.st_size = msg.get_int64() + if self._flags & self.FLAG_UIDGID: + self.st_uid = msg.get_int() + self.st_gid = msg.get_int() + if self._flags & self.FLAG_PERMISSIONS: + self.st_mode = msg.get_int() + if self._flags & self.FLAG_AMTIME: + self.st_atime = msg.get_int() + self.st_mtime = msg.get_int() + if self._flags & self.FLAG_EXTENDED: + count = msg.get_int() + for i in range(count): + self.attr[msg.get_string()] = msg.get_string() + + def _pack(self, msg): + self._flags = 0 + if self.st_size is not None: + self._flags |= self.FLAG_SIZE + if (self.st_uid is not None) and (self.st_gid is not None): + self._flags |= self.FLAG_UIDGID + if self.st_mode is not None: + self._flags |= self.FLAG_PERMISSIONS + if (self.st_atime is not None) and (self.st_mtime is not None): + self._flags |= self.FLAG_AMTIME + if len(self.attr) > 0: + self._flags |= self.FLAG_EXTENDED + msg.add_int(self._flags) + if self._flags & self.FLAG_SIZE: + msg.add_int64(self.st_size) + if self._flags & self.FLAG_UIDGID: + msg.add_int(self.st_uid) + msg.add_int(self.st_gid) + if self._flags & self.FLAG_PERMISSIONS: + msg.add_int(self.st_mode) + if self._flags & self.FLAG_AMTIME: + # throw away any fractional seconds + msg.add_int(int(self.st_atime)) + msg.add_int(int(self.st_mtime)) + if self._flags & self.FLAG_EXTENDED: + msg.add_int(len(self.attr)) + for key, val in self.attr.items(): + msg.add_string(key) + msg.add_string(val) + return + + def _debug_str(self): + out = "[ " + if self.st_size is not None: + out += "size={} ".format(self.st_size) + if (self.st_uid is not None) and (self.st_gid is not None): + out += "uid={} gid={} ".format(self.st_uid, self.st_gid) + if self.st_mode is not None: + out += "mode=" + oct(self.st_mode) + " " + if (self.st_atime is not None) and (self.st_mtime is not None): + out += "atime={} mtime={} ".format(self.st_atime, self.st_mtime) + for k, v in self.attr.items(): + out += '"{}"={!r} '.format(str(k), v) + out += "]" + return out + + @staticmethod + def _rwx(n, suid, sticky=False): + if suid: + suid = 2 + out = "-r"[n >> 2] + "-w"[(n >> 1) & 1] + if sticky: + out += "-xTt"[suid + (n & 1)] + else: + out += "-xSs"[suid + (n & 1)] + return out + + def __str__(self): + """create a unix-style long description of the file (like ls -l)""" + if self.st_mode is not None: + kind = stat.S_IFMT(self.st_mode) + if kind == stat.S_IFIFO: + ks = "p" + elif kind == stat.S_IFCHR: + ks = "c" + elif kind == stat.S_IFDIR: + ks = "d" + elif kind == stat.S_IFBLK: + ks = "b" + elif kind == stat.S_IFREG: + ks = "-" + elif kind == stat.S_IFLNK: + ks = "l" + elif kind == stat.S_IFSOCK: + ks = "s" + else: + ks = "?" + ks += self._rwx( + (self.st_mode & o700) >> 6, self.st_mode & stat.S_ISUID + ) + ks += self._rwx( + (self.st_mode & o70) >> 3, self.st_mode & stat.S_ISGID + ) + ks += self._rwx( + self.st_mode & 7, self.st_mode & stat.S_ISVTX, True + ) + else: + ks = "?---------" + # compute display date + if (self.st_mtime is None) or (self.st_mtime == xffffffff): + # shouldn't really happen + datestr = "(unknown date)" + else: + time_tuple = time.localtime(self.st_mtime) + if abs(time.time() - self.st_mtime) > 15_552_000: + # (15,552,000s = 6 months) + datestr = time.strftime("%d %b %Y", time_tuple) + else: + datestr = time.strftime("%d %b %H:%M", time_tuple) + filename = getattr(self, "filename", "?") + + # not all servers support uid/gid + uid = self.st_uid + gid = self.st_gid + size = self.st_size + if uid is None: + uid = 0 + if gid is None: + gid = 0 + if size is None: + size = 0 + + # TODO: not sure this actually worked as expected beforehand, leaving + # it untouched for the time being, re: .format() upgrade, until someone + # has time to doublecheck + return "%s 1 %-8d %-8d %8d %-12s %s" % ( + ks, + uid, + gid, + size, + datestr, + filename, + ) + + def asbytes(self): + return str(self).encode() diff --git a/lib/paramiko/sftp_client.py b/lib/paramiko/sftp_client.py new file mode 100644 index 0000000..066cd83 --- /dev/null +++ b/lib/paramiko/sftp_client.py @@ -0,0 +1,965 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of Paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from binascii import hexlify +import errno +import os +import stat +import threading +import time +import weakref +from paramiko import util +from paramiko.channel import Channel +from paramiko.message import Message +from paramiko.common import INFO, DEBUG, o777 +from paramiko.sftp import ( + BaseSFTP, + CMD_OPENDIR, + CMD_HANDLE, + SFTPError, + CMD_READDIR, + CMD_NAME, + CMD_CLOSE, + SFTP_FLAG_READ, + SFTP_FLAG_WRITE, + SFTP_FLAG_CREATE, + SFTP_FLAG_TRUNC, + SFTP_FLAG_APPEND, + SFTP_FLAG_EXCL, + CMD_OPEN, + CMD_REMOVE, + CMD_RENAME, + CMD_MKDIR, + CMD_RMDIR, + CMD_STAT, + CMD_ATTRS, + CMD_LSTAT, + CMD_SYMLINK, + CMD_SETSTAT, + CMD_READLINK, + CMD_REALPATH, + CMD_STATUS, + CMD_EXTENDED, + SFTP_OK, + SFTP_EOF, + SFTP_NO_SUCH_FILE, + SFTP_PERMISSION_DENIED, + int64, +) + +from paramiko.sftp_attr import SFTPAttributes +from paramiko.ssh_exception import SSHException +from paramiko.sftp_file import SFTPFile +from paramiko.util import ClosingContextManager, b, u + + +def _to_unicode(s): + """ + decode a string as ascii or utf8 if possible (as required by the sftp + protocol). if neither works, just return a byte string because the server + probably doesn't know the filename's encoding. + """ + try: + return s.encode("ascii") + except (UnicodeError, AttributeError): + try: + return s.decode("utf-8") + except UnicodeError: + return s + + +b_slash = b"/" + + +class SFTPClient(BaseSFTP, ClosingContextManager): + """ + SFTP client object. + + Used to open an SFTP session across an open SSH `.Transport` and perform + remote file operations. + + Instances of this class may be used as context managers. + """ + + def __init__(self, sock): + """ + Create an SFTP client from an existing `.Channel`. The channel + should already have requested the ``"sftp"`` subsystem. + + An alternate way to create an SFTP client context is by using + `from_transport`. + + :param .Channel sock: an open `.Channel` using the ``"sftp"`` subsystem + + :raises: + `.SSHException` -- if there's an exception while negotiating sftp + """ + BaseSFTP.__init__(self) + self.sock = sock + self.ultra_debug = False + self.request_number = 1 + # lock for request_number + self._lock = threading.Lock() + self._cwd = None + # request # -> SFTPFile + self._expecting = weakref.WeakValueDictionary() + if type(sock) is Channel: + # override default logger + transport = self.sock.get_transport() + self.logger = util.get_logger( + transport.get_log_channel() + ".sftp" + ) + self.ultra_debug = transport.get_hexdump() + try: + server_version = self._send_version() + except EOFError: + raise SSHException("EOF during negotiation") + self._log( + INFO, + "Opened sftp connection (server version {})".format( + server_version + ), + ) + + @classmethod + def from_transport(cls, t, window_size=None, max_packet_size=None): + """ + Create an SFTP client channel from an open `.Transport`. + + Setting the window and packet sizes might affect the transfer speed. + The default settings in the `.Transport` class are the same as in + OpenSSH and should work adequately for both files transfers and + interactive sessions. + + :param .Transport t: an open `.Transport` which is already + authenticated + :param int window_size: + optional window size for the `.SFTPClient` session. + :param int max_packet_size: + optional max packet size for the `.SFTPClient` session.. + + :return: + a new `.SFTPClient` object, referring to an sftp session (channel) + across the transport + + .. versionchanged:: 1.15 + Added the ``window_size`` and ``max_packet_size`` arguments. + """ + chan = t.open_session( + window_size=window_size, max_packet_size=max_packet_size + ) + if chan is None: + return None + chan.invoke_subsystem("sftp") + return cls(chan) + + def _log(self, level, msg, *args): + if isinstance(msg, list): + for m in msg: + self._log(level, m, *args) + else: + # NOTE: these bits MUST continue using %-style format junk because + # logging.Logger.log() explicitly requires it. Grump. + # escape '%' in msg (they could come from file or directory names) + # before logging + msg = msg.replace("%", "%%") + super()._log( + level, + "[chan %s] " + msg, + *([self.sock.get_name()] + list(args)) + ) + + def close(self): + """ + Close the SFTP session and its underlying channel. + + .. versionadded:: 1.4 + """ + self._log(INFO, "sftp session closed.") + self.sock.close() + + def get_channel(self): + """ + Return the underlying `.Channel` object for this SFTP session. This + might be useful for doing things like setting a timeout on the channel. + + .. versionadded:: 1.7.1 + """ + return self.sock + + def listdir(self, path="."): + """ + Return a list containing the names of the entries in the given + ``path``. + + The list is in arbitrary order. It does not include the special + entries ``'.'`` and ``'..'`` even if they are present in the folder. + This method is meant to mirror ``os.listdir`` as closely as possible. + For a list of full `.SFTPAttributes` objects, see `listdir_attr`. + + :param str path: path to list (defaults to ``'.'``) + """ + return [f.filename for f in self.listdir_attr(path)] + + def listdir_attr(self, path="."): + """ + Return a list containing `.SFTPAttributes` objects corresponding to + files in the given ``path``. The list is in arbitrary order. It does + not include the special entries ``'.'`` and ``'..'`` even if they are + present in the folder. + + The returned `.SFTPAttributes` objects will each have an additional + field: ``longname``, which may contain a formatted string of the file's + attributes, in unix format. The content of this string will probably + depend on the SFTP server implementation. + + :param str path: path to list (defaults to ``'.'``) + :return: list of `.SFTPAttributes` objects + + .. versionadded:: 1.2 + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "listdir({!r})".format(path)) + t, msg = self._request(CMD_OPENDIR, path) + if t != CMD_HANDLE: + raise SFTPError("Expected handle") + handle = msg.get_binary() + filelist = [] + while True: + try: + t, msg = self._request(CMD_READDIR, handle) + except EOFError: + # done with handle + break + if t != CMD_NAME: + raise SFTPError("Expected name response") + count = msg.get_int() + for i in range(count): + filename = msg.get_text() + longname = msg.get_text() + attr = SFTPAttributes._from_msg(msg, filename, longname) + if (filename != ".") and (filename != ".."): + filelist.append(attr) + self._request(CMD_CLOSE, handle) + return filelist + + def listdir_iter(self, path=".", read_aheads=50): + """ + Generator version of `.listdir_attr`. + + See the API docs for `.listdir_attr` for overall details. + + This function adds one more kwarg on top of `.listdir_attr`: + ``read_aheads``, an integer controlling how many + ``SSH_FXP_READDIR`` requests are made to the server. The default of 50 + should suffice for most file listings as each request/response cycle + may contain multiple files (dependent on server implementation.) + + .. versionadded:: 1.15 + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "listdir({!r})".format(path)) + t, msg = self._request(CMD_OPENDIR, path) + + if t != CMD_HANDLE: + raise SFTPError("Expected handle") + + handle = msg.get_string() + + nums = list() + while True: + try: + # Send out a bunch of readdir requests so that we can read the + # responses later on Section 6.7 of the SSH file transfer RFC + # explains this + # http://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt + for i in range(read_aheads): + num = self._async_request(type(None), CMD_READDIR, handle) + nums.append(num) + + # For each of our sent requests + # Read and parse the corresponding packets + # If we're at the end of our queued requests, then fire off + # some more requests + # Exit the loop when we've reached the end of the directory + # handle + for num in nums: + t, pkt_data = self._read_packet() + msg = Message(pkt_data) + new_num = msg.get_int() + if num == new_num: + if t == CMD_STATUS: + self._convert_status(msg) + count = msg.get_int() + for i in range(count): + filename = msg.get_text() + longname = msg.get_text() + attr = SFTPAttributes._from_msg( + msg, filename, longname + ) + if (filename != ".") and (filename != ".."): + yield attr + + # If we've hit the end of our queued requests, reset nums. + nums = list() + + except EOFError: + self._request(CMD_CLOSE, handle) + return + + def open(self, filename, mode="r", bufsize=-1): + """ + Open a file on the remote server. The arguments are the same as for + Python's built-in `python:file` (aka `python:open`). A file-like + object is returned, which closely mimics the behavior of a normal + Python file object, including the ability to be used as a context + manager. + + The mode indicates how the file is to be opened: ``'r'`` for reading, + ``'w'`` for writing (truncating an existing file), ``'a'`` for + appending, ``'r+'`` for reading/writing, ``'w+'`` for reading/writing + (truncating an existing file), ``'a+'`` for reading/appending. The + Python ``'b'`` flag is ignored, since SSH treats all files as binary. + The ``'U'`` flag is supported in a compatible way. + + Since 1.5.2, an ``'x'`` flag indicates that the operation should only + succeed if the file was created and did not previously exist. This has + no direct mapping to Python's file flags, but is commonly known as the + ``O_EXCL`` flag in posix. + + The file will be buffered in standard Python style by default, but + can be altered with the ``bufsize`` parameter. ``<=0`` turns off + buffering, ``1`` uses line buffering, and any number greater than 1 + (``>1``) uses that specific buffer size. + + :param str filename: name of the file to open + :param str mode: mode (Python-style) to open in + :param int bufsize: desired buffering (default: ``-1``) + :return: an `.SFTPFile` object representing the open file + + :raises: ``IOError`` -- if the file could not be opened. + """ + filename = self._adjust_cwd(filename) + self._log(DEBUG, "open({!r}, {!r})".format(filename, mode)) + imode = 0 + if ("r" in mode) or ("+" in mode): + imode |= SFTP_FLAG_READ + if ("w" in mode) or ("+" in mode) or ("a" in mode): + imode |= SFTP_FLAG_WRITE + if "w" in mode: + imode |= SFTP_FLAG_CREATE | SFTP_FLAG_TRUNC + if "a" in mode: + imode |= SFTP_FLAG_CREATE | SFTP_FLAG_APPEND + if "x" in mode: + imode |= SFTP_FLAG_CREATE | SFTP_FLAG_EXCL + attrblock = SFTPAttributes() + t, msg = self._request(CMD_OPEN, filename, imode, attrblock) + if t != CMD_HANDLE: + raise SFTPError("Expected handle") + handle = msg.get_binary() + self._log( + DEBUG, + "open({!r}, {!r}) -> {}".format( + filename, mode, u(hexlify(handle)) + ), + ) + return SFTPFile(self, handle, mode, bufsize) + + # Python continues to vacillate about "open" vs "file"... + file = open + + def remove(self, path): + """ + Remove the file at the given path. This only works on files; for + removing folders (directories), use `rmdir`. + + :param str path: path (absolute or relative) of the file to remove + + :raises: ``IOError`` -- if the path refers to a folder (directory) + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "remove({!r})".format(path)) + self._request(CMD_REMOVE, path) + + unlink = remove + + def rename(self, oldpath, newpath): + """ + Rename a file or folder from ``oldpath`` to ``newpath``. + + .. note:: + This method implements 'standard' SFTP ``RENAME`` behavior; those + seeking the OpenSSH "POSIX rename" extension behavior should use + `posix_rename`. + + :param str oldpath: + existing name of the file or folder + :param str newpath: + new name for the file or folder, must not exist already + + :raises: + ``IOError`` -- if ``newpath`` is a folder, or something else goes + wrong + """ + oldpath = self._adjust_cwd(oldpath) + newpath = self._adjust_cwd(newpath) + self._log(DEBUG, "rename({!r}, {!r})".format(oldpath, newpath)) + self._request(CMD_RENAME, oldpath, newpath) + + def posix_rename(self, oldpath, newpath): + """ + Rename a file or folder from ``oldpath`` to ``newpath``, following + posix conventions. + + :param str oldpath: existing name of the file or folder + :param str newpath: new name for the file or folder, will be + overwritten if it already exists + + :raises: + ``IOError`` -- if ``newpath`` is a folder, posix-rename is not + supported by the server or something else goes wrong + + :versionadded: 2.2 + """ + oldpath = self._adjust_cwd(oldpath) + newpath = self._adjust_cwd(newpath) + self._log(DEBUG, "posix_rename({!r}, {!r})".format(oldpath, newpath)) + self._request( + CMD_EXTENDED, "posix-rename@openssh.com", oldpath, newpath + ) + + def mkdir(self, path, mode=o777): + """ + Create a folder (directory) named ``path`` with numeric mode ``mode``. + The default mode is 0777 (octal). On some systems, mode is ignored. + Where it is used, the current umask value is first masked out. + + :param str path: name of the folder to create + :param int mode: permissions (posix-style) for the newly-created folder + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "mkdir({!r}, {!r})".format(path, mode)) + attr = SFTPAttributes() + attr.st_mode = mode + self._request(CMD_MKDIR, path, attr) + + def rmdir(self, path): + """ + Remove the folder named ``path``. + + :param str path: name of the folder to remove + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "rmdir({!r})".format(path)) + self._request(CMD_RMDIR, path) + + def stat(self, path): + """ + Retrieve information about a file on the remote system. The return + value is an object whose attributes correspond to the attributes of + Python's ``stat`` structure as returned by ``os.stat``, except that it + contains fewer fields. An SFTP server may return as much or as little + info as it wants, so the results may vary from server to server. + + Unlike a Python `python:stat` object, the result may not be accessed as + a tuple. This is mostly due to the author's slack factor. + + The fields supported are: ``st_mode``, ``st_size``, ``st_uid``, + ``st_gid``, ``st_atime``, and ``st_mtime``. + + :param str path: the filename to stat + :return: + an `.SFTPAttributes` object containing attributes about the given + file + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "stat({!r})".format(path)) + t, msg = self._request(CMD_STAT, path) + if t != CMD_ATTRS: + raise SFTPError("Expected attributes") + return SFTPAttributes._from_msg(msg) + + def lstat(self, path): + """ + Retrieve information about a file on the remote system, without + following symbolic links (shortcuts). This otherwise behaves exactly + the same as `stat`. + + :param str path: the filename to stat + :return: + an `.SFTPAttributes` object containing attributes about the given + file + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "lstat({!r})".format(path)) + t, msg = self._request(CMD_LSTAT, path) + if t != CMD_ATTRS: + raise SFTPError("Expected attributes") + return SFTPAttributes._from_msg(msg) + + def symlink(self, source, dest): + """ + Create a symbolic link to the ``source`` path at ``destination``. + + :param str source: path of the original file + :param str dest: path of the newly created symlink + """ + dest = self._adjust_cwd(dest) + self._log(DEBUG, "symlink({!r}, {!r})".format(source, dest)) + source = b(source) + self._request(CMD_SYMLINK, source, dest) + + def chmod(self, path, mode): + """ + Change the mode (permissions) of a file. The permissions are + unix-style and identical to those used by Python's `os.chmod` + function. + + :param str path: path of the file to change the permissions of + :param int mode: new permissions + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "chmod({!r}, {!r})".format(path, mode)) + attr = SFTPAttributes() + attr.st_mode = mode + self._request(CMD_SETSTAT, path, attr) + + def chown(self, path, uid, gid): + """ + Change the owner (``uid``) and group (``gid``) of a file. As with + Python's `os.chown` function, you must pass both arguments, so if you + only want to change one, use `stat` first to retrieve the current + owner and group. + + :param str path: path of the file to change the owner and group of + :param int uid: new owner's uid + :param int gid: new group id + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "chown({!r}, {!r}, {!r})".format(path, uid, gid)) + attr = SFTPAttributes() + attr.st_uid, attr.st_gid = uid, gid + self._request(CMD_SETSTAT, path, attr) + + def utime(self, path, times): + """ + Set the access and modified times of the file specified by ``path``. + If ``times`` is ``None``, then the file's access and modified times + are set to the current time. Otherwise, ``times`` must be a 2-tuple + of numbers, of the form ``(atime, mtime)``, which is used to set the + access and modified times, respectively. This bizarre API is mimicked + from Python for the sake of consistency -- I apologize. + + :param str path: path of the file to modify + :param tuple times: + ``None`` or a tuple of (access time, modified time) in standard + internet epoch time (seconds since 01 January 1970 GMT) + """ + path = self._adjust_cwd(path) + if times is None: + times = (time.time(), time.time()) + self._log(DEBUG, "utime({!r}, {!r})".format(path, times)) + attr = SFTPAttributes() + attr.st_atime, attr.st_mtime = times + self._request(CMD_SETSTAT, path, attr) + + def truncate(self, path, size): + """ + Change the size of the file specified by ``path``. This usually + extends or shrinks the size of the file, just like the `~file.truncate` + method on Python file objects. + + :param str path: path of the file to modify + :param int size: the new size of the file + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "truncate({!r}, {!r})".format(path, size)) + attr = SFTPAttributes() + attr.st_size = size + self._request(CMD_SETSTAT, path, attr) + + def readlink(self, path): + """ + Return the target of a symbolic link (shortcut). You can use + `symlink` to create these. The result may be either an absolute or + relative pathname. + + :param str path: path of the symbolic link file + :return: target path, as a `str` + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "readlink({!r})".format(path)) + t, msg = self._request(CMD_READLINK, path) + if t != CMD_NAME: + raise SFTPError("Expected name response") + count = msg.get_int() + if count == 0: + return None + if count != 1: + raise SFTPError("Readlink returned {} results".format(count)) + return _to_unicode(msg.get_string()) + + def normalize(self, path): + """ + Return the normalized path (on the server) of a given path. This + can be used to quickly resolve symbolic links or determine what the + server is considering to be the "current folder" (by passing ``'.'`` + as ``path``). + + :param str path: path to be normalized + :return: normalized form of the given path (as a `str`) + + :raises: ``IOError`` -- if the path can't be resolved on the server + """ + path = self._adjust_cwd(path) + self._log(DEBUG, "normalize({!r})".format(path)) + t, msg = self._request(CMD_REALPATH, path) + if t != CMD_NAME: + raise SFTPError("Expected name response") + count = msg.get_int() + if count != 1: + raise SFTPError("Realpath returned {} results".format(count)) + return msg.get_text() + + def chdir(self, path=None): + """ + Change the "current directory" of this SFTP session. Since SFTP + doesn't really have the concept of a current working directory, this is + emulated by Paramiko. Once you use this method to set a working + directory, all operations on this `.SFTPClient` object will be relative + to that path. You can pass in ``None`` to stop using a current working + directory. + + :param str path: new current working directory + + :raises: + ``IOError`` -- if the requested path doesn't exist on the server + + .. versionadded:: 1.4 + """ + if path is None: + self._cwd = None + return + if not stat.S_ISDIR(self.stat(path).st_mode): + code = errno.ENOTDIR + raise SFTPError(code, "{}: {}".format(os.strerror(code), path)) + self._cwd = b(self.normalize(path)) + + def getcwd(self): + """ + Return the "current working directory" for this SFTP session, as + emulated by Paramiko. If no directory has been set with `chdir`, + this method will return ``None``. + + .. versionadded:: 1.4 + """ + # TODO: make class initialize with self._cwd set to self.normalize('.') + return self._cwd and u(self._cwd) + + def _transfer_with_callback(self, reader, writer, file_size, callback): + size = 0 + while True: + data = reader.read(32768) + writer.write(data) + size += len(data) + if len(data) == 0: + break + if callback is not None: + callback(size, file_size) + return size + + def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True): + """ + Copy the contents of an open file object (``fl``) to the SFTP server as + ``remotepath``. Any exception raised by operations will be passed + through. + + The SFTP operations use pipelining for speed. + + :param fl: opened file or file-like object to copy + :param str remotepath: the destination path on the SFTP server + :param int file_size: + optional size parameter passed to callback. If none is specified, + size defaults to 0 + :param callable callback: + optional callback function (form: ``func(int, int)``) that accepts + the bytes transferred so far and the total bytes to be transferred + (since 1.7.4) + :param bool confirm: + whether to do a stat() on the file afterwards to confirm the file + size (since 1.7.7) + + :return: + an `.SFTPAttributes` object containing attributes about the given + file. + + .. versionadded:: 1.10 + """ + with self.file(remotepath, "wb") as fr: + fr.set_pipelined(True) + size = self._transfer_with_callback( + reader=fl, writer=fr, file_size=file_size, callback=callback + ) + if confirm: + s = self.stat(remotepath) + if s.st_size != size: + raise IOError( + "size mismatch in put! {} != {}".format(s.st_size, size) + ) + else: + s = SFTPAttributes() + return s + + def put(self, localpath, remotepath, callback=None, confirm=True): + """ + Copy a local file (``localpath``) to the SFTP server as ``remotepath``. + Any exception raised by operations will be passed through. This + method is primarily provided as a convenience. + + The SFTP operations use pipelining for speed. + + :param str localpath: the local file to copy + :param str remotepath: the destination path on the SFTP server. Note + that the filename should be included. Only specifying a directory + may result in an error. + :param callable callback: + optional callback function (form: ``func(int, int)``) that accepts + the bytes transferred so far and the total bytes to be transferred + :param bool confirm: + whether to do a stat() on the file afterwards to confirm the file + size + + :return: an `.SFTPAttributes` object containing attributes about the + given file + + .. versionadded:: 1.4 + .. versionchanged:: 1.7.4 + ``callback`` and rich attribute return value added. + .. versionchanged:: 1.7.7 + ``confirm`` param added. + """ + file_size = os.stat(localpath).st_size + with open(localpath, "rb") as fl: + return self.putfo(fl, remotepath, file_size, callback, confirm) + + def getfo( + self, + remotepath, + fl, + callback=None, + prefetch=True, + max_concurrent_prefetch_requests=None, + ): + """ + Copy a remote file (``remotepath``) from the SFTP server and write to + an open file or file-like object, ``fl``. Any exception raised by + operations will be passed through. This method is primarily provided + as a convenience. + + :param object remotepath: opened file or file-like object to copy to + :param str fl: + the destination path on the local host or open file object + :param callable callback: + optional callback function (form: ``func(int, int)``) that accepts + the bytes transferred so far and the total bytes to be transferred + :param bool prefetch: + controls whether prefetching is performed (default: True) + :param int max_concurrent_prefetch_requests: + The maximum number of concurrent read requests to prefetch. See + `.SFTPClient.get` (its ``max_concurrent_prefetch_requests`` param) + for details. + :return: the `number ` of bytes written to the opened file object + + .. versionadded:: 1.10 + .. versionchanged:: 2.8 + Added the ``prefetch`` keyword argument. + .. versionchanged:: 3.3 + Added ``max_concurrent_prefetch_requests``. + """ + file_size = self.stat(remotepath).st_size + with self.open(remotepath, "rb") as fr: + if prefetch: + fr.prefetch(file_size, max_concurrent_prefetch_requests) + return self._transfer_with_callback( + reader=fr, writer=fl, file_size=file_size, callback=callback + ) + + def get( + self, + remotepath, + localpath, + callback=None, + prefetch=True, + max_concurrent_prefetch_requests=None, + ): + """ + Copy a remote file (``remotepath``) from the SFTP server to the local + host as ``localpath``. Any exception raised by operations will be + passed through. This method is primarily provided as a convenience. + + :param str remotepath: the remote file to copy + :param str localpath: the destination path on the local host + :param callable callback: + optional callback function (form: ``func(int, int)``) that accepts + the bytes transferred so far and the total bytes to be transferred + :param bool prefetch: + controls whether prefetching is performed (default: True) + :param int max_concurrent_prefetch_requests: + The maximum number of concurrent read requests to prefetch. + When this is ``None`` (the default), do not limit the number of + concurrent prefetch requests. Note: OpenSSH's sftp internally + imposes a limit of 64 concurrent requests, while Paramiko imposes + no limit by default; consider setting a limit if a file can be + successfully received with sftp but hangs with Paramiko. + + .. versionadded:: 1.4 + .. versionchanged:: 1.7.4 + Added the ``callback`` param + .. versionchanged:: 2.8 + Added the ``prefetch`` keyword argument. + .. versionchanged:: 3.3 + Added ``max_concurrent_prefetch_requests``. + """ + with open(localpath, "wb") as fl: + size = self.getfo( + remotepath, + fl, + callback, + prefetch, + max_concurrent_prefetch_requests, + ) + s = os.stat(localpath) + if s.st_size != size: + raise IOError( + "size mismatch in get! {} != {}".format(s.st_size, size) + ) + + # ...internals... + + def _request(self, t, *args): + num = self._async_request(type(None), t, *args) + return self._read_response(num) + + def _async_request(self, fileobj, t, *args): + # this method may be called from other threads (prefetch) + self._lock.acquire() + try: + msg = Message() + msg.add_int(self.request_number) + for item in args: + if isinstance(item, int64): + msg.add_int64(item) + elif isinstance(item, int): + msg.add_int(item) + elif isinstance(item, SFTPAttributes): + item._pack(msg) + else: + # For all other types, rely on as_string() to either coerce + # to bytes before writing or raise a suitable exception. + msg.add_string(item) + num = self.request_number + self._expecting[num] = fileobj + self.request_number += 1 + finally: + self._lock.release() + self._send_packet(t, msg) + return num + + def _read_response(self, waitfor=None): + while True: + try: + t, data = self._read_packet() + except EOFError as e: + raise SSHException("Server connection dropped: {}".format(e)) + msg = Message(data) + num = msg.get_int() + self._lock.acquire() + try: + if num not in self._expecting: + # might be response for a file that was closed before + # responses came back + self._log(DEBUG, "Unexpected response #{}".format(num)) + if waitfor is None: + # just doing a single check + break + continue + fileobj = self._expecting[num] + del self._expecting[num] + finally: + self._lock.release() + if num == waitfor: + # synchronous + if t == CMD_STATUS: + self._convert_status(msg) + return t, msg + + # can not rewrite this to deal with E721, either as a None check + # nor as not an instance of None or NoneType + if fileobj is not type(None): # noqa + fileobj._async_response(t, msg, num) + if waitfor is None: + # just doing a single check + break + return None, None + + def _finish_responses(self, fileobj): + while fileobj in self._expecting.values(): + self._read_response() + fileobj._check_exception() + + def _convert_status(self, msg): + """ + Raises EOFError or IOError on error status; otherwise does nothing. + """ + code = msg.get_int() + text = msg.get_text() + if code == SFTP_OK: + return + elif code == SFTP_EOF: + raise EOFError(text) + elif code == SFTP_NO_SUCH_FILE: + # clever idea from john a. meinel: map the error codes to errno + raise IOError(errno.ENOENT, text) + elif code == SFTP_PERMISSION_DENIED: + raise IOError(errno.EACCES, text) + else: + raise IOError(text) + + def _adjust_cwd(self, path): + """ + Return an adjusted path if we're emulating a "current working + directory" for the server. + """ + path = b(path) + if self._cwd is None: + return path + if len(path) and path[0:1] == b_slash: + # absolute path + return path + if self._cwd == b_slash: + return self._cwd + path + return self._cwd + b_slash + path + + +class SFTP(SFTPClient): + """ + An alias for `.SFTPClient` for backwards compatibility. + """ + + pass diff --git a/lib/paramiko/sftp_file.py b/lib/paramiko/sftp_file.py new file mode 100644 index 0000000..c74695e --- /dev/null +++ b/lib/paramiko/sftp_file.py @@ -0,0 +1,594 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +SFTP file object +""" + + +from binascii import hexlify +from collections import deque +import socket +import threading +import time +from paramiko.common import DEBUG, io_sleep + +from paramiko.file import BufferedFile +from paramiko.util import u +from paramiko.sftp import ( + CMD_CLOSE, + CMD_READ, + CMD_DATA, + SFTPError, + CMD_WRITE, + CMD_STATUS, + CMD_FSTAT, + CMD_ATTRS, + CMD_FSETSTAT, + CMD_EXTENDED, + int64, +) +from paramiko.sftp_attr import SFTPAttributes + + +class SFTPFile(BufferedFile): + """ + Proxy object for a file on the remote server, in client mode SFTP. + + Instances of this class may be used as context managers in the same way + that built-in Python file objects are. + """ + + # Some sftp servers will choke if you send read/write requests larger than + # this size. + MAX_REQUEST_SIZE = 32768 + + def __init__(self, sftp, handle, mode="r", bufsize=-1): + BufferedFile.__init__(self) + self.sftp = sftp + self.handle = handle + BufferedFile._set_mode(self, mode, bufsize) + self.pipelined = False + self._prefetching = False + self._prefetch_done = False + self._prefetch_data = {} + self._prefetch_extents = {} + self._prefetch_lock = threading.Lock() + self._saved_exception = None + self._reqs = deque() + + def __del__(self): + self._close(async_=True) + + def close(self): + """ + Close the file. + """ + self._close(async_=False) + + def _close(self, async_=False): + # We allow double-close without signaling an error, because real + # Python file objects do. However, we must protect against actually + # sending multiple CMD_CLOSE packets, because after we close our + # handle, the same handle may be re-allocated by the server, and we + # may end up mysteriously closing some random other file. (This is + # especially important because we unconditionally call close() from + # __del__.) + if self._closed: + return + self.sftp._log(DEBUG, "close({})".format(u(hexlify(self.handle)))) + if self.pipelined: + self.sftp._finish_responses(self) + BufferedFile.close(self) + try: + if async_: + # GC'd file handle could be called from an arbitrary thread + # -- don't wait for a response + self.sftp._async_request(type(None), CMD_CLOSE, self.handle) + else: + self.sftp._request(CMD_CLOSE, self.handle) + except EOFError: + # may have outlived the Transport connection + pass + except (IOError, socket.error): + # may have outlived the Transport connection + pass + + def _data_in_prefetch_requests(self, offset, size): + k = [ + x for x in list(self._prefetch_extents.values()) if x[0] <= offset + ] + if len(k) == 0: + return False + k.sort(key=lambda x: x[0]) + buf_offset, buf_size = k[-1] + if buf_offset + buf_size <= offset: + # prefetch request ends before this one begins + return False + if buf_offset + buf_size >= offset + size: + # inclusive + return True + # well, we have part of the request. see if another chunk has + # the rest. + return self._data_in_prefetch_requests( + buf_offset + buf_size, offset + size - buf_offset - buf_size + ) + + def _data_in_prefetch_buffers(self, offset): + """ + if a block of data is present in the prefetch buffers, at the given + offset, return the offset of the relevant prefetch buffer. otherwise, + return None. this guarantees nothing about the number of bytes + collected in the prefetch buffer so far. + """ + k = [i for i in self._prefetch_data.keys() if i <= offset] + if len(k) == 0: + return None + index = max(k) + buf_offset = offset - index + if buf_offset >= len(self._prefetch_data[index]): + # it's not here + return None + return index + + def _read_prefetch(self, size): + """ + read data out of the prefetch buffer, if possible. if the data isn't + in the buffer, return None. otherwise, behaves like a normal read. + """ + # while not closed, and haven't fetched past the current position, + # and haven't reached EOF... + while True: + offset = self._data_in_prefetch_buffers(self._realpos) + if offset is not None: + break + if self._prefetch_done or self._closed: + break + self.sftp._read_response() + self._check_exception() + if offset is None: + self._prefetching = False + return None + prefetch = self._prefetch_data[offset] + del self._prefetch_data[offset] + + buf_offset = self._realpos - offset + if buf_offset > 0: + self._prefetch_data[offset] = prefetch[:buf_offset] + prefetch = prefetch[buf_offset:] + if size < len(prefetch): + self._prefetch_data[self._realpos + size] = prefetch[size:] + prefetch = prefetch[:size] + return prefetch + + def _read(self, size): + size = min(size, self.MAX_REQUEST_SIZE) + if self._prefetching: + data = self._read_prefetch(size) + if data is not None: + return data + t, msg = self.sftp._request( + CMD_READ, self.handle, int64(self._realpos), int(size) + ) + if t != CMD_DATA: + raise SFTPError("Expected data") + return msg.get_string() + + def _write(self, data): + # may write less than requested if it would exceed max packet size + chunk = min(len(data), self.MAX_REQUEST_SIZE) + sftp_async_request = self.sftp._async_request( + type(None), + CMD_WRITE, + self.handle, + int64(self._realpos), + data[:chunk], + ) + self._reqs.append(sftp_async_request) + if not self.pipelined or ( + len(self._reqs) > 100 and self.sftp.sock.recv_ready() + ): + while len(self._reqs): + req = self._reqs.popleft() + t, msg = self.sftp._read_response(req) + if t != CMD_STATUS: + raise SFTPError("Expected status") + # convert_status already called + return chunk + + def settimeout(self, timeout): + """ + Set a timeout on read/write operations on the underlying socket or + ssh `.Channel`. + + :param float timeout: + seconds to wait for a pending read/write operation before raising + ``socket.timeout``, or ``None`` for no timeout + + .. seealso:: `.Channel.settimeout` + """ + self.sftp.sock.settimeout(timeout) + + def gettimeout(self): + """ + Returns the timeout in seconds (as a `float`) associated with the + socket or ssh `.Channel` used for this file. + + .. seealso:: `.Channel.gettimeout` + """ + return self.sftp.sock.gettimeout() + + def setblocking(self, blocking): + """ + Set blocking or non-blocking mode on the underiying socket or ssh + `.Channel`. + + :param int blocking: + 0 to set non-blocking mode; non-0 to set blocking mode. + + .. seealso:: `.Channel.setblocking` + """ + self.sftp.sock.setblocking(blocking) + + def seekable(self): + """ + Check if the file supports random access. + + :return: + `True` if the file supports random access. If `False`, + :meth:`seek` will raise an exception + """ + return True + + def seek(self, offset, whence=0): + """ + Set the file's current position. + + See `file.seek` for details. + """ + self.flush() + if whence == self.SEEK_SET: + self._realpos = self._pos = offset + elif whence == self.SEEK_CUR: + self._pos += offset + self._realpos = self._pos + else: + self._realpos = self._pos = self._get_size() + offset + self._rbuffer = bytes() + + def stat(self): + """ + Retrieve information about this file from the remote system. This is + exactly like `.SFTPClient.stat`, except that it operates on an + already-open file. + + :returns: + an `.SFTPAttributes` object containing attributes about this file. + """ + t, msg = self.sftp._request(CMD_FSTAT, self.handle) + if t != CMD_ATTRS: + raise SFTPError("Expected attributes") + return SFTPAttributes._from_msg(msg) + + def chmod(self, mode): + """ + Change the mode (permissions) of this file. The permissions are + unix-style and identical to those used by Python's `os.chmod` + function. + + :param int mode: new permissions + """ + self.sftp._log( + DEBUG, "chmod({}, {!r})".format(hexlify(self.handle), mode) + ) + attr = SFTPAttributes() + attr.st_mode = mode + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def chown(self, uid, gid): + """ + Change the owner (``uid``) and group (``gid``) of this file. As with + Python's `os.chown` function, you must pass both arguments, so if you + only want to change one, use `stat` first to retrieve the current + owner and group. + + :param int uid: new owner's uid + :param int gid: new group id + """ + self.sftp._log( + DEBUG, + "chown({}, {!r}, {!r})".format(hexlify(self.handle), uid, gid), + ) + attr = SFTPAttributes() + attr.st_uid, attr.st_gid = uid, gid + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def utime(self, times): + """ + Set the access and modified times of this file. If + ``times`` is ``None``, then the file's access and modified times are + set to the current time. Otherwise, ``times`` must be a 2-tuple of + numbers, of the form ``(atime, mtime)``, which is used to set the + access and modified times, respectively. This bizarre API is mimicked + from Python for the sake of consistency -- I apologize. + + :param tuple times: + ``None`` or a tuple of (access time, modified time) in standard + internet epoch time (seconds since 01 January 1970 GMT) + """ + if times is None: + times = (time.time(), time.time()) + self.sftp._log( + DEBUG, "utime({}, {!r})".format(hexlify(self.handle), times) + ) + attr = SFTPAttributes() + attr.st_atime, attr.st_mtime = times + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def truncate(self, size): + """ + Change the size of this file. This usually extends + or shrinks the size of the file, just like the ``truncate()`` method on + Python file objects. + + :param size: the new size of the file + """ + self.sftp._log( + DEBUG, "truncate({}, {!r})".format(hexlify(self.handle), size) + ) + attr = SFTPAttributes() + attr.st_size = size + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def check(self, hash_algorithm, offset=0, length=0, block_size=0): + """ + Ask the server for a hash of a section of this file. This can be used + to verify a successful upload or download, or for various rsync-like + operations. + + The file is hashed from ``offset``, for ``length`` bytes. + If ``length`` is 0, the remainder of the file is hashed. Thus, if both + ``offset`` and ``length`` are zero, the entire file is hashed. + + Normally, ``block_size`` will be 0 (the default), and this method will + return a byte string representing the requested hash (for example, a + string of length 16 for MD5, or 20 for SHA-1). If a non-zero + ``block_size`` is given, each chunk of the file (from ``offset`` to + ``offset + length``) of ``block_size`` bytes is computed as a separate + hash. The hash results are all concatenated and returned as a single + string. + + For example, ``check('sha1', 0, 1024, 512)`` will return a string of + length 40. The first 20 bytes will be the SHA-1 of the first 512 bytes + of the file, and the last 20 bytes will be the SHA-1 of the next 512 + bytes. + + :param str hash_algorithm: + the name of the hash algorithm to use (normally ``"sha1"`` or + ``"md5"``) + :param offset: + offset into the file to begin hashing (0 means to start from the + beginning) + :param length: + number of bytes to hash (0 means continue to the end of the file) + :param int block_size: + number of bytes to hash per result (must not be less than 256; 0 + means to compute only one hash of the entire segment) + :return: + `str` of bytes representing the hash of each block, concatenated + together + + :raises: + ``IOError`` -- if the server doesn't support the "check-file" + extension, or possibly doesn't support the hash algorithm requested + + .. note:: Many (most?) servers don't support this extension yet. + + .. versionadded:: 1.4 + """ + t, msg = self.sftp._request( + CMD_EXTENDED, + "check-file", + self.handle, + hash_algorithm, + int64(offset), + int64(length), + block_size, + ) + msg.get_text() # ext + msg.get_text() # alg + data = msg.get_remainder() + return data + + def set_pipelined(self, pipelined=True): + """ + Turn on/off the pipelining of write operations to this file. When + pipelining is on, paramiko won't wait for the server response after + each write operation. Instead, they're collected as they come in. At + the first non-write operation (including `.close`), all remaining + server responses are collected. This means that if there was an error + with one of your later writes, an exception might be thrown from within + `.close` instead of `.write`. + + By default, files are not pipelined. + + :param bool pipelined: + ``True`` if pipelining should be turned on for this file; ``False`` + otherwise + + .. versionadded:: 1.5 + """ + self.pipelined = pipelined + + def prefetch(self, file_size=None, max_concurrent_requests=None): + """ + Pre-fetch the remaining contents of this file in anticipation of future + `.read` calls. If reading the entire file, pre-fetching can + dramatically improve the download speed by avoiding roundtrip latency. + The file's contents are incrementally buffered in a background thread. + + The prefetched data is stored in a buffer until read via the `.read` + method. Once data has been read, it's removed from the buffer. The + data may be read in a random order (using `.seek`); chunks of the + buffer that haven't been read will continue to be buffered. + + :param int file_size: + When this is ``None`` (the default), this method calls `stat` to + determine the remote file size. In some situations, doing so can + cause exceptions or hangs (see `#562 + `_); as a + workaround, one may call `stat` explicitly and pass its value in + via this parameter. + :param int max_concurrent_requests: + The maximum number of concurrent read requests to prefetch. See + `.SFTPClient.get` (its ``max_concurrent_prefetch_requests`` param) + for details. + + .. versionadded:: 1.5.1 + .. versionchanged:: 1.16.0 + The ``file_size`` parameter was added (with no default value). + .. versionchanged:: 1.16.1 + The ``file_size`` parameter was made optional for backwards + compatibility. + .. versionchanged:: 3.3 + Added ``max_concurrent_requests``. + """ + if file_size is None: + file_size = self.stat().st_size + + # queue up async reads for the rest of the file + chunks = [] + n = self._realpos + while n < file_size: + chunk = min(self.MAX_REQUEST_SIZE, file_size - n) + chunks.append((n, chunk)) + n += chunk + if len(chunks) > 0: + self._start_prefetch(chunks, max_concurrent_requests) + + def readv(self, chunks, max_concurrent_prefetch_requests=None): + """ + Read a set of blocks from the file by (offset, length). This is more + efficient than doing a series of `.seek` and `.read` calls, since the + prefetch machinery is used to retrieve all the requested blocks at + once. + + :param chunks: + a list of ``(offset, length)`` tuples indicating which sections of + the file to read + :param int max_concurrent_prefetch_requests: + The maximum number of concurrent read requests to prefetch. See + `.SFTPClient.get` (its ``max_concurrent_prefetch_requests`` param) + for details. + :return: a list of blocks read, in the same order as in ``chunks`` + + .. versionadded:: 1.5.4 + .. versionchanged:: 3.3 + Added ``max_concurrent_prefetch_requests``. + """ + self.sftp._log( + DEBUG, "readv({}, {!r})".format(hexlify(self.handle), chunks) + ) + + read_chunks = [] + for offset, size in chunks: + # don't fetch data that's already in the prefetch buffer + if self._data_in_prefetch_buffers( + offset + ) or self._data_in_prefetch_requests(offset, size): + continue + + # break up anything larger than the max read size + while size > 0: + chunk_size = min(size, self.MAX_REQUEST_SIZE) + read_chunks.append((offset, chunk_size)) + offset += chunk_size + size -= chunk_size + + self._start_prefetch(read_chunks, max_concurrent_prefetch_requests) + # now we can just devolve to a bunch of read()s :) + for x in chunks: + self.seek(x[0]) + yield self.read(x[1]) + + # ...internals... + + def _get_size(self): + try: + return self.stat().st_size + except: + return 0 + + def _start_prefetch(self, chunks, max_concurrent_requests=None): + self._prefetching = True + self._prefetch_done = False + + t = threading.Thread( + target=self._prefetch_thread, + args=(chunks, max_concurrent_requests), + ) + t.daemon = True + t.start() + + def _prefetch_thread(self, chunks, max_concurrent_requests): + # do these read requests in a temporary thread because there may be + # a lot of them, so it may block. + for offset, length in chunks: + # Limit the number of concurrent requests in a busy-loop + if max_concurrent_requests is not None: + while True: + with self._prefetch_lock: + pf_len = len(self._prefetch_extents) + if pf_len < max_concurrent_requests: + break + time.sleep(io_sleep) + + num = self.sftp._async_request( + self, CMD_READ, self.handle, int64(offset), int(length) + ) + with self._prefetch_lock: + self._prefetch_extents[num] = (offset, length) + + def _async_response(self, t, msg, num): + if t == CMD_STATUS: + # save exception and re-raise it on next file operation + try: + self.sftp._convert_status(msg) + except Exception as e: + self._saved_exception = e + return + if t != CMD_DATA: + raise SFTPError("Expected data") + data = msg.get_string() + while True: + with self._prefetch_lock: + # spin if in race with _prefetch_thread + if num in self._prefetch_extents: + offset, length = self._prefetch_extents[num] + self._prefetch_data[offset] = data + del self._prefetch_extents[num] + if len(self._prefetch_extents) == 0: + self._prefetch_done = True + break + + def _check_exception(self): + """if there's a saved exception, raise & clear it""" + if self._saved_exception is not None: + x = self._saved_exception + self._saved_exception = None + raise x diff --git a/lib/paramiko/sftp_handle.py b/lib/paramiko/sftp_handle.py new file mode 100644 index 0000000..b204652 --- /dev/null +++ b/lib/paramiko/sftp_handle.py @@ -0,0 +1,196 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Abstraction of an SFTP file handle (for server mode). +""" + +import os +from paramiko.sftp import SFTP_OP_UNSUPPORTED, SFTP_OK +from paramiko.util import ClosingContextManager + + +class SFTPHandle(ClosingContextManager): + """ + Abstract object representing a handle to an open file (or folder) in an + SFTP server implementation. Each handle has a string representation used + by the client to refer to the underlying file. + + Server implementations can (and should) subclass SFTPHandle to implement + features of a file handle, like `stat` or `chattr`. + + Instances of this class may be used as context managers. + """ + + def __init__(self, flags=0): + """ + Create a new file handle representing a local file being served over + SFTP. If ``flags`` is passed in, it's used to determine if the file + is open in append mode. + + :param int flags: optional flags as passed to + `.SFTPServerInterface.open` + """ + self.__flags = flags + self.__name = None + # only for handles to folders: + self.__files = {} + self.__tell = None + + def close(self): + """ + When a client closes a file, this method is called on the handle. + Normally you would use this method to close the underlying OS level + file object(s). + + The default implementation checks for attributes on ``self`` named + ``readfile`` and/or ``writefile``, and if either or both are present, + their ``close()`` methods are called. This means that if you are + using the default implementations of `read` and `write`, this + method's default implementation should be fine also. + """ + readfile = getattr(self, "readfile", None) + if readfile is not None: + readfile.close() + writefile = getattr(self, "writefile", None) + if writefile is not None: + writefile.close() + + def read(self, offset, length): + """ + Read up to ``length`` bytes from this file, starting at position + ``offset``. The offset may be a Python long, since SFTP allows it + to be 64 bits. + + If the end of the file has been reached, this method may return an + empty string to signify EOF, or it may also return ``SFTP_EOF``. + + The default implementation checks for an attribute on ``self`` named + ``readfile``, and if present, performs the read operation on the Python + file-like object found there. (This is meant as a time saver for the + common case where you are wrapping a Python file object.) + + :param offset: position in the file to start reading from. + :param int length: number of bytes to attempt to read. + :return: the `bytes` read, or an error code `int`. + """ + readfile = getattr(self, "readfile", None) + if readfile is None: + return SFTP_OP_UNSUPPORTED + try: + if self.__tell is None: + self.__tell = readfile.tell() + if offset != self.__tell: + readfile.seek(offset) + self.__tell = offset + data = readfile.read(length) + except IOError as e: + self.__tell = None + return SFTPServer.convert_errno(e.errno) + self.__tell += len(data) + return data + + def write(self, offset, data): + """ + Write ``data`` into this file at position ``offset``. Extending the + file past its original end is expected. Unlike Python's normal + ``write()`` methods, this method cannot do a partial write: it must + write all of ``data`` or else return an error. + + The default implementation checks for an attribute on ``self`` named + ``writefile``, and if present, performs the write operation on the + Python file-like object found there. The attribute is named + differently from ``readfile`` to make it easy to implement read-only + (or write-only) files, but if both attributes are present, they should + refer to the same file. + + :param offset: position in the file to start reading from. + :param bytes data: data to write into the file. + :return: an SFTP error code like ``SFTP_OK``. + """ + writefile = getattr(self, "writefile", None) + if writefile is None: + return SFTP_OP_UNSUPPORTED + try: + # in append mode, don't care about seeking + if (self.__flags & os.O_APPEND) == 0: + if self.__tell is None: + self.__tell = writefile.tell() + if offset != self.__tell: + writefile.seek(offset) + self.__tell = offset + writefile.write(data) + writefile.flush() + except IOError as e: + self.__tell = None + return SFTPServer.convert_errno(e.errno) + if self.__tell is not None: + self.__tell += len(data) + return SFTP_OK + + def stat(self): + """ + Return an `.SFTPAttributes` object referring to this open file, or an + error code. This is equivalent to `.SFTPServerInterface.stat`, except + it's called on an open file instead of a path. + + :return: + an attributes object for the given file, or an SFTP error code + (like ``SFTP_PERMISSION_DENIED``). + :rtype: `.SFTPAttributes` or error code + """ + return SFTP_OP_UNSUPPORTED + + def chattr(self, attr): + """ + Change the attributes of this file. The ``attr`` object will contain + only those fields provided by the client in its request, so you should + check for the presence of fields before using them. + + :param .SFTPAttributes attr: the attributes to change on this file. + :return: an `int` error code like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED + + # ...internals... + + def _set_files(self, files): + """ + Used by the SFTP server code to cache a directory listing. (In + the SFTP protocol, listing a directory is a multi-stage process + requiring a temporary handle.) + """ + self.__files = files + + def _get_next_files(self): + """ + Used by the SFTP server code to retrieve a cached directory + listing. + """ + fnlist = self.__files[:16] + self.__files = self.__files[16:] + return fnlist + + def _get_name(self): + return self.__name + + def _set_name(self, name): + self.__name = name + + +from paramiko.sftp_server import SFTPServer diff --git a/lib/paramiko/sftp_server.py b/lib/paramiko/sftp_server.py new file mode 100644 index 0000000..cd3910d --- /dev/null +++ b/lib/paramiko/sftp_server.py @@ -0,0 +1,537 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Server-mode SFTP support. +""" + +import os +import errno +import sys +from hashlib import md5, sha1 + +from paramiko import util +from paramiko.sftp import ( + BaseSFTP, + Message, + SFTP_FAILURE, + SFTP_PERMISSION_DENIED, + SFTP_NO_SUCH_FILE, + int64, +) +from paramiko.sftp_si import SFTPServerInterface +from paramiko.sftp_attr import SFTPAttributes +from paramiko.common import DEBUG +from paramiko.server import SubsystemHandler +from paramiko.util import b + + +# known hash algorithms for the "check-file" extension +from paramiko.sftp import ( + CMD_HANDLE, + SFTP_DESC, + CMD_STATUS, + SFTP_EOF, + CMD_NAME, + SFTP_BAD_MESSAGE, + CMD_EXTENDED_REPLY, + SFTP_FLAG_READ, + SFTP_FLAG_WRITE, + SFTP_FLAG_APPEND, + SFTP_FLAG_CREATE, + SFTP_FLAG_TRUNC, + SFTP_FLAG_EXCL, + CMD_NAMES, + CMD_OPEN, + CMD_CLOSE, + SFTP_OK, + CMD_READ, + CMD_DATA, + CMD_WRITE, + CMD_REMOVE, + CMD_RENAME, + CMD_MKDIR, + CMD_RMDIR, + CMD_OPENDIR, + CMD_READDIR, + CMD_STAT, + CMD_ATTRS, + CMD_LSTAT, + CMD_FSTAT, + CMD_SETSTAT, + CMD_FSETSTAT, + CMD_READLINK, + CMD_SYMLINK, + CMD_REALPATH, + CMD_EXTENDED, + SFTP_OP_UNSUPPORTED, +) + +_hash_class = {"sha1": sha1, "md5": md5} + + +class SFTPServer(BaseSFTP, SubsystemHandler): + """ + Server-side SFTP subsystem support. Since this is a `.SubsystemHandler`, + it can be (and is meant to be) set as the handler for ``"sftp"`` requests. + Use `.Transport.set_subsystem_handler` to activate this class. + """ + + def __init__( + self, + channel, + name, + server, + sftp_si=SFTPServerInterface, + *args, + **kwargs + ): + """ + The constructor for SFTPServer is meant to be called from within the + `.Transport` as a subsystem handler. ``server`` and any additional + parameters or keyword parameters are passed from the original call to + `.Transport.set_subsystem_handler`. + + :param .Channel channel: channel passed from the `.Transport`. + :param str name: name of the requested subsystem. + :param .ServerInterface server: + the server object associated with this channel and subsystem + :param sftp_si: + a subclass of `.SFTPServerInterface` to use for handling individual + requests. + """ + BaseSFTP.__init__(self) + SubsystemHandler.__init__(self, channel, name, server) + transport = channel.get_transport() + self.logger = util.get_logger(transport.get_log_channel() + ".sftp") + self.ultra_debug = transport.get_hexdump() + self.next_handle = 1 + # map of handle-string to SFTPHandle for files & folders: + self.file_table = {} + self.folder_table = {} + self.server = sftp_si(server, *args, **kwargs) + + def _log(self, level, msg): + if issubclass(type(msg), list): + for m in msg: + super()._log(level, "[chan " + self.sock.get_name() + "] " + m) + else: + super()._log(level, "[chan " + self.sock.get_name() + "] " + msg) + + def start_subsystem(self, name, transport, channel): + self.sock = channel + self._log(DEBUG, "Started sftp server on channel {!r}".format(channel)) + self._send_server_version() + self.server.session_started() + while True: + try: + t, data = self._read_packet() + except EOFError: + self._log(DEBUG, "EOF -- end of session") + return + except Exception as e: + self._log(DEBUG, "Exception on channel: " + str(e)) + self._log(DEBUG, util.tb_strings()) + return + msg = Message(data) + request_number = msg.get_int() + try: + self._process(t, request_number, msg) + except Exception as e: + self._log(DEBUG, "Exception in server processing: " + str(e)) + self._log(DEBUG, util.tb_strings()) + # send some kind of failure message, at least + try: + self._send_status(request_number, SFTP_FAILURE) + except: + pass + + def finish_subsystem(self): + self.server.session_ended() + super().finish_subsystem() + # close any file handles that were left open + # (so we can return them to the OS quickly) + for f in self.file_table.values(): + f.close() + for f in self.folder_table.values(): + f.close() + self.file_table = {} + self.folder_table = {} + + @staticmethod + def convert_errno(e): + """ + Convert an errno value (as from an ``OSError`` or ``IOError``) into a + standard SFTP result code. This is a convenience function for trapping + exceptions in server code and returning an appropriate result. + + :param int e: an errno code, as from ``OSError.errno``. + :return: an `int` SFTP error code like ``SFTP_NO_SUCH_FILE``. + """ + if e == errno.EACCES: + # permission denied + return SFTP_PERMISSION_DENIED + elif (e == errno.ENOENT) or (e == errno.ENOTDIR): + # no such file + return SFTP_NO_SUCH_FILE + else: + return SFTP_FAILURE + + @staticmethod + def set_file_attr(filename, attr): + """ + Change a file's attributes on the local filesystem. The contents of + ``attr`` are used to change the permissions, owner, group ownership, + and/or modification & access time of the file, depending on which + attributes are present in ``attr``. + + This is meant to be a handy helper function for translating SFTP file + requests into local file operations. + + :param str filename: + name of the file to alter (should usually be an absolute path). + :param .SFTPAttributes attr: attributes to change. + """ + if sys.platform != "win32": + # mode operations are meaningless on win32 + if attr._flags & attr.FLAG_PERMISSIONS: + os.chmod(filename, attr.st_mode) + if attr._flags & attr.FLAG_UIDGID: + os.chown(filename, attr.st_uid, attr.st_gid) + if attr._flags & attr.FLAG_AMTIME: + os.utime(filename, (attr.st_atime, attr.st_mtime)) + if attr._flags & attr.FLAG_SIZE: + with open(filename, "w+") as f: + f.truncate(attr.st_size) + + # ...internals... + + def _response(self, request_number, t, *args): + msg = Message() + msg.add_int(request_number) + for item in args: + # NOTE: this is a very silly tiny class used for SFTPFile mostly + if isinstance(item, int64): + msg.add_int64(item) + elif isinstance(item, int): + msg.add_int(item) + elif isinstance(item, (str, bytes)): + msg.add_string(item) + elif type(item) is SFTPAttributes: + item._pack(msg) + else: + raise Exception( + "unknown type for {!r} type {!r}".format(item, type(item)) + ) + self._send_packet(t, msg) + + def _send_handle_response(self, request_number, handle, folder=False): + if not issubclass(type(handle), SFTPHandle): + # must be error code + self._send_status(request_number, handle) + return + handle._set_name(b("hx{:d}".format(self.next_handle))) + self.next_handle += 1 + if folder: + self.folder_table[handle._get_name()] = handle + else: + self.file_table[handle._get_name()] = handle + self._response(request_number, CMD_HANDLE, handle._get_name()) + + def _send_status(self, request_number, code, desc=None): + if desc is None: + try: + desc = SFTP_DESC[code] + except IndexError: + desc = "Unknown" + # some clients expect a "language" tag at the end + # (but don't mind it being blank) + self._response(request_number, CMD_STATUS, code, desc, "") + + def _open_folder(self, request_number, path): + resp = self.server.list_folder(path) + if issubclass(type(resp), list): + # got an actual list of filenames in the folder + folder = SFTPHandle() + folder._set_files(resp) + self._send_handle_response(request_number, folder, True) + return + # must be an error code + self._send_status(request_number, resp) + + def _read_folder(self, request_number, folder): + flist = folder._get_next_files() + if len(flist) == 0: + self._send_status(request_number, SFTP_EOF) + return + msg = Message() + msg.add_int(request_number) + msg.add_int(len(flist)) + for attr in flist: + msg.add_string(attr.filename) + msg.add_string(attr) + attr._pack(msg) + self._send_packet(CMD_NAME, msg) + + def _check_file(self, request_number, msg): + # this extension actually comes from v6 protocol, but since it's an + # extension, i feel like we can reasonably support it backported. + # it's very useful for verifying uploaded files or checking for + # rsync-like differences between local and remote files. + handle = msg.get_binary() + alg_list = msg.get_list() + start = msg.get_int64() + length = msg.get_int64() + block_size = msg.get_int() + if handle not in self.file_table: + self._send_status( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + return + f = self.file_table[handle] + for x in alg_list: + if x in _hash_class: + algname = x + alg = _hash_class[x] + break + else: + self._send_status( + request_number, SFTP_FAILURE, "No supported hash types found" + ) + return + if length == 0: + st = f.stat() + if not issubclass(type(st), SFTPAttributes): + self._send_status(request_number, st, "Unable to stat file") + return + length = st.st_size - start + if block_size == 0: + block_size = length + if block_size < 256: + self._send_status( + request_number, SFTP_FAILURE, "Block size too small" + ) + return + + sum_out = bytes() + offset = start + while offset < start + length: + blocklen = min(block_size, start + length - offset) + # don't try to read more than about 64KB at a time + chunklen = min(blocklen, 65536) + count = 0 + hash_obj = alg() + while count < blocklen: + data = f.read(offset, chunklen) + if not isinstance(data, bytes): + self._send_status( + request_number, data, "Unable to hash file" + ) + return + hash_obj.update(data) + count += len(data) + offset += count + sum_out += hash_obj.digest() + + msg = Message() + msg.add_int(request_number) + msg.add_string("check-file") + msg.add_string(algname) + msg.add_bytes(sum_out) + self._send_packet(CMD_EXTENDED_REPLY, msg) + + def _convert_pflags(self, pflags): + """convert SFTP-style open() flags to Python's os.open() flags""" + if (pflags & SFTP_FLAG_READ) and (pflags & SFTP_FLAG_WRITE): + flags = os.O_RDWR + elif pflags & SFTP_FLAG_WRITE: + flags = os.O_WRONLY + else: + flags = os.O_RDONLY + if pflags & SFTP_FLAG_APPEND: + flags |= os.O_APPEND + if pflags & SFTP_FLAG_CREATE: + flags |= os.O_CREAT + if pflags & SFTP_FLAG_TRUNC: + flags |= os.O_TRUNC + if pflags & SFTP_FLAG_EXCL: + flags |= os.O_EXCL + return flags + + def _process(self, t, request_number, msg): + self._log(DEBUG, "Request: {}".format(CMD_NAMES[t])) + if t == CMD_OPEN: + path = msg.get_text() + flags = self._convert_pflags(msg.get_int()) + attr = SFTPAttributes._from_msg(msg) + self._send_handle_response( + request_number, self.server.open(path, flags, attr) + ) + elif t == CMD_CLOSE: + handle = msg.get_binary() + if handle in self.folder_table: + del self.folder_table[handle] + self._send_status(request_number, SFTP_OK) + return + if handle in self.file_table: + self.file_table[handle].close() + del self.file_table[handle] + self._send_status(request_number, SFTP_OK) + return + self._send_status( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + elif t == CMD_READ: + handle = msg.get_binary() + offset = msg.get_int64() + length = msg.get_int() + if handle not in self.file_table: + self._send_status( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + return + data = self.file_table[handle].read(offset, length) + if isinstance(data, (bytes, str)): + if len(data) == 0: + self._send_status(request_number, SFTP_EOF) + else: + self._response(request_number, CMD_DATA, data) + else: + self._send_status(request_number, data) + elif t == CMD_WRITE: + handle = msg.get_binary() + offset = msg.get_int64() + data = msg.get_binary() + if handle not in self.file_table: + self._send_status( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + return + self._send_status( + request_number, self.file_table[handle].write(offset, data) + ) + elif t == CMD_REMOVE: + path = msg.get_text() + self._send_status(request_number, self.server.remove(path)) + elif t == CMD_RENAME: + oldpath = msg.get_text() + newpath = msg.get_text() + self._send_status( + request_number, self.server.rename(oldpath, newpath) + ) + elif t == CMD_MKDIR: + path = msg.get_text() + attr = SFTPAttributes._from_msg(msg) + self._send_status(request_number, self.server.mkdir(path, attr)) + elif t == CMD_RMDIR: + path = msg.get_text() + self._send_status(request_number, self.server.rmdir(path)) + elif t == CMD_OPENDIR: + path = msg.get_text() + self._open_folder(request_number, path) + return + elif t == CMD_READDIR: + handle = msg.get_binary() + if handle not in self.folder_table: + self._send_status( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + return + folder = self.folder_table[handle] + self._read_folder(request_number, folder) + elif t == CMD_STAT: + path = msg.get_text() + resp = self.server.stat(path) + if issubclass(type(resp), SFTPAttributes): + self._response(request_number, CMD_ATTRS, resp) + else: + self._send_status(request_number, resp) + elif t == CMD_LSTAT: + path = msg.get_text() + resp = self.server.lstat(path) + if issubclass(type(resp), SFTPAttributes): + self._response(request_number, CMD_ATTRS, resp) + else: + self._send_status(request_number, resp) + elif t == CMD_FSTAT: + handle = msg.get_binary() + if handle not in self.file_table: + self._send_status( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + return + resp = self.file_table[handle].stat() + if issubclass(type(resp), SFTPAttributes): + self._response(request_number, CMD_ATTRS, resp) + else: + self._send_status(request_number, resp) + elif t == CMD_SETSTAT: + path = msg.get_text() + attr = SFTPAttributes._from_msg(msg) + self._send_status(request_number, self.server.chattr(path, attr)) + elif t == CMD_FSETSTAT: + handle = msg.get_binary() + attr = SFTPAttributes._from_msg(msg) + if handle not in self.file_table: + self._response( + request_number, SFTP_BAD_MESSAGE, "Invalid handle" + ) + return + self._send_status( + request_number, self.file_table[handle].chattr(attr) + ) + elif t == CMD_READLINK: + path = msg.get_text() + resp = self.server.readlink(path) + if isinstance(resp, (bytes, str)): + self._response( + request_number, CMD_NAME, 1, resp, "", SFTPAttributes() + ) + else: + self._send_status(request_number, resp) + elif t == CMD_SYMLINK: + # the sftp 2 draft is incorrect here! + # path always follows target_path + target_path = msg.get_text() + path = msg.get_text() + self._send_status( + request_number, self.server.symlink(target_path, path) + ) + elif t == CMD_REALPATH: + path = msg.get_text() + rpath = self.server.canonicalize(path) + self._response( + request_number, CMD_NAME, 1, rpath, "", SFTPAttributes() + ) + elif t == CMD_EXTENDED: + tag = msg.get_text() + if tag == "check-file": + self._check_file(request_number, msg) + elif tag == "posix-rename@openssh.com": + oldpath = msg.get_text() + newpath = msg.get_text() + self._send_status( + request_number, self.server.posix_rename(oldpath, newpath) + ) + else: + self._send_status(request_number, SFTP_OP_UNSUPPORTED) + else: + self._send_status(request_number, SFTP_OP_UNSUPPORTED) + + +from paramiko.sftp_handle import SFTPHandle diff --git a/lib/paramiko/sftp_si.py b/lib/paramiko/sftp_si.py new file mode 100644 index 0000000..72b5db9 --- /dev/null +++ b/lib/paramiko/sftp_si.py @@ -0,0 +1,316 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +An interface to override for SFTP server support. +""" + +import os +import sys +from paramiko.sftp import SFTP_OP_UNSUPPORTED + + +class SFTPServerInterface: + """ + This class defines an interface for controlling the behavior of paramiko + when using the `.SFTPServer` subsystem to provide an SFTP server. + + Methods on this class are called from the SFTP session's thread, so you can + block as long as necessary without affecting other sessions (even other + SFTP sessions). However, raising an exception will usually cause the SFTP + session to abruptly end, so you will usually want to catch exceptions and + return an appropriate error code. + + All paths are in string form instead of unicode because not all SFTP + clients & servers obey the requirement that paths be encoded in UTF-8. + """ + + def __init__(self, server, *args, **kwargs): + """ + Create a new SFTPServerInterface object. This method does nothing by + default and is meant to be overridden by subclasses. + + :param .ServerInterface server: + the server object associated with this channel and SFTP subsystem + """ + super().__init__(*args, **kwargs) + + def session_started(self): + """ + The SFTP server session has just started. This method is meant to be + overridden to perform any necessary setup before handling callbacks + from SFTP operations. + """ + pass + + def session_ended(self): + """ + The SFTP server session has just ended, either cleanly or via an + exception. This method is meant to be overridden to perform any + necessary cleanup before this `.SFTPServerInterface` object is + destroyed. + """ + pass + + def open(self, path, flags, attr): + """ + Open a file on the server and create a handle for future operations + on that file. On success, a new object subclassed from `.SFTPHandle` + should be returned. This handle will be used for future operations + on the file (read, write, etc). On failure, an error code such as + ``SFTP_PERMISSION_DENIED`` should be returned. + + ``flags`` contains the requested mode for opening (read-only, + write-append, etc) as a bitset of flags from the ``os`` module: + + - ``os.O_RDONLY`` + - ``os.O_WRONLY`` + - ``os.O_RDWR`` + - ``os.O_APPEND`` + - ``os.O_CREAT`` + - ``os.O_TRUNC`` + - ``os.O_EXCL`` + + (One of ``os.O_RDONLY``, ``os.O_WRONLY``, or ``os.O_RDWR`` will always + be set.) + + The ``attr`` object contains requested attributes of the file if it + has to be created. Some or all attribute fields may be missing if + the client didn't specify them. + + .. note:: The SFTP protocol defines all files to be in "binary" mode. + There is no equivalent to Python's "text" mode. + + :param str path: + the requested path (relative or absolute) of the file to be opened. + :param int flags: + flags or'd together from the ``os`` module indicating the requested + mode for opening the file. + :param .SFTPAttributes attr: + requested attributes of the file if it is newly created. + :return: a new `.SFTPHandle` or error code. + """ + return SFTP_OP_UNSUPPORTED + + def list_folder(self, path): + """ + Return a list of files within a given folder. The ``path`` will use + posix notation (``"/"`` separates folder names) and may be an absolute + or relative path. + + The list of files is expected to be a list of `.SFTPAttributes` + objects, which are similar in structure to the objects returned by + ``os.stat``. In addition, each object should have its ``filename`` + field filled in, since this is important to a directory listing and + not normally present in ``os.stat`` results. The method + `.SFTPAttributes.from_stat` will usually do what you want. + + In case of an error, you should return one of the ``SFTP_*`` error + codes, such as ``SFTP_PERMISSION_DENIED``. + + :param str path: the requested path (relative or absolute) to be + listed. + :return: + a list of the files in the given folder, using `.SFTPAttributes` + objects. + + .. note:: + You should normalize the given ``path`` first (see the `os.path` + module) and check appropriate permissions before returning the list + of files. Be careful of malicious clients attempting to use + relative paths to escape restricted folders, if you're doing a + direct translation from the SFTP server path to your local + filesystem. + """ + return SFTP_OP_UNSUPPORTED + + def stat(self, path): + """ + Return an `.SFTPAttributes` object for a path on the server, or an + error code. If your server supports symbolic links (also known as + "aliases"), you should follow them. (`lstat` is the corresponding + call that doesn't follow symlinks/aliases.) + + :param str path: + the requested path (relative or absolute) to fetch file statistics + for. + :return: + an `.SFTPAttributes` object for the given file, or an SFTP error + code (like ``SFTP_PERMISSION_DENIED``). + """ + return SFTP_OP_UNSUPPORTED + + def lstat(self, path): + """ + Return an `.SFTPAttributes` object for a path on the server, or an + error code. If your server supports symbolic links (also known as + "aliases"), you should not follow them -- instead, you should + return data on the symlink or alias itself. (`stat` is the + corresponding call that follows symlinks/aliases.) + + :param str path: + the requested path (relative or absolute) to fetch file statistics + for. + :type path: str + :return: + an `.SFTPAttributes` object for the given file, or an SFTP error + code (like ``SFTP_PERMISSION_DENIED``). + """ + return SFTP_OP_UNSUPPORTED + + def remove(self, path): + """ + Delete a file, if possible. + + :param str path: + the requested path (relative or absolute) of the file to delete. + :return: an SFTP error code `int` like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED + + def rename(self, oldpath, newpath): + """ + Rename (or move) a file. The SFTP specification implies that this + method can be used to move an existing file into a different folder, + and since there's no other (easy) way to move files via SFTP, it's + probably a good idea to implement "move" in this method too, even for + files that cross disk partition boundaries, if at all possible. + + .. note:: You should return an error if a file with the same name as + ``newpath`` already exists. (The rename operation should be + non-desctructive.) + + .. note:: + This method implements 'standard' SFTP ``RENAME`` behavior; those + seeking the OpenSSH "POSIX rename" extension behavior should use + `posix_rename`. + + :param str oldpath: + the requested path (relative or absolute) of the existing file. + :param str newpath: the requested new path of the file. + :return: an SFTP error code `int` like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED + + def posix_rename(self, oldpath, newpath): + """ + Rename (or move) a file, following posix conventions. If newpath + already exists, it will be overwritten. + + :param str oldpath: + the requested path (relative or absolute) of the existing file. + :param str newpath: the requested new path of the file. + :return: an SFTP error code `int` like ``SFTP_OK``. + + :versionadded: 2.2 + """ + return SFTP_OP_UNSUPPORTED + + def mkdir(self, path, attr): + """ + Create a new directory with the given attributes. The ``attr`` + object may be considered a "hint" and ignored. + + The ``attr`` object will contain only those fields provided by the + client in its request, so you should use ``hasattr`` to check for + the presence of fields before using them. In some cases, the ``attr`` + object may be completely empty. + + :param str path: + requested path (relative or absolute) of the new folder. + :param .SFTPAttributes attr: requested attributes of the new folder. + :return: an SFTP error code `int` like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED + + def rmdir(self, path): + """ + Remove a directory if it exists. The ``path`` should refer to an + existing, empty folder -- otherwise this method should return an + error. + + :param str path: + requested path (relative or absolute) of the folder to remove. + :return: an SFTP error code `int` like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED + + def chattr(self, path, attr): + """ + Change the attributes of a file. The ``attr`` object will contain + only those fields provided by the client in its request, so you + should check for the presence of fields before using them. + + :param str path: + requested path (relative or absolute) of the file to change. + :param attr: + requested attributes to change on the file (an `.SFTPAttributes` + object) + :return: an error code `int` like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED + + def canonicalize(self, path): + """ + Return the canonical form of a path on the server. For example, + if the server's home folder is ``/home/foo``, the path + ``"../betty"`` would be canonicalized to ``"/home/betty"``. Note + the obvious security issues: if you're serving files only from a + specific folder, you probably don't want this method to reveal path + names outside that folder. + + You may find the Python methods in ``os.path`` useful, especially + ``os.path.normpath`` and ``os.path.realpath``. + + The default implementation returns ``os.path.normpath('/' + path)``. + """ + if os.path.isabs(path): + out = os.path.normpath(path) + else: + out = os.path.normpath("/" + path) + if sys.platform == "win32": + # on windows, normalize backslashes to sftp/posix format + out = out.replace("\\", "/") + return out + + def readlink(self, path): + """ + Return the target of a symbolic link (or shortcut) on the server. + If the specified path doesn't refer to a symbolic link, an error + should be returned. + + :param str path: path (relative or absolute) of the symbolic link. + :return: + the target `str` path of the symbolic link, or an error code like + ``SFTP_NO_SUCH_FILE``. + """ + return SFTP_OP_UNSUPPORTED + + def symlink(self, target_path, path): + """ + Create a symbolic link on the server, as new pathname ``path``, + with ``target_path`` as the target of the link. + + :param str target_path: + path (relative or absolute) of the target for this new symbolic + link. + :param str path: + path (relative or absolute) of the symbolic link to create. + :return: an error code `int` like ``SFTP_OK``. + """ + return SFTP_OP_UNSUPPORTED diff --git a/lib/paramiko/ssh_exception.py b/lib/paramiko/ssh_exception.py new file mode 100644 index 0000000..2b68ebe --- /dev/null +++ b/lib/paramiko/ssh_exception.py @@ -0,0 +1,250 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import socket + + +class SSHException(Exception): + """ + Exception raised by failures in SSH2 protocol negotiation or logic errors. + """ + + pass + + +class AuthenticationException(SSHException): + """ + Exception raised when authentication failed for some reason. It may be + possible to retry with different credentials. (Other classes specify more + specific reasons.) + + .. versionadded:: 1.6 + """ + + pass + + +class PasswordRequiredException(AuthenticationException): + """ + Exception raised when a password is needed to unlock a private key file. + """ + + pass + + +class BadAuthenticationType(AuthenticationException): + """ + Exception raised when an authentication type (like password) is used, but + the server isn't allowing that type. (It may only allow public-key, for + example.) + + .. versionadded:: 1.1 + """ + + allowed_types = [] + + # TODO 4.0: remove explanation kwarg + def __init__(self, explanation, types): + # TODO 4.0: remove this supercall unless it's actually required for + # pickling (after fixing pickling) + AuthenticationException.__init__(self, explanation, types) + self.explanation = explanation + self.allowed_types = types + + def __str__(self): + return "{}; allowed types: {!r}".format( + self.explanation, self.allowed_types + ) + + +class PartialAuthentication(AuthenticationException): + """ + An internal exception thrown in the case of partial authentication. + """ + + allowed_types = [] + + def __init__(self, types): + AuthenticationException.__init__(self, types) + self.allowed_types = types + + def __str__(self): + return "Partial authentication; allowed types: {!r}".format( + self.allowed_types + ) + + +# TODO 4.0: stop inheriting from SSHException, move to auth.py +class UnableToAuthenticate(AuthenticationException): + pass + + +class ChannelException(SSHException): + """ + Exception raised when an attempt to open a new `.Channel` fails. + + :param int code: the error code returned by the server + + .. versionadded:: 1.6 + """ + + def __init__(self, code, text): + SSHException.__init__(self, code, text) + self.code = code + self.text = text + + def __str__(self): + return "ChannelException({!r}, {!r})".format(self.code, self.text) + + +class BadHostKeyException(SSHException): + """ + The host key given by the SSH server did not match what we were expecting. + + :param str hostname: the hostname of the SSH server + :param PKey got_key: the host key presented by the server + :param PKey expected_key: the host key expected + + .. versionadded:: 1.6 + """ + + def __init__(self, hostname, got_key, expected_key): + SSHException.__init__(self, hostname, got_key, expected_key) + self.hostname = hostname + self.key = got_key + self.expected_key = expected_key + + def __str__(self): + msg = "Host key for server '{}' does not match: got '{}', expected '{}'" # noqa + return msg.format( + self.hostname, + self.key.get_base64(), + self.expected_key.get_base64(), + ) + + +class IncompatiblePeer(SSHException): + """ + A disagreement arose regarding an algorithm required for key exchange. + + .. versionadded:: 2.9 + """ + + # TODO 4.0: consider making this annotate w/ 1..N 'missing' algorithms, + # either just the first one that would halt kex, or even updating the + # Transport logic so we record /all/ that /could/ halt kex. + # TODO: update docstrings where this may end up raised so they are more + # specific. + pass + + +class ProxyCommandFailure(SSHException): + """ + The "ProxyCommand" found in the .ssh/config file returned an error. + + :param str command: The command line that is generating this exception. + :param str error: The error captured from the proxy command output. + """ + + def __init__(self, command, error): + SSHException.__init__(self, command, error) + self.command = command + self.error = error + + def __str__(self): + return 'ProxyCommand("{}") returned nonzero exit status: {}'.format( + self.command, self.error + ) + + +class NoValidConnectionsError(socket.error): + """ + Multiple connection attempts were made and no families succeeded. + + This exception class wraps multiple "real" underlying connection errors, + all of which represent failed connection attempts. Because these errors are + not guaranteed to all be of the same error type (i.e. different errno, + `socket.error` subclass, message, etc) we expose a single unified error + message and a ``None`` errno so that instances of this class match most + normal handling of `socket.error` objects. + + To see the wrapped exception objects, access the ``errors`` attribute. + ``errors`` is a dict whose keys are address tuples (e.g. ``('127.0.0.1', + 22)``) and whose values are the exception encountered trying to connect to + that address. + + It is implied/assumed that all the errors given to a single instance of + this class are from connecting to the same hostname + port (and thus that + the differences are in the resolution of the hostname - e.g. IPv4 vs v6). + + .. versionadded:: 1.16 + """ + + def __init__(self, errors): + """ + :param dict errors: + The errors dict to store, as described by class docstring. + """ + addrs = sorted(errors.keys()) + body = ", ".join([x[0] for x in addrs[:-1]]) + tail = addrs[-1][0] + if body: + msg = "Unable to connect to port {0} on {1} or {2}" + else: + msg = "Unable to connect to port {0} on {2}" + super().__init__( + None, msg.format(addrs[0][1], body, tail) # stand-in for errno + ) + self.errors = errors + + def __reduce__(self): + return (self.__class__, (self.errors,)) + + +class CouldNotCanonicalize(SSHException): + """ + Raised when hostname canonicalization fails & fallback is disabled. + + .. versionadded:: 2.7 + """ + + pass + + +class ConfigParseError(SSHException): + """ + A fatal error was encountered trying to parse SSH config data. + + Typically this means a config file violated the ``ssh_config`` + specification in a manner that requires exiting immediately, such as not + matching ``key = value`` syntax or misusing certain ``Match`` keywords. + + .. versionadded:: 2.7 + """ + + pass + + +class MessageOrderError(SSHException): + """ + Out-of-order protocol messages were received, violating "strict kex" mode. + + .. versionadded:: 3.4 + """ + + pass diff --git a/lib/paramiko/ssh_gss.py b/lib/paramiko/ssh_gss.py new file mode 100644 index 0000000..30a2541 --- /dev/null +++ b/lib/paramiko/ssh_gss.py @@ -0,0 +1,772 @@ +# Copyright (C) 2013-2014 science + computing ag +# Author: Sebastian Deiss +# +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +""" +This module provides GSS-API / SSPI authentication as defined in :rfc:`4462`. + +.. note:: Credential delegation is not supported in server mode. + +.. seealso:: :doc:`/api/kex_gss` + +.. versionadded:: 1.15 +""" + +import struct +import os +import sys + + +#: A boolean constraint that indicates if GSS-API / SSPI is available. +GSS_AUTH_AVAILABLE = True + + +#: A tuple of the exception types used by the underlying GSSAPI implementation. +GSS_EXCEPTIONS = () + + +#: :var str _API: Constraint for the used API +_API = None + +try: + import gssapi + + if hasattr(gssapi, "__title__") and gssapi.__title__ == "python-gssapi": + # old, unmaintained python-gssapi package + _API = "MIT" # keep this for compatibility + GSS_EXCEPTIONS = (gssapi.GSSException,) + else: + _API = "PYTHON-GSSAPI-NEW" + GSS_EXCEPTIONS = ( + gssapi.exceptions.GeneralError, + gssapi.raw.misc.GSSError, + ) +except (ImportError, OSError): + try: + import pywintypes + import sspicon + import sspi + + _API = "SSPI" + GSS_EXCEPTIONS = (pywintypes.error,) + except ImportError: + GSS_AUTH_AVAILABLE = False + _API = None + +from paramiko.common import MSG_USERAUTH_REQUEST +from paramiko.ssh_exception import SSHException + + +def GSSAuth(auth_method, gss_deleg_creds=True): + """ + Provide SSH2 GSS-API / SSPI authentication. + + :param str auth_method: The name of the SSH authentication mechanism + (gssapi-with-mic or gss-keyex) + :param bool gss_deleg_creds: Delegate client credentials or not. + We delegate credentials by default. + :return: Either an `._SSH_GSSAPI_OLD` or `._SSH_GSSAPI_NEW` (Unix) + object or an `_SSH_SSPI` (Windows) object + :rtype: object + + :raises: ``ImportError`` -- If no GSS-API / SSPI module could be imported. + + :see: `RFC 4462 `_ + :note: Check for the available API and return either an `._SSH_GSSAPI_OLD` + (MIT GSSAPI using python-gssapi package) object, an + `._SSH_GSSAPI_NEW` (MIT GSSAPI using gssapi package) object + or an `._SSH_SSPI` (MS SSPI) object. + If there is no supported API available, + ``None`` will be returned. + """ + if _API == "MIT": + return _SSH_GSSAPI_OLD(auth_method, gss_deleg_creds) + elif _API == "PYTHON-GSSAPI-NEW": + return _SSH_GSSAPI_NEW(auth_method, gss_deleg_creds) + elif _API == "SSPI" and os.name == "nt": + return _SSH_SSPI(auth_method, gss_deleg_creds) + else: + raise ImportError("Unable to import a GSS-API / SSPI module!") + + +class _SSH_GSSAuth: + """ + Contains the shared variables and methods of `._SSH_GSSAPI_OLD`, + `._SSH_GSSAPI_NEW` and `._SSH_SSPI`. + """ + + def __init__(self, auth_method, gss_deleg_creds): + """ + :param str auth_method: The name of the SSH authentication mechanism + (gssapi-with-mic or gss-keyex) + :param bool gss_deleg_creds: Delegate client credentials or not + """ + self._auth_method = auth_method + self._gss_deleg_creds = gss_deleg_creds + self._gss_host = None + self._username = None + self._session_id = None + self._service = "ssh-connection" + """ + OpenSSH supports Kerberos V5 mechanism only for GSS-API authentication, + so we also support the krb5 mechanism only. + """ + self._krb5_mech = "1.2.840.113554.1.2.2" + + # client mode + self._gss_ctxt = None + self._gss_ctxt_status = False + + # server mode + self._gss_srv_ctxt = None + self._gss_srv_ctxt_status = False + self.cc_file = None + + def set_service(self, service): + """ + This is just a setter to use a non default service. + I added this method, because RFC 4462 doesn't specify "ssh-connection" + as the only service value. + + :param str service: The desired SSH service + """ + if service.find("ssh-"): + self._service = service + + def set_username(self, username): + """ + Setter for C{username}. If GSS-API Key Exchange is performed, the + username is not set by C{ssh_init_sec_context}. + + :param str username: The name of the user who attempts to login + """ + self._username = username + + def ssh_gss_oids(self, mode="client"): + """ + This method returns a single OID, because we only support the + Kerberos V5 mechanism. + + :param str mode: Client for client mode and server for server mode + :return: A byte sequence containing the number of supported + OIDs, the length of the OID and the actual OID encoded with + DER + :note: In server mode we just return the OID length and the DER encoded + OID. + """ + from pyasn1.type.univ import ObjectIdentifier + from pyasn1.codec.der import encoder + + OIDs = self._make_uint32(1) + krb5_OID = encoder.encode(ObjectIdentifier(self._krb5_mech)) + OID_len = self._make_uint32(len(krb5_OID)) + if mode == "server": + return OID_len + krb5_OID + return OIDs + OID_len + krb5_OID + + def ssh_check_mech(self, desired_mech): + """ + Check if the given OID is the Kerberos V5 OID (server mode). + + :param str desired_mech: The desired GSS-API mechanism of the client + :return: ``True`` if the given OID is supported, otherwise C{False} + """ + from pyasn1.codec.der import decoder + + mech, __ = decoder.decode(desired_mech) + if mech.__str__() != self._krb5_mech: + return False + return True + + # Internals + # ------------------------------------------------------------------------- + def _make_uint32(self, integer): + """ + Create a 32 bit unsigned integer (The byte sequence of an integer). + + :param int integer: The integer value to convert + :return: The byte sequence of an 32 bit integer + """ + return struct.pack("!I", integer) + + def _ssh_build_mic(self, session_id, username, service, auth_method): + """ + Create the SSH2 MIC filed for gssapi-with-mic. + + :param str session_id: The SSH session ID + :param str username: The name of the user who attempts to login + :param str service: The requested SSH service + :param str auth_method: The requested SSH authentication mechanism + :return: The MIC as defined in RFC 4462. The contents of the + MIC field are: + string session_identifier, + byte SSH_MSG_USERAUTH_REQUEST, + string user-name, + string service (ssh-connection), + string authentication-method + (gssapi-with-mic or gssapi-keyex) + """ + mic = self._make_uint32(len(session_id)) + mic += session_id + mic += struct.pack("B", MSG_USERAUTH_REQUEST) + mic += self._make_uint32(len(username)) + mic += username.encode() + mic += self._make_uint32(len(service)) + mic += service.encode() + mic += self._make_uint32(len(auth_method)) + mic += auth_method.encode() + return mic + + +class _SSH_GSSAPI_OLD(_SSH_GSSAuth): + """ + Implementation of the GSS-API MIT Kerberos Authentication for SSH2, + using the older (unmaintained) python-gssapi package. + + :see: `.GSSAuth` + """ + + def __init__(self, auth_method, gss_deleg_creds): + """ + :param str auth_method: The name of the SSH authentication mechanism + (gssapi-with-mic or gss-keyex) + :param bool gss_deleg_creds: Delegate client credentials or not + """ + _SSH_GSSAuth.__init__(self, auth_method, gss_deleg_creds) + + if self._gss_deleg_creds: + self._gss_flags = ( + gssapi.C_PROT_READY_FLAG, + gssapi.C_INTEG_FLAG, + gssapi.C_MUTUAL_FLAG, + gssapi.C_DELEG_FLAG, + ) + else: + self._gss_flags = ( + gssapi.C_PROT_READY_FLAG, + gssapi.C_INTEG_FLAG, + gssapi.C_MUTUAL_FLAG, + ) + + def ssh_init_sec_context( + self, target, desired_mech=None, username=None, recv_token=None + ): + """ + Initialize a GSS-API context. + + :param str username: The name of the user who attempts to login + :param str target: The hostname of the target to connect to + :param str desired_mech: The negotiated GSS-API mechanism + ("pseudo negotiated" mechanism, because we + support just the krb5 mechanism :-)) + :param str recv_token: The GSS-API token received from the Server + :raises: + `.SSHException` -- Is raised if the desired mechanism of the client + is not supported + :return: A ``String`` if the GSS-API has returned a token or + ``None`` if no token was returned + """ + from pyasn1.codec.der import decoder + + self._username = username + self._gss_host = target + targ_name = gssapi.Name( + "host@" + self._gss_host, gssapi.C_NT_HOSTBASED_SERVICE + ) + ctx = gssapi.Context() + ctx.flags = self._gss_flags + if desired_mech is None: + krb5_mech = gssapi.OID.mech_from_string(self._krb5_mech) + else: + mech, __ = decoder.decode(desired_mech) + if mech.__str__() != self._krb5_mech: + raise SSHException("Unsupported mechanism OID.") + else: + krb5_mech = gssapi.OID.mech_from_string(self._krb5_mech) + token = None + try: + if recv_token is None: + self._gss_ctxt = gssapi.InitContext( + peer_name=targ_name, + mech_type=krb5_mech, + req_flags=ctx.flags, + ) + token = self._gss_ctxt.step(token) + else: + token = self._gss_ctxt.step(recv_token) + except gssapi.GSSException: + message = "{} Target: {}".format(sys.exc_info()[1], self._gss_host) + raise gssapi.GSSException(message) + self._gss_ctxt_status = self._gss_ctxt.established + return token + + def ssh_get_mic(self, session_id, gss_kex=False): + """ + Create the MIC token for a SSH2 message. + + :param str session_id: The SSH session ID + :param bool gss_kex: Generate the MIC for GSS-API Key Exchange or not + :return: gssapi-with-mic: + Returns the MIC token from GSS-API for the message we created + with ``_ssh_build_mic``. + gssapi-keyex: + Returns the MIC token from GSS-API with the SSH session ID as + message. + """ + self._session_id = session_id + if not gss_kex: + mic_field = self._ssh_build_mic( + self._session_id, + self._username, + self._service, + self._auth_method, + ) + mic_token = self._gss_ctxt.get_mic(mic_field) + else: + # for key exchange with gssapi-keyex + mic_token = self._gss_srv_ctxt.get_mic(self._session_id) + return mic_token + + def ssh_accept_sec_context(self, hostname, recv_token, username=None): + """ + Accept a GSS-API context (server mode). + + :param str hostname: The servers hostname + :param str username: The name of the user who attempts to login + :param str recv_token: The GSS-API Token received from the server, + if it's not the initial call. + :return: A ``String`` if the GSS-API has returned a token or ``None`` + if no token was returned + """ + # hostname and username are not required for GSSAPI, but for SSPI + self._gss_host = hostname + self._username = username + if self._gss_srv_ctxt is None: + self._gss_srv_ctxt = gssapi.AcceptContext() + token = self._gss_srv_ctxt.step(recv_token) + self._gss_srv_ctxt_status = self._gss_srv_ctxt.established + return token + + def ssh_check_mic(self, mic_token, session_id, username=None): + """ + Verify the MIC token for a SSH2 message. + + :param str mic_token: The MIC token received from the client + :param str session_id: The SSH session ID + :param str username: The name of the user who attempts to login + :return: None if the MIC check was successful + :raises: ``gssapi.GSSException`` -- if the MIC check failed + """ + self._session_id = session_id + self._username = username + if self._username is not None: + # server mode + mic_field = self._ssh_build_mic( + self._session_id, + self._username, + self._service, + self._auth_method, + ) + self._gss_srv_ctxt.verify_mic(mic_field, mic_token) + else: + # for key exchange with gssapi-keyex + # client mode + self._gss_ctxt.verify_mic(self._session_id, mic_token) + + @property + def credentials_delegated(self): + """ + Checks if credentials are delegated (server mode). + + :return: ``True`` if credentials are delegated, otherwise ``False`` + """ + if self._gss_srv_ctxt.delegated_cred is not None: + return True + return False + + def save_client_creds(self, client_token): + """ + Save the Client token in a file. This is used by the SSH server + to store the client credentials if credentials are delegated + (server mode). + + :param str client_token: The GSS-API token received form the client + :raises: + ``NotImplementedError`` -- Credential delegation is currently not + supported in server mode + """ + raise NotImplementedError + + +class _SSH_GSSAPI_NEW(_SSH_GSSAuth): + """ + Implementation of the GSS-API MIT Kerberos Authentication for SSH2, + using the newer, currently maintained gssapi package. + + :see: `.GSSAuth` + """ + + def __init__(self, auth_method, gss_deleg_creds): + """ + :param str auth_method: The name of the SSH authentication mechanism + (gssapi-with-mic or gss-keyex) + :param bool gss_deleg_creds: Delegate client credentials or not + """ + _SSH_GSSAuth.__init__(self, auth_method, gss_deleg_creds) + + if self._gss_deleg_creds: + self._gss_flags = ( + gssapi.RequirementFlag.protection_ready, + gssapi.RequirementFlag.integrity, + gssapi.RequirementFlag.mutual_authentication, + gssapi.RequirementFlag.delegate_to_peer, + ) + else: + self._gss_flags = ( + gssapi.RequirementFlag.protection_ready, + gssapi.RequirementFlag.integrity, + gssapi.RequirementFlag.mutual_authentication, + ) + + def ssh_init_sec_context( + self, target, desired_mech=None, username=None, recv_token=None + ): + """ + Initialize a GSS-API context. + + :param str username: The name of the user who attempts to login + :param str target: The hostname of the target to connect to + :param str desired_mech: The negotiated GSS-API mechanism + ("pseudo negotiated" mechanism, because we + support just the krb5 mechanism :-)) + :param str recv_token: The GSS-API token received from the Server + :raises: `.SSHException` -- Is raised if the desired mechanism of the + client is not supported + :raises: ``gssapi.exceptions.GSSError`` if there is an error signaled + by the GSS-API implementation + :return: A ``String`` if the GSS-API has returned a token or ``None`` + if no token was returned + """ + from pyasn1.codec.der import decoder + + self._username = username + self._gss_host = target + targ_name = gssapi.Name( + "host@" + self._gss_host, + name_type=gssapi.NameType.hostbased_service, + ) + if desired_mech is not None: + mech, __ = decoder.decode(desired_mech) + if mech.__str__() != self._krb5_mech: + raise SSHException("Unsupported mechanism OID.") + krb5_mech = gssapi.MechType.kerberos + token = None + if recv_token is None: + self._gss_ctxt = gssapi.SecurityContext( + name=targ_name, + flags=self._gss_flags, + mech=krb5_mech, + usage="initiate", + ) + token = self._gss_ctxt.step(token) + else: + token = self._gss_ctxt.step(recv_token) + self._gss_ctxt_status = self._gss_ctxt.complete + return token + + def ssh_get_mic(self, session_id, gss_kex=False): + """ + Create the MIC token for a SSH2 message. + + :param str session_id: The SSH session ID + :param bool gss_kex: Generate the MIC for GSS-API Key Exchange or not + :return: gssapi-with-mic: + Returns the MIC token from GSS-API for the message we created + with ``_ssh_build_mic``. + gssapi-keyex: + Returns the MIC token from GSS-API with the SSH session ID as + message. + :rtype: str + """ + self._session_id = session_id + if not gss_kex: + mic_field = self._ssh_build_mic( + self._session_id, + self._username, + self._service, + self._auth_method, + ) + mic_token = self._gss_ctxt.get_signature(mic_field) + else: + # for key exchange with gssapi-keyex + mic_token = self._gss_srv_ctxt.get_signature(self._session_id) + return mic_token + + def ssh_accept_sec_context(self, hostname, recv_token, username=None): + """ + Accept a GSS-API context (server mode). + + :param str hostname: The servers hostname + :param str username: The name of the user who attempts to login + :param str recv_token: The GSS-API Token received from the server, + if it's not the initial call. + :return: A ``String`` if the GSS-API has returned a token or ``None`` + if no token was returned + """ + # hostname and username are not required for GSSAPI, but for SSPI + self._gss_host = hostname + self._username = username + if self._gss_srv_ctxt is None: + self._gss_srv_ctxt = gssapi.SecurityContext(usage="accept") + token = self._gss_srv_ctxt.step(recv_token) + self._gss_srv_ctxt_status = self._gss_srv_ctxt.complete + return token + + def ssh_check_mic(self, mic_token, session_id, username=None): + """ + Verify the MIC token for a SSH2 message. + + :param str mic_token: The MIC token received from the client + :param str session_id: The SSH session ID + :param str username: The name of the user who attempts to login + :return: None if the MIC check was successful + :raises: ``gssapi.exceptions.GSSError`` -- if the MIC check failed + """ + self._session_id = session_id + self._username = username + if self._username is not None: + # server mode + mic_field = self._ssh_build_mic( + self._session_id, + self._username, + self._service, + self._auth_method, + ) + self._gss_srv_ctxt.verify_signature(mic_field, mic_token) + else: + # for key exchange with gssapi-keyex + # client mode + self._gss_ctxt.verify_signature(self._session_id, mic_token) + + @property + def credentials_delegated(self): + """ + Checks if credentials are delegated (server mode). + + :return: ``True`` if credentials are delegated, otherwise ``False`` + :rtype: bool + """ + if self._gss_srv_ctxt.delegated_creds is not None: + return True + return False + + def save_client_creds(self, client_token): + """ + Save the Client token in a file. This is used by the SSH server + to store the client credentials if credentials are delegated + (server mode). + + :param str client_token: The GSS-API token received form the client + :raises: ``NotImplementedError`` -- Credential delegation is currently + not supported in server mode + """ + raise NotImplementedError + + +class _SSH_SSPI(_SSH_GSSAuth): + """ + Implementation of the Microsoft SSPI Kerberos Authentication for SSH2. + + :see: `.GSSAuth` + """ + + def __init__(self, auth_method, gss_deleg_creds): + """ + :param str auth_method: The name of the SSH authentication mechanism + (gssapi-with-mic or gss-keyex) + :param bool gss_deleg_creds: Delegate client credentials or not + """ + _SSH_GSSAuth.__init__(self, auth_method, gss_deleg_creds) + + if self._gss_deleg_creds: + self._gss_flags = ( + sspicon.ISC_REQ_INTEGRITY + | sspicon.ISC_REQ_MUTUAL_AUTH + | sspicon.ISC_REQ_DELEGATE + ) + else: + self._gss_flags = ( + sspicon.ISC_REQ_INTEGRITY | sspicon.ISC_REQ_MUTUAL_AUTH + ) + + def ssh_init_sec_context( + self, target, desired_mech=None, username=None, recv_token=None + ): + """ + Initialize a SSPI context. + + :param str username: The name of the user who attempts to login + :param str target: The FQDN of the target to connect to + :param str desired_mech: The negotiated SSPI mechanism + ("pseudo negotiated" mechanism, because we + support just the krb5 mechanism :-)) + :param recv_token: The SSPI token received from the Server + :raises: + `.SSHException` -- Is raised if the desired mechanism of the client + is not supported + :return: A ``String`` if the SSPI has returned a token or ``None`` if + no token was returned + """ + from pyasn1.codec.der import decoder + + self._username = username + self._gss_host = target + error = 0 + targ_name = "host/" + self._gss_host + if desired_mech is not None: + mech, __ = decoder.decode(desired_mech) + if mech.__str__() != self._krb5_mech: + raise SSHException("Unsupported mechanism OID.") + try: + if recv_token is None: + self._gss_ctxt = sspi.ClientAuth( + "Kerberos", scflags=self._gss_flags, targetspn=targ_name + ) + error, token = self._gss_ctxt.authorize(recv_token) + token = token[0].Buffer + except pywintypes.error as e: + e.strerror += ", Target: {}".format(self._gss_host) + raise + + if error == 0: + """ + if the status is GSS_COMPLETE (error = 0) the context is fully + established an we can set _gss_ctxt_status to True. + """ + self._gss_ctxt_status = True + token = None + """ + You won't get another token if the context is fully established, + so i set token to None instead of "" + """ + return token + + def ssh_get_mic(self, session_id, gss_kex=False): + """ + Create the MIC token for a SSH2 message. + + :param str session_id: The SSH session ID + :param bool gss_kex: Generate the MIC for Key Exchange with SSPI or not + :return: gssapi-with-mic: + Returns the MIC token from SSPI for the message we created + with ``_ssh_build_mic``. + gssapi-keyex: + Returns the MIC token from SSPI with the SSH session ID as + message. + """ + self._session_id = session_id + if not gss_kex: + mic_field = self._ssh_build_mic( + self._session_id, + self._username, + self._service, + self._auth_method, + ) + mic_token = self._gss_ctxt.sign(mic_field) + else: + # for key exchange with gssapi-keyex + mic_token = self._gss_srv_ctxt.sign(self._session_id) + return mic_token + + def ssh_accept_sec_context(self, hostname, username, recv_token): + """ + Accept a SSPI context (server mode). + + :param str hostname: The servers FQDN + :param str username: The name of the user who attempts to login + :param str recv_token: The SSPI Token received from the server, + if it's not the initial call. + :return: A ``String`` if the SSPI has returned a token or ``None`` if + no token was returned + """ + self._gss_host = hostname + self._username = username + targ_name = "host/" + self._gss_host + self._gss_srv_ctxt = sspi.ServerAuth("Kerberos", spn=targ_name) + error, token = self._gss_srv_ctxt.authorize(recv_token) + token = token[0].Buffer + if error == 0: + self._gss_srv_ctxt_status = True + token = None + return token + + def ssh_check_mic(self, mic_token, session_id, username=None): + """ + Verify the MIC token for a SSH2 message. + + :param str mic_token: The MIC token received from the client + :param str session_id: The SSH session ID + :param str username: The name of the user who attempts to login + :return: None if the MIC check was successful + :raises: ``sspi.error`` -- if the MIC check failed + """ + self._session_id = session_id + self._username = username + if username is not None: + # server mode + mic_field = self._ssh_build_mic( + self._session_id, + self._username, + self._service, + self._auth_method, + ) + # Verifies data and its signature. If verification fails, an + # sspi.error will be raised. + self._gss_srv_ctxt.verify(mic_field, mic_token) + else: + # for key exchange with gssapi-keyex + # client mode + # Verifies data and its signature. If verification fails, an + # sspi.error will be raised. + self._gss_ctxt.verify(self._session_id, mic_token) + + @property + def credentials_delegated(self): + """ + Checks if credentials are delegated (server mode). + + :return: ``True`` if credentials are delegated, otherwise ``False`` + """ + return self._gss_flags & sspicon.ISC_REQ_DELEGATE and ( + self._gss_srv_ctxt_status or self._gss_flags + ) + + def save_client_creds(self, client_token): + """ + Save the Client token in a file. This is used by the SSH server + to store the client credentials if credentials are delegated + (server mode). + + :param str client_token: The SSPI token received form the client + :raises: + ``NotImplementedError`` -- Credential delegation is currently not + supported in server mode + """ + raise NotImplementedError diff --git a/lib/paramiko/transport.py b/lib/paramiko/transport.py new file mode 100644 index 0000000..472ec6c --- /dev/null +++ b/lib/paramiko/transport.py @@ -0,0 +1,3456 @@ +# Copyright (C) 2003-2007 Robey Pointer +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Core protocol implementation +""" + +import os +import socket +import sys +import threading +import time +import weakref +from hashlib import md5, sha1, sha256, sha512 + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import ( + algorithms, + Cipher, + modes, + aead, +) + +import paramiko +from paramiko import util +from paramiko.auth_handler import AuthHandler, AuthOnlyHandler +from paramiko.ssh_gss import GSSAuth +from paramiko.channel import Channel +from paramiko.common import ( + xffffffff, + cMSG_CHANNEL_OPEN, + cMSG_IGNORE, + cMSG_GLOBAL_REQUEST, + DEBUG, + MSG_KEXINIT, + MSG_IGNORE, + MSG_DISCONNECT, + MSG_DEBUG, + ERROR, + WARNING, + cMSG_UNIMPLEMENTED, + INFO, + cMSG_KEXINIT, + cMSG_NEWKEYS, + MSG_NEWKEYS, + cMSG_REQUEST_SUCCESS, + cMSG_REQUEST_FAILURE, + CONNECTION_FAILED_CODE, + OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, + OPEN_SUCCEEDED, + cMSG_CHANNEL_OPEN_FAILURE, + cMSG_CHANNEL_OPEN_SUCCESS, + MSG_GLOBAL_REQUEST, + MSG_REQUEST_SUCCESS, + MSG_REQUEST_FAILURE, + cMSG_SERVICE_REQUEST, + MSG_SERVICE_ACCEPT, + MSG_CHANNEL_OPEN_SUCCESS, + MSG_CHANNEL_OPEN_FAILURE, + MSG_CHANNEL_OPEN, + MSG_CHANNEL_SUCCESS, + MSG_CHANNEL_FAILURE, + MSG_CHANNEL_DATA, + MSG_CHANNEL_EXTENDED_DATA, + MSG_CHANNEL_WINDOW_ADJUST, + MSG_CHANNEL_REQUEST, + MSG_CHANNEL_EOF, + MSG_CHANNEL_CLOSE, + MIN_WINDOW_SIZE, + MIN_PACKET_SIZE, + MAX_WINDOW_SIZE, + DEFAULT_WINDOW_SIZE, + DEFAULT_MAX_PACKET_SIZE, + HIGHEST_USERAUTH_MESSAGE_ID, + MSG_UNIMPLEMENTED, + MSG_NAMES, + MSG_EXT_INFO, + cMSG_EXT_INFO, + byte_ord, +) +from paramiko.compress import ZlibCompressor, ZlibDecompressor +from paramiko.ed25519key import Ed25519Key +from paramiko.kex_curve25519 import KexCurve25519 +from paramiko.kex_gex import KexGex, KexGexSHA256 +from paramiko.kex_group1 import KexGroup1 +from paramiko.kex_group14 import KexGroup14, KexGroup14SHA256 +from paramiko.kex_group16 import KexGroup16SHA512 +from paramiko.kex_ecdh_nist import KexNistp256, KexNistp384, KexNistp521 +from paramiko.kex_gss import KexGSSGex, KexGSSGroup1, KexGSSGroup14 +from paramiko.message import Message +from paramiko.packet import Packetizer, NeedRekeyException +from paramiko.primes import ModulusPack +from paramiko.rsakey import RSAKey +from paramiko.ecdsakey import ECDSAKey +from paramiko.server import ServerInterface +from paramiko.sftp_client import SFTPClient +from paramiko.ssh_exception import ( + BadAuthenticationType, + ChannelException, + IncompatiblePeer, + MessageOrderError, + ProxyCommandFailure, + SSHException, +) +from paramiko.util import ( + ClosingContextManager, + clamp_value, + b, +) + + +# TripleDES is moving from `cryptography.hazmat.primitives.ciphers.algorithms` +# in cryptography>=43.0.0 to `cryptography.hazmat.decrepit.ciphers.algorithms` +# It will be removed from `cryptography.hazmat.primitives.ciphers.algorithms` +# in cryptography==48.0.0. +# +# Source References: +# - https://github.com/pyca/cryptography/commit/722a6393e61b3ac +# - https://github.com/pyca/cryptography/pull/11407/files +try: + from cryptography.hazmat.decrepit.ciphers.algorithms import TripleDES +except ImportError: + from cryptography.hazmat.primitives.ciphers.algorithms import TripleDES + + +# for thread cleanup +_active_threads = [] + + +def _join_lingering_threads(): + for thr in _active_threads: + thr.stop_thread() + + +import atexit + +atexit.register(_join_lingering_threads) + + +class Transport(threading.Thread, ClosingContextManager): + """ + An SSH Transport attaches to a stream (usually a socket), negotiates an + encrypted session, authenticates, and then creates stream tunnels, called + `channels <.Channel>`, across the session. Multiple channels can be + multiplexed across a single session (and often are, in the case of port + forwardings). + + Instances of this class may be used as context managers. + """ + + _ENCRYPT = object() + _DECRYPT = object() + + _PROTO_ID = "2.0" + _CLIENT_ID = "paramiko_{}".format(paramiko.__version__) + + # These tuples of algorithm identifiers are in preference order; do not + # reorder without reason! + # NOTE: if you need to modify these, we suggest leveraging the + # `disabled_algorithms` constructor argument (also available in SSHClient) + # instead of monkeypatching or subclassing. + _preferred_ciphers = ( + "aes128-ctr", + "aes192-ctr", + "aes256-ctr", + "aes128-cbc", + "aes192-cbc", + "aes256-cbc", + "3des-cbc", + "aes128-gcm@openssh.com", + "aes256-gcm@openssh.com", + ) + _preferred_macs = ( + "hmac-sha2-256", + "hmac-sha2-512", + "hmac-sha2-256-etm@openssh.com", + "hmac-sha2-512-etm@openssh.com", + "hmac-sha1", + "hmac-md5", + "hmac-sha1-96", + "hmac-md5-96", + ) + # ~= HostKeyAlgorithms in OpenSSH land + _preferred_keys = ( + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256", + "ssh-rsa", + ) + # ~= PubKeyAcceptedAlgorithms + _preferred_pubkeys = ( + "ssh-ed25519", + "ecdsa-sha2-nistp256", + "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp521", + "rsa-sha2-512", + "rsa-sha2-256", + "ssh-rsa", + ) + _preferred_kex = ( + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "ecdh-sha2-nistp521", + "diffie-hellman-group16-sha512", + "diffie-hellman-group-exchange-sha256", + "diffie-hellman-group14-sha256", + "diffie-hellman-group-exchange-sha1", + "diffie-hellman-group14-sha1", + "diffie-hellman-group1-sha1", + ) + if KexCurve25519.is_available(): + _preferred_kex = ("curve25519-sha256@libssh.org",) + _preferred_kex + _preferred_gsskex = ( + "gss-gex-sha1-toWM5Slw5Ew8Mqkay+al2g==", + "gss-group14-sha1-toWM5Slw5Ew8Mqkay+al2g==", + "gss-group1-sha1-toWM5Slw5Ew8Mqkay+al2g==", + ) + _preferred_compression = ("none",) + + _cipher_info = { + "aes128-ctr": { + "class": algorithms.AES, + "mode": modes.CTR, + "block-size": 16, + "key-size": 16, + }, + "aes192-ctr": { + "class": algorithms.AES, + "mode": modes.CTR, + "block-size": 16, + "key-size": 24, + }, + "aes256-ctr": { + "class": algorithms.AES, + "mode": modes.CTR, + "block-size": 16, + "key-size": 32, + }, + "aes128-cbc": { + "class": algorithms.AES, + "mode": modes.CBC, + "block-size": 16, + "key-size": 16, + }, + "aes192-cbc": { + "class": algorithms.AES, + "mode": modes.CBC, + "block-size": 16, + "key-size": 24, + }, + "aes256-cbc": { + "class": algorithms.AES, + "mode": modes.CBC, + "block-size": 16, + "key-size": 32, + }, + "3des-cbc": { + "class": TripleDES, + "mode": modes.CBC, + "block-size": 8, + "key-size": 24, + }, + "aes128-gcm@openssh.com": { + "class": aead.AESGCM, + "block-size": 16, + "iv-size": 12, + "key-size": 16, + "is_aead": True, + }, + "aes256-gcm@openssh.com": { + "class": aead.AESGCM, + "block-size": 16, + "iv-size": 12, + "key-size": 32, + "is_aead": True, + }, + } + + _mac_info = { + "hmac-sha1": {"class": sha1, "size": 20}, + "hmac-sha1-96": {"class": sha1, "size": 12}, + "hmac-sha2-256": {"class": sha256, "size": 32}, + "hmac-sha2-256-etm@openssh.com": {"class": sha256, "size": 32}, + "hmac-sha2-512": {"class": sha512, "size": 64}, + "hmac-sha2-512-etm@openssh.com": {"class": sha512, "size": 64}, + "hmac-md5": {"class": md5, "size": 16}, + "hmac-md5-96": {"class": md5, "size": 12}, + } + + _key_info = { + # TODO: at some point we will want to drop this as it's no longer + # considered secure due to using SHA-1 for signatures. OpenSSH 8.8 no + # longer supports it. Question becomes at what point do we want to + # prevent users with older setups from using this? + "ssh-rsa": RSAKey, + "ssh-rsa-cert-v01@openssh.com": RSAKey, + "rsa-sha2-256": RSAKey, + "rsa-sha2-256-cert-v01@openssh.com": RSAKey, + "rsa-sha2-512": RSAKey, + "rsa-sha2-512-cert-v01@openssh.com": RSAKey, + "ecdsa-sha2-nistp256": ECDSAKey, + "ecdsa-sha2-nistp256-cert-v01@openssh.com": ECDSAKey, + "ecdsa-sha2-nistp384": ECDSAKey, + "ecdsa-sha2-nistp384-cert-v01@openssh.com": ECDSAKey, + "ecdsa-sha2-nistp521": ECDSAKey, + "ecdsa-sha2-nistp521-cert-v01@openssh.com": ECDSAKey, + "ssh-ed25519": Ed25519Key, + "ssh-ed25519-cert-v01@openssh.com": Ed25519Key, + } + + _kex_info = { + "diffie-hellman-group1-sha1": KexGroup1, + "diffie-hellman-group14-sha1": KexGroup14, + "diffie-hellman-group-exchange-sha1": KexGex, + "diffie-hellman-group-exchange-sha256": KexGexSHA256, + "diffie-hellman-group14-sha256": KexGroup14SHA256, + "diffie-hellman-group16-sha512": KexGroup16SHA512, + "gss-group1-sha1-toWM5Slw5Ew8Mqkay+al2g==": KexGSSGroup1, + "gss-group14-sha1-toWM5Slw5Ew8Mqkay+al2g==": KexGSSGroup14, + "gss-gex-sha1-toWM5Slw5Ew8Mqkay+al2g==": KexGSSGex, + "ecdh-sha2-nistp256": KexNistp256, + "ecdh-sha2-nistp384": KexNistp384, + "ecdh-sha2-nistp521": KexNistp521, + } + if KexCurve25519.is_available(): + _kex_info["curve25519-sha256@libssh.org"] = KexCurve25519 + + _compression_info = { + # zlib@openssh.com is just zlib, but only turned on after a successful + # authentication. openssh servers may only offer this type because + # they've had troubles with security holes in zlib in the past. + "zlib@openssh.com": (ZlibCompressor, ZlibDecompressor), + "zlib": (ZlibCompressor, ZlibDecompressor), + "none": (None, None), + } + + _modulus_pack = None + _active_check_timeout = 0.1 + + def __init__( + self, + sock, + default_window_size=DEFAULT_WINDOW_SIZE, + default_max_packet_size=DEFAULT_MAX_PACKET_SIZE, + gss_kex=False, + gss_deleg_creds=True, + disabled_algorithms=None, + server_sig_algs=True, + strict_kex=True, + packetizer_class=None, + ): + """ + Create a new SSH session over an existing socket, or socket-like + object. This only creates the `.Transport` object; it doesn't begin + the SSH session yet. Use `connect` or `start_client` to begin a client + session, or `start_server` to begin a server session. + + If the object is not actually a socket, it must have the following + methods: + + - ``send(bytes)``: Writes from 1 to ``len(bytes)`` bytes, and returns + an int representing the number of bytes written. Returns + 0 or raises ``EOFError`` if the stream has been closed. + - ``recv(int)``: Reads from 1 to ``int`` bytes and returns them as a + string. Returns 0 or raises ``EOFError`` if the stream has been + closed. + - ``close()``: Closes the socket. + - ``settimeout(n)``: Sets a (float) timeout on I/O operations. + + For ease of use, you may also pass in an address (as a tuple) or a host + string as the ``sock`` argument. (A host string is a hostname with an + optional port (separated by ``":"``) which will be converted into a + tuple of ``(hostname, port)``.) A socket will be connected to this + address and used for communication. Exceptions from the ``socket`` + call may be thrown in this case. + + .. note:: + Modifying the the window and packet sizes might have adverse + effects on your channels created from this transport. The default + values are the same as in the OpenSSH code base and have been + battle tested. + + :param socket sock: + a socket or socket-like object to create the session over. + :param int default_window_size: + sets the default window size on the transport. (defaults to + 2097152) + :param int default_max_packet_size: + sets the default max packet size on the transport. (defaults to + 32768) + :param bool gss_kex: + Whether to enable GSSAPI key exchange when GSSAPI is in play. + Default: ``False``. + :param bool gss_deleg_creds: + Whether to enable GSSAPI credential delegation when GSSAPI is in + play. Default: ``True``. + :param dict disabled_algorithms: + If given, must be a dictionary mapping algorithm type to an + iterable of algorithm identifiers, which will be disabled for the + lifetime of the transport. + + Keys should match the last word in the class' builtin algorithm + tuple attributes, such as ``"ciphers"`` to disable names within + ``_preferred_ciphers``; or ``"kex"`` to disable something defined + inside ``_preferred_kex``. Values should exactly match members of + the matching attribute. + + For example, if you need to disable + ``diffie-hellman-group16-sha512`` key exchange (perhaps because + your code talks to a server which implements it differently from + Paramiko), specify ``disabled_algorithms={"kex": + ["diffie-hellman-group16-sha512"]}``. + :param bool server_sig_algs: + Whether to send an extra message to compatible clients, in server + mode, with a list of supported pubkey algorithms. Default: + ``True``. + :param bool strict_kex: + Whether to advertise (and implement, if client also advertises + support for) a "strict kex" mode for safer handshaking. Default: + ``True``. + :param packetizer_class: + Which class to use for instantiating the internal packet handler. + Default: ``None`` (i.e.: use `Packetizer` as normal). + + .. versionchanged:: 1.15 + Added the ``default_window_size`` and ``default_max_packet_size`` + arguments. + .. versionchanged:: 1.15 + Added the ``gss_kex`` and ``gss_deleg_creds`` kwargs. + .. versionchanged:: 2.6 + Added the ``disabled_algorithms`` kwarg. + .. versionchanged:: 2.9 + Added the ``server_sig_algs`` kwarg. + .. versionchanged:: 3.4 + Added the ``strict_kex`` kwarg. + .. versionchanged:: 3.4 + Added the ``packetizer_class`` kwarg. + """ + self.active = False + self.hostname = None + self.server_extensions = {} + self.advertise_strict_kex = strict_kex + self.agreed_on_strict_kex = False + + # TODO: these two overrides on sock's type should go away sometime, too + # many ways to do it! + if isinstance(sock, str): + # convert "host:port" into (host, port) + hl = sock.split(":", 1) + self.hostname = hl[0] + if len(hl) == 1: + sock = (hl[0], 22) + else: + sock = (hl[0], int(hl[1])) + if type(sock) is tuple: + # connect to the given (host, port) + hostname, port = sock + self.hostname = hostname + reason = "No suitable address family" + addrinfos = socket.getaddrinfo( + hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM + ) + for family, socktype, proto, canonname, sockaddr in addrinfos: + if socktype == socket.SOCK_STREAM: + af = family + # addr = sockaddr + sock = socket.socket(af, socket.SOCK_STREAM) + try: + sock.connect((hostname, port)) + except socket.error as e: + reason = str(e) + else: + break + else: + raise SSHException( + "Unable to connect to {}: {}".format(hostname, reason) + ) + # okay, normal socket-ish flow here... + threading.Thread.__init__(self) + self.daemon = True + self.sock = sock + # we set the timeout so we can check self.active periodically to + # see if we should bail. socket.timeout exception is never propagated. + self.sock.settimeout(self._active_check_timeout) + + # negotiated crypto parameters + self.packetizer = (packetizer_class or Packetizer)(sock) + self.local_version = "SSH-" + self._PROTO_ID + "-" + self._CLIENT_ID + self.remote_version = "" + self.local_cipher = self.remote_cipher = "" + self.local_kex_init = self.remote_kex_init = None + self.local_mac = self.remote_mac = None + self.local_compression = self.remote_compression = None + self.session_id = None + self.host_key_type = None + self.host_key = None + + # GSS-API / SSPI Key Exchange + self.use_gss_kex = gss_kex + # This will be set to True if GSS-API Key Exchange was performed + self.gss_kex_used = False + self.kexgss_ctxt = None + self.gss_host = None + if self.use_gss_kex: + self.kexgss_ctxt = GSSAuth("gssapi-keyex", gss_deleg_creds) + self._preferred_kex = self._preferred_gsskex + self._preferred_kex + + # state used during negotiation + self.kex_engine = None + self.H = None + self.K = None + + self.initial_kex_done = False + self.in_kex = False + self.authenticated = False + self._expected_packet = tuple() + # synchronization (always higher level than write_lock) + self.lock = threading.Lock() + + # tracking open channels + self._channels = ChannelMap() + self.channel_events = {} # (id -> Event) + self.channels_seen = {} # (id -> True) + self._channel_counter = 0 + self.default_max_packet_size = default_max_packet_size + self.default_window_size = default_window_size + self._forward_agent_handler = None + self._x11_handler = None + self._tcp_handler = None + + self.saved_exception = None + self.clear_to_send = threading.Event() + self.clear_to_send_lock = threading.Lock() + self.clear_to_send_timeout = 30.0 + self.log_name = "paramiko.transport" + self.logger = util.get_logger(self.log_name) + self.packetizer.set_log(self.logger) + self.auth_handler = None + # response Message from an arbitrary global request + self.global_response = None + # user-defined event callbacks + self.completion_event = None + # how long (seconds) to wait for the SSH banner + self.banner_timeout = 15 + # how long (seconds) to wait for the handshake to finish after SSH + # banner sent. + self.handshake_timeout = 15 + # how long (seconds) to wait for the auth response. + self.auth_timeout = 30 + # how long (seconds) to wait for opening a channel + self.channel_timeout = 60 * 60 + self.disabled_algorithms = disabled_algorithms or {} + self.server_sig_algs = server_sig_algs + + # server mode: + self.server_mode = False + self.server_object = None + self.server_key_dict = {} + self.server_accepts = [] + self.server_accept_cv = threading.Condition(self.lock) + self.subsystem_table = {} + + # Handler table, now set at init time for easier per-instance + # manipulation and subclass twiddling. + self._handler_table = { + MSG_EXT_INFO: self._parse_ext_info, + MSG_NEWKEYS: self._parse_newkeys, + MSG_GLOBAL_REQUEST: self._parse_global_request, + MSG_REQUEST_SUCCESS: self._parse_request_success, + MSG_REQUEST_FAILURE: self._parse_request_failure, + MSG_CHANNEL_OPEN_SUCCESS: self._parse_channel_open_success, + MSG_CHANNEL_OPEN_FAILURE: self._parse_channel_open_failure, + MSG_CHANNEL_OPEN: self._parse_channel_open, + MSG_KEXINIT: self._negotiate_keys, + } + + def _filter_algorithm(self, type_): + default = getattr(self, "_preferred_{}".format(type_)) + return tuple( + x + for x in default + if x not in self.disabled_algorithms.get(type_, []) + ) + + @property + def preferred_ciphers(self): + return self._filter_algorithm("ciphers") + + @property + def preferred_macs(self): + return self._filter_algorithm("macs") + + @property + def preferred_keys(self): + # Interleave cert variants here; resistant to various background + # overwriting of _preferred_keys, and necessary as hostkeys can't use + # the logic pubkey auth does re: injecting/checking for certs at + # runtime + filtered = self._filter_algorithm("keys") + return tuple( + filtered + + tuple("{}-cert-v01@openssh.com".format(x) for x in filtered) + ) + + @property + def preferred_pubkeys(self): + return self._filter_algorithm("pubkeys") + + @property + def preferred_kex(self): + return self._filter_algorithm("kex") + + @property + def preferred_compression(self): + return self._filter_algorithm("compression") + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + """ + id_ = hex(id(self) & xffffffff) + out = "` or + `auth_publickey `. + + .. note:: `connect` is a simpler method for connecting as a client. + + .. note:: + After calling this method (or `start_server` or `connect`), you + should no longer directly read from or write to the original socket + object. + + :param .threading.Event event: + an event to trigger when negotiation is complete (optional) + + :param float timeout: + a timeout, in seconds, for SSH2 session negotiation (optional) + + :raises: + `.SSHException` -- if negotiation fails (and no ``event`` was + passed in) + """ + self.active = True + if event is not None: + # async, return immediately and let the app poll for completion + self.completion_event = event + self.start() + return + + # synchronous, wait for a result + self.completion_event = event = threading.Event() + self.start() + max_time = time.time() + timeout if timeout is not None else None + while True: + event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is not None: + raise e + raise SSHException("Negotiation failed.") + if event.is_set() or ( + timeout is not None and time.time() >= max_time + ): + break + + def start_server(self, event=None, server=None): + """ + Negotiate a new SSH2 session as a server. This is the first step after + creating a new `.Transport` and setting up your server host key(s). A + separate thread is created for protocol negotiation. + + If an event is passed in, this method returns immediately. When + negotiation is done (successful or not), the given ``Event`` will + be triggered. On failure, `is_active` will return ``False``. + + (Since 1.4) If ``event`` is ``None``, this method will not return until + negotiation is done. On success, the method returns normally. + Otherwise an SSHException is raised. + + After a successful negotiation, the client will need to authenticate. + Override the methods `get_allowed_auths + <.ServerInterface.get_allowed_auths>`, `check_auth_none + <.ServerInterface.check_auth_none>`, `check_auth_password + <.ServerInterface.check_auth_password>`, and `check_auth_publickey + <.ServerInterface.check_auth_publickey>` in the given ``server`` object + to control the authentication process. + + After a successful authentication, the client should request to open a + channel. Override `check_channel_request + <.ServerInterface.check_channel_request>` in the given ``server`` + object to allow channels to be opened. + + .. note:: + After calling this method (or `start_client` or `connect`), you + should no longer directly read from or write to the original socket + object. + + :param .threading.Event event: + an event to trigger when negotiation is complete. + :param .ServerInterface server: + an object used to perform authentication and create `channels + <.Channel>` + + :raises: + `.SSHException` -- if negotiation fails (and no ``event`` was + passed in) + """ + if server is None: + server = ServerInterface() + self.server_mode = True + self.server_object = server + self.active = True + if event is not None: + # async, return immediately and let the app poll for completion + self.completion_event = event + self.start() + return + + # synchronous, wait for a result + self.completion_event = event = threading.Event() + self.start() + while True: + event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is not None: + raise e + raise SSHException("Negotiation failed.") + if event.is_set(): + break + + def add_server_key(self, key): + """ + Add a host key to the list of keys used for server mode. When behaving + as a server, the host key is used to sign certain packets during the + SSH2 negotiation, so that the client can trust that we are who we say + we are. Because this is used for signing, the key must contain private + key info, not just the public half. Only one key of each type is kept. + + :param .PKey key: + the host key (instance of some subclass) to add + """ + self.server_key_dict[key.get_name()] = key + # Handle SHA-2 extensions for RSA by ensuring that lookups into + # self.server_key_dict will yield this key for any of the algorithm + # names. + if isinstance(key, RSAKey): + self.server_key_dict["rsa-sha2-256"] = key + self.server_key_dict["rsa-sha2-512"] = key + + def get_server_key(self): + """ + Return the active host key, in server mode. After negotiating with the + client, this method will return the negotiated host key. If only one + type of host key was set with `add_server_key`, that's the only key + that will ever be returned. But in cases where you have set more than + one type of host key, the key type will be negotiated by the client, + and this method will return the key of the type agreed on. If the host + key has not been negotiated yet, ``None`` is returned. In client mode, + the behavior is undefined. + + :return: + host key (`.PKey`) of the type negotiated by the client, or + ``None``. + """ + try: + return self.server_key_dict[self.host_key_type] + except KeyError: + pass + return None + + @staticmethod + def load_server_moduli(filename=None): + """ + (optional) + Load a file of prime moduli for use in doing group-exchange key + negotiation in server mode. It's a rather obscure option and can be + safely ignored. + + In server mode, the remote client may request "group-exchange" key + negotiation, which asks the server to send a random prime number that + fits certain criteria. These primes are pretty difficult to compute, + so they can't be generated on demand. But many systems contain a file + of suitable primes (usually named something like ``/etc/ssh/moduli``). + If you call `load_server_moduli` and it returns ``True``, then this + file of primes has been loaded and we will support "group-exchange" in + server mode. Otherwise server mode will just claim that it doesn't + support that method of key negotiation. + + :param str filename: + optional path to the moduli file, if you happen to know that it's + not in a standard location. + :return: + True if a moduli file was successfully loaded; False otherwise. + + .. note:: This has no effect when used in client mode. + """ + Transport._modulus_pack = ModulusPack() + # places to look for the openssh "moduli" file + file_list = ["/etc/ssh/moduli", "/usr/local/etc/moduli"] + if filename is not None: + file_list.insert(0, filename) + for fn in file_list: + try: + Transport._modulus_pack.read_file(fn) + return True + except IOError: + pass + # none succeeded + Transport._modulus_pack = None + return False + + def close(self): + """ + Close this session, and any open channels that are tied to it. + """ + if not self.active: + return + self.stop_thread() + for chan in list(self._channels.values()): + chan._unlink() + self.sock.close() + + def get_remote_server_key(self): + """ + Return the host key of the server (in client mode). + + .. note:: + Previously this call returned a tuple of ``(key type, key + string)``. You can get the same effect by calling `.PKey.get_name` + for the key type, and ``str(key)`` for the key string. + + :raises: `.SSHException` -- if no session is currently active. + + :return: public key (`.PKey`) of the remote server + """ + if (not self.active) or (not self.initial_kex_done): + raise SSHException("No existing session") + return self.host_key + + def is_active(self): + """ + Return true if this session is active (open). + + :return: + True if the session is still active (open); False if the session is + closed + """ + return self.active + + def open_session( + self, window_size=None, max_packet_size=None, timeout=None + ): + """ + Request a new channel to the server, of type ``"session"``. This is + just an alias for calling `open_channel` with an argument of + ``"session"``. + + .. note:: Modifying the the window and packet sizes might have adverse + effects on the session created. The default values are the same + as in the OpenSSH code base and have been battle tested. + + :param int window_size: + optional window size for this session. + :param int max_packet_size: + optional max packet size for this session. + + :return: a new `.Channel` + + :raises: + `.SSHException` -- if the request is rejected or the session ends + prematurely + + .. versionchanged:: 1.13.4/1.14.3/1.15.3 + Added the ``timeout`` argument. + .. versionchanged:: 1.15 + Added the ``window_size`` and ``max_packet_size`` arguments. + """ + return self.open_channel( + "session", + window_size=window_size, + max_packet_size=max_packet_size, + timeout=timeout, + ) + + def open_x11_channel(self, src_addr=None): + """ + Request a new channel to the client, of type ``"x11"``. This + is just an alias for ``open_channel('x11', src_addr=src_addr)``. + + :param tuple src_addr: + the source address (``(str, int)``) of the x11 server (port is the + x11 port, ie. 6010) + :return: a new `.Channel` + + :raises: + `.SSHException` -- if the request is rejected or the session ends + prematurely + """ + return self.open_channel("x11", src_addr=src_addr) + + def open_forward_agent_channel(self): + """ + Request a new channel to the client, of type + ``"auth-agent@openssh.com"``. + + This is just an alias for ``open_channel('auth-agent@openssh.com')``. + + :return: a new `.Channel` + + :raises: `.SSHException` -- + if the request is rejected or the session ends prematurely + """ + return self.open_channel("auth-agent@openssh.com") + + def open_forwarded_tcpip_channel(self, src_addr, dest_addr): + """ + Request a new channel back to the client, of type ``forwarded-tcpip``. + + This is used after a client has requested port forwarding, for sending + incoming connections back to the client. + + :param src_addr: originator's address + :param dest_addr: local (server) connected address + """ + return self.open_channel("forwarded-tcpip", dest_addr, src_addr) + + def open_channel( + self, + kind, + dest_addr=None, + src_addr=None, + window_size=None, + max_packet_size=None, + timeout=None, + ): + """ + Request a new channel to the server. `Channels <.Channel>` are + socket-like objects used for the actual transfer of data across the + session. You may only request a channel after negotiating encryption + (using `connect` or `start_client`) and authenticating. + + .. note:: Modifying the the window and packet sizes might have adverse + effects on the channel created. The default values are the same + as in the OpenSSH code base and have been battle tested. + + :param str kind: + the kind of channel requested (usually ``"session"``, + ``"forwarded-tcpip"``, ``"direct-tcpip"``, or ``"x11"``) + :param tuple dest_addr: + the destination address (address + port tuple) of this port + forwarding, if ``kind`` is ``"forwarded-tcpip"`` or + ``"direct-tcpip"`` (ignored for other channel types) + :param src_addr: the source address of this port forwarding, if + ``kind`` is ``"forwarded-tcpip"``, ``"direct-tcpip"``, or ``"x11"`` + :param int window_size: + optional window size for this session. + :param int max_packet_size: + optional max packet size for this session. + :param float timeout: + optional timeout opening a channel, default 3600s (1h) + + :return: a new `.Channel` on success + + :raises: + `.SSHException` -- if the request is rejected, the session ends + prematurely or there is a timeout opening a channel + + .. versionchanged:: 1.15 + Added the ``window_size`` and ``max_packet_size`` arguments. + """ + if not self.active: + raise SSHException("SSH session not active") + timeout = self.channel_timeout if timeout is None else timeout + self.lock.acquire() + try: + window_size = self._sanitize_window_size(window_size) + max_packet_size = self._sanitize_packet_size(max_packet_size) + chanid = self._next_channel() + m = Message() + m.add_byte(cMSG_CHANNEL_OPEN) + m.add_string(kind) + m.add_int(chanid) + m.add_int(window_size) + m.add_int(max_packet_size) + if (kind == "forwarded-tcpip") or (kind == "direct-tcpip"): + m.add_string(dest_addr[0]) + m.add_int(dest_addr[1]) + m.add_string(src_addr[0]) + m.add_int(src_addr[1]) + elif kind == "x11": + m.add_string(src_addr[0]) + m.add_int(src_addr[1]) + chan = Channel(chanid) + self._channels.put(chanid, chan) + self.channel_events[chanid] = event = threading.Event() + self.channels_seen[chanid] = True + chan._set_transport(self) + chan._set_window(window_size, max_packet_size) + finally: + self.lock.release() + self._send_user_message(m) + start_ts = time.time() + while True: + event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is None: + e = SSHException("Unable to open channel.") + raise e + if event.is_set(): + break + elif start_ts + timeout < time.time(): + raise SSHException("Timeout opening channel.") + chan = self._channels.get(chanid) + if chan is not None: + return chan + e = self.get_exception() + if e is None: + e = SSHException("Unable to open channel.") + raise e + + def request_port_forward(self, address, port, handler=None): + """ + Ask the server to forward TCP connections from a listening port on + the server, across this SSH session. + + If a handler is given, that handler is called from a different thread + whenever a forwarded connection arrives. The handler parameters are:: + + handler( + channel, + (origin_addr, origin_port), + (server_addr, server_port), + ) + + where ``server_addr`` and ``server_port`` are the address and port that + the server was listening on. + + If no handler is set, the default behavior is to send new incoming + forwarded connections into the accept queue, to be picked up via + `accept`. + + :param str address: the address to bind when forwarding + :param int port: + the port to forward, or 0 to ask the server to allocate any port + :param callable handler: + optional handler for incoming forwarded connections, of the form + ``func(Channel, (str, int), (str, int))``. + + :return: the port number (`int`) allocated by the server + + :raises: + `.SSHException` -- if the server refused the TCP forward request + """ + if not self.active: + raise SSHException("SSH session not active") + port = int(port) + response = self.global_request( + "tcpip-forward", (address, port), wait=True + ) + if response is None: + raise SSHException("TCP forwarding request denied") + if port == 0: + port = response.get_int() + if handler is None: + + def default_handler(channel, src_addr, dest_addr_port): + # src_addr, src_port = src_addr_port + # dest_addr, dest_port = dest_addr_port + self._queue_incoming_channel(channel) + + handler = default_handler + self._tcp_handler = handler + return port + + def cancel_port_forward(self, address, port): + """ + Ask the server to cancel a previous port-forwarding request. No more + connections to the given address & port will be forwarded across this + ssh connection. + + :param str address: the address to stop forwarding + :param int port: the port to stop forwarding + """ + if not self.active: + return + self._tcp_handler = None + self.global_request("cancel-tcpip-forward", (address, port), wait=True) + + def open_sftp_client(self): + """ + Create an SFTP client channel from an open transport. On success, an + SFTP session will be opened with the remote host, and a new + `.SFTPClient` object will be returned. + + :return: + a new `.SFTPClient` referring to an sftp session (channel) across + this transport + """ + return SFTPClient.from_transport(self) + + def send_ignore(self, byte_count=None): + """ + Send a junk packet across the encrypted link. This is sometimes used + to add "noise" to a connection to confuse would-be attackers. It can + also be used as a keep-alive for long lived connections traversing + firewalls. + + :param int byte_count: + the number of random bytes to send in the payload of the ignored + packet -- defaults to a random number from 10 to 41. + """ + m = Message() + m.add_byte(cMSG_IGNORE) + if byte_count is None: + byte_count = (byte_ord(os.urandom(1)) % 32) + 10 + m.add_bytes(os.urandom(byte_count)) + self._send_user_message(m) + + def renegotiate_keys(self): + """ + Force this session to switch to new keys. Normally this is done + automatically after the session hits a certain number of packets or + bytes sent or received, but this method gives you the option of forcing + new keys whenever you want. Negotiating new keys causes a pause in + traffic both ways as the two sides swap keys and do computations. This + method returns when the session has switched to new keys. + + :raises: + `.SSHException` -- if the key renegotiation failed (which causes + the session to end) + """ + self.completion_event = threading.Event() + self._send_kex_init() + while True: + self.completion_event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is not None: + raise e + raise SSHException("Negotiation failed.") + if self.completion_event.is_set(): + break + return + + def set_keepalive(self, interval): + """ + Turn on/off keepalive packets (default is off). If this is set, after + ``interval`` seconds without sending any data over the connection, a + "keepalive" packet will be sent (and ignored by the remote host). This + can be useful to keep connections alive over a NAT, for example. + + :param int interval: + seconds to wait before sending a keepalive packet (or + 0 to disable keepalives). + """ + + def _request(x=weakref.proxy(self)): + return x.global_request("keepalive@lag.net", wait=False) + + self.packetizer.set_keepalive(interval, _request) + + def global_request(self, kind, data=None, wait=True): + """ + Make a global request to the remote host. These are normally + extensions to the SSH2 protocol. + + :param str kind: name of the request. + :param tuple data: + an optional tuple containing additional data to attach to the + request. + :param bool wait: + ``True`` if this method should not return until a response is + received; ``False`` otherwise. + :return: + a `.Message` containing possible additional data if the request was + successful (or an empty `.Message` if ``wait`` was ``False``); + ``None`` if the request was denied. + """ + if wait: + self.completion_event = threading.Event() + m = Message() + m.add_byte(cMSG_GLOBAL_REQUEST) + m.add_string(kind) + m.add_boolean(wait) + if data is not None: + m.add(*data) + self._log(DEBUG, 'Sending global request "{}"'.format(kind)) + self._send_user_message(m) + if not wait: + return None + while True: + self.completion_event.wait(0.1) + if not self.active: + return None + if self.completion_event.is_set(): + break + return self.global_response + + def accept(self, timeout=None): + """ + Return the next channel opened by the client over this transport, in + server mode. If no channel is opened before the given timeout, + ``None`` is returned. + + :param int timeout: + seconds to wait for a channel, or ``None`` to wait forever + :return: a new `.Channel` opened by the client + """ + self.lock.acquire() + try: + if len(self.server_accepts) > 0: + chan = self.server_accepts.pop(0) + else: + self.server_accept_cv.wait(timeout) + if len(self.server_accepts) > 0: + chan = self.server_accepts.pop(0) + else: + # timeout + chan = None + finally: + self.lock.release() + return chan + + def connect( + self, + hostkey=None, + username="", + password=None, + pkey=None, + gss_host=None, + gss_auth=False, + gss_kex=False, + gss_deleg_creds=True, + gss_trust_dns=True, + ): + """ + Negotiate an SSH2 session, and optionally verify the server's host key + and authenticate using a password or private key. This is a shortcut + for `start_client`, `get_remote_server_key`, and + `Transport.auth_password` or `Transport.auth_publickey`. Use those + methods if you want more control. + + You can use this method immediately after creating a Transport to + negotiate encryption with a server. If it fails, an exception will be + thrown. On success, the method will return cleanly, and an encrypted + session exists. You may immediately call `open_channel` or + `open_session` to get a `.Channel` object, which is used for data + transfer. + + .. note:: + If you fail to supply a password or private key, this method may + succeed, but a subsequent `open_channel` or `open_session` call may + fail because you haven't authenticated yet. + + :param .PKey hostkey: + the host key expected from the server, or ``None`` if you don't + want to do host key verification. + :param str username: the username to authenticate as. + :param str password: + a password to use for authentication, if you want to use password + authentication; otherwise ``None``. + :param .PKey pkey: + a private key to use for authentication, if you want to use private + key authentication; otherwise ``None``. + :param str gss_host: + The target's name in the kerberos database. Default: hostname + :param bool gss_auth: + ``True`` if you want to use GSS-API authentication. + :param bool gss_kex: + Perform GSS-API Key Exchange and user authentication. + :param bool gss_deleg_creds: + Whether to delegate GSS-API client credentials. + :param gss_trust_dns: + Indicates whether or not the DNS is trusted to securely + canonicalize the name of the host being connected to (default + ``True``). + + :raises: `.SSHException` -- if the SSH2 negotiation fails, the host key + supplied by the server is incorrect, or authentication fails. + + .. versionchanged:: 2.3 + Added the ``gss_trust_dns`` argument. + """ + if hostkey is not None: + # TODO: a more robust implementation would be to ask each key class + # for its nameS plural, and just use that. + # TODO: that could be used in a bunch of other spots too + if isinstance(hostkey, RSAKey): + self._preferred_keys = [ + "rsa-sha2-512", + "rsa-sha2-256", + "ssh-rsa", + ] + else: + self._preferred_keys = [hostkey.get_name()] + + self.set_gss_host( + gss_host=gss_host, + trust_dns=gss_trust_dns, + gssapi_requested=gss_kex or gss_auth, + ) + + self.start_client() + + # check host key if we were given one + # If GSS-API Key Exchange was performed, we are not required to check + # the host key. + if (hostkey is not None) and not gss_kex: + key = self.get_remote_server_key() + if ( + key.get_name() != hostkey.get_name() + or key.asbytes() != hostkey.asbytes() + ): + self._log(DEBUG, "Bad host key from server") + self._log( + DEBUG, + "Expected: {}: {}".format( + hostkey.get_name(), repr(hostkey.asbytes()) + ), + ) + self._log( + DEBUG, + "Got : {}: {}".format( + key.get_name(), repr(key.asbytes()) + ), + ) + raise SSHException("Bad host key from server") + self._log( + DEBUG, "Host key verified ({})".format(hostkey.get_name()) + ) + + if (pkey is not None) or (password is not None) or gss_auth or gss_kex: + if gss_auth: + self._log( + DEBUG, "Attempting GSS-API auth... (gssapi-with-mic)" + ) # noqa + self.auth_gssapi_with_mic( + username, self.gss_host, gss_deleg_creds + ) + elif gss_kex: + self._log(DEBUG, "Attempting GSS-API auth... (gssapi-keyex)") + self.auth_gssapi_keyex(username) + elif pkey is not None: + self._log(DEBUG, "Attempting public-key auth...") + self.auth_publickey(username, pkey) + else: + self._log(DEBUG, "Attempting password auth...") + self.auth_password(username, password) + + return + + def get_exception(self): + """ + Return any exception that happened during the last server request. + This can be used to fetch more specific error information after using + calls like `start_client`. The exception (if any) is cleared after + this call. + + :return: + an exception, or ``None`` if there is no stored exception. + + .. versionadded:: 1.1 + """ + self.lock.acquire() + try: + e = self.saved_exception + self.saved_exception = None + return e + finally: + self.lock.release() + + def set_subsystem_handler(self, name, handler, *args, **kwargs): + """ + Set the handler class for a subsystem in server mode. If a request + for this subsystem is made on an open ssh channel later, this handler + will be constructed and called -- see `.SubsystemHandler` for more + detailed documentation. + + Any extra parameters (including keyword arguments) are saved and + passed to the `.SubsystemHandler` constructor later. + + :param str name: name of the subsystem. + :param handler: + subclass of `.SubsystemHandler` that handles this subsystem. + """ + try: + self.lock.acquire() + self.subsystem_table[name] = (handler, args, kwargs) + finally: + self.lock.release() + + def is_authenticated(self): + """ + Return true if this session is active and authenticated. + + :return: + True if the session is still open and has been authenticated + successfully; False if authentication failed and/or the session is + closed. + """ + return ( + self.active + and self.auth_handler is not None + and self.auth_handler.is_authenticated() + ) + + def get_username(self): + """ + Return the username this connection is authenticated for. If the + session is not authenticated (or authentication failed), this method + returns ``None``. + + :return: username that was authenticated (a `str`), or ``None``. + """ + if not self.active or (self.auth_handler is None): + return None + return self.auth_handler.get_username() + + def get_banner(self): + """ + Return the banner supplied by the server upon connect. If no banner is + supplied, this method returns ``None``. + + :returns: server supplied banner (`str`), or ``None``. + + .. versionadded:: 1.13 + """ + if not self.active or (self.auth_handler is None): + return None + return self.auth_handler.banner + + def auth_none(self, username): + """ + Try to authenticate to the server using no authentication at all. + This will almost always fail. It may be useful for determining the + list of authentication types supported by the server, by catching the + `.BadAuthenticationType` exception raised. + + :param str username: the username to authenticate as + :return: + list of auth types permissible for the next stage of + authentication (normally empty) + + :raises: + `.BadAuthenticationType` -- if "none" authentication isn't allowed + by the server for this user + :raises: + `.SSHException` -- if the authentication failed due to a network + error + + .. versionadded:: 1.5 + """ + if (not self.active) or (not self.initial_kex_done): + raise SSHException("No existing session") + my_event = threading.Event() + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_none(username, my_event) + return self.auth_handler.wait_for_response(my_event) + + def auth_password(self, username, password, event=None, fallback=True): + """ + Authenticate to the server using a password. The username and password + are sent over an encrypted link. + + If an ``event`` is passed in, this method will return immediately, and + the event will be triggered once authentication succeeds or fails. On + success, `is_authenticated` will return ``True``. On failure, you may + use `get_exception` to get more detailed error information. + + Since 1.1, if no event is passed, this method will block until the + authentication succeeds or fails. On failure, an exception is raised. + Otherwise, the method simply returns. + + Since 1.5, if no event is passed and ``fallback`` is ``True`` (the + default), if the server doesn't support plain password authentication + but does support so-called "keyboard-interactive" mode, an attempt + will be made to authenticate using this interactive mode. If it fails, + the normal exception will be thrown as if the attempt had never been + made. This is useful for some recent Gentoo and Debian distributions, + which turn off plain password authentication in a misguided belief + that interactive authentication is "more secure". (It's not.) + + If the server requires multi-step authentication (which is very rare), + this method will return a list of auth types permissible for the next + step. Otherwise, in the normal case, an empty list is returned. + + :param str username: the username to authenticate as + :param basestring password: the password to authenticate with + :param .threading.Event event: + an event to trigger when the authentication attempt is complete + (whether it was successful or not) + :param bool fallback: + ``True`` if an attempt at an automated "interactive" password auth + should be made if the server doesn't support normal password auth + :return: + list of auth types permissible for the next stage of + authentication (normally empty) + + :raises: + `.BadAuthenticationType` -- if password authentication isn't + allowed by the server for this user (and no event was passed in) + :raises: + `.AuthenticationException` -- if the authentication failed (and no + event was passed in) + :raises: `.SSHException` -- if there was a network error + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to send the password unless we're on a secure + # link + raise SSHException("No existing session") + if event is None: + my_event = threading.Event() + else: + my_event = event + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_password(username, password, my_event) + if event is not None: + # caller wants to wait for event themselves + return [] + try: + return self.auth_handler.wait_for_response(my_event) + except BadAuthenticationType as e: + # if password auth isn't allowed, but keyboard-interactive *is*, + # try to fudge it + if not fallback or ("keyboard-interactive" not in e.allowed_types): + raise + try: + + def handler(title, instructions, fields): + if len(fields) > 1: + raise SSHException("Fallback authentication failed.") + if len(fields) == 0: + # for some reason, at least on os x, a 2nd request will + # be made with zero fields requested. maybe it's just + # to try to fake out automated scripting of the exact + # type we're doing here. *shrug* :) + return [] + return [password] + + return self.auth_interactive(username, handler) + except SSHException: + # attempt failed; just raise the original exception + raise e + + def auth_publickey(self, username, key, event=None): + """ + Authenticate to the server using a private key. The key is used to + sign data from the server, so it must include the private part. + + If an ``event`` is passed in, this method will return immediately, and + the event will be triggered once authentication succeeds or fails. On + success, `is_authenticated` will return ``True``. On failure, you may + use `get_exception` to get more detailed error information. + + Since 1.1, if no event is passed, this method will block until the + authentication succeeds or fails. On failure, an exception is raised. + Otherwise, the method simply returns. + + If the server requires multi-step authentication (which is very rare), + this method will return a list of auth types permissible for the next + step. Otherwise, in the normal case, an empty list is returned. + + :param str username: the username to authenticate as + :param .PKey key: the private key to authenticate with + :param .threading.Event event: + an event to trigger when the authentication attempt is complete + (whether it was successful or not) + :return: + list of auth types permissible for the next stage of + authentication (normally empty) + + :raises: + `.BadAuthenticationType` -- if public-key authentication isn't + allowed by the server for this user (and no event was passed in) + :raises: + `.AuthenticationException` -- if the authentication failed (and no + event was passed in) + :raises: `.SSHException` -- if there was a network error + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to authenticate unless we're on a secure link + raise SSHException("No existing session") + if event is None: + my_event = threading.Event() + else: + my_event = event + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_publickey(username, key, my_event) + if event is not None: + # caller wants to wait for event themselves + return [] + return self.auth_handler.wait_for_response(my_event) + + def auth_interactive(self, username, handler, submethods=""): + """ + Authenticate to the server interactively. A handler is used to answer + arbitrary questions from the server. On many servers, this is just a + dumb wrapper around PAM. + + This method will block until the authentication succeeds or fails, + periodically calling the handler asynchronously to get answers to + authentication questions. The handler may be called more than once + if the server continues to ask questions. + + The handler is expected to be a callable that will handle calls of the + form: ``handler(title, instructions, prompt_list)``. The ``title`` is + meant to be a dialog-window title, and the ``instructions`` are user + instructions (both are strings). ``prompt_list`` will be a list of + prompts, each prompt being a tuple of ``(str, bool)``. The string is + the prompt and the boolean indicates whether the user text should be + echoed. + + A sample call would thus be: + ``handler('title', 'instructions', [('Password:', False)])``. + + The handler should return a list or tuple of answers to the server's + questions. + + If the server requires multi-step authentication (which is very rare), + this method will return a list of auth types permissible for the next + step. Otherwise, in the normal case, an empty list is returned. + + :param str username: the username to authenticate as + :param callable handler: a handler for responding to server questions + :param str submethods: a string list of desired submethods (optional) + :return: + list of auth types permissible for the next stage of + authentication (normally empty). + + :raises: `.BadAuthenticationType` -- if public-key authentication isn't + allowed by the server for this user + :raises: `.AuthenticationException` -- if the authentication failed + :raises: `.SSHException` -- if there was a network error + + .. versionadded:: 1.5 + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to authenticate unless we're on a secure link + raise SSHException("No existing session") + my_event = threading.Event() + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_interactive( + username, handler, my_event, submethods + ) + return self.auth_handler.wait_for_response(my_event) + + def auth_interactive_dumb(self, username, handler=None, submethods=""): + """ + Authenticate to the server interactively but dumber. + Just print the prompt and / or instructions to stdout and send back + the response. This is good for situations where partial auth is + achieved by key and then the user has to enter a 2fac token. + """ + + if not handler: + + def handler(title, instructions, prompt_list): + answers = [] + if title: + print(title.strip()) + if instructions: + print(instructions.strip()) + for prompt, show_input in prompt_list: + print(prompt.strip(), end=" ") + answers.append(input()) + return answers + + return self.auth_interactive(username, handler, submethods) + + def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds): + """ + Authenticate to the Server using GSS-API / SSPI. + + :param str username: The username to authenticate as + :param str gss_host: The target host + :param bool gss_deleg_creds: Delegate credentials or not + :return: list of auth types permissible for the next stage of + authentication (normally empty) + :raises: `.BadAuthenticationType` -- if gssapi-with-mic isn't + allowed by the server (and no event was passed in) + :raises: + `.AuthenticationException` -- if the authentication failed (and no + event was passed in) + :raises: `.SSHException` -- if there was a network error + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to authenticate unless we're on a secure link + raise SSHException("No existing session") + my_event = threading.Event() + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_gssapi_with_mic( + username, gss_host, gss_deleg_creds, my_event + ) + return self.auth_handler.wait_for_response(my_event) + + def auth_gssapi_keyex(self, username): + """ + Authenticate to the server with GSS-API/SSPI if GSS-API kex is in use. + + :param str username: The username to authenticate as. + :returns: + a list of auth types permissible for the next stage of + authentication (normally empty) + :raises: `.BadAuthenticationType` -- + if GSS-API Key Exchange was not performed (and no event was passed + in) + :raises: `.AuthenticationException` -- + if the authentication failed (and no event was passed in) + :raises: `.SSHException` -- if there was a network error + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to authenticate unless we're on a secure link + raise SSHException("No existing session") + my_event = threading.Event() + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_gssapi_keyex(username, my_event) + return self.auth_handler.wait_for_response(my_event) + + def set_log_channel(self, name): + """ + Set the channel for this transport's logging. The default is + ``"paramiko.transport"`` but it can be set to anything you want. (See + the `.logging` module for more info.) SSH Channels will log to a + sub-channel of the one specified. + + :param str name: new channel name for logging + + .. versionadded:: 1.1 + """ + self.log_name = name + self.logger = util.get_logger(name) + self.packetizer.set_log(self.logger) + + def get_log_channel(self): + """ + Return the channel name used for this transport's logging. + + :return: channel name as a `str` + + .. versionadded:: 1.2 + """ + return self.log_name + + def set_hexdump(self, hexdump): + """ + Turn on/off logging a hex dump of protocol traffic at DEBUG level in + the logs. Normally you would want this off (which is the default), + but if you are debugging something, it may be useful. + + :param bool hexdump: + ``True`` to log protocol traffix (in hex) to the log; ``False`` + otherwise. + """ + self.packetizer.set_hexdump(hexdump) + + def get_hexdump(self): + """ + Return ``True`` if the transport is currently logging hex dumps of + protocol traffic. + + :return: ``True`` if hex dumps are being logged, else ``False``. + + .. versionadded:: 1.4 + """ + return self.packetizer.get_hexdump() + + def use_compression(self, compress=True): + """ + Turn on/off compression. This will only have an affect before starting + the transport (ie before calling `connect`, etc). By default, + compression is off since it negatively affects interactive sessions. + + :param bool compress: + ``True`` to ask the remote client/server to compress traffic; + ``False`` to refuse compression + + .. versionadded:: 1.5.2 + """ + if compress: + self._preferred_compression = ("zlib@openssh.com", "zlib", "none") + else: + self._preferred_compression = ("none",) + + def getpeername(self): + """ + Return the address of the remote side of this Transport, if possible. + + This is effectively a wrapper around ``getpeername`` on the underlying + socket. If the socket-like object has no ``getpeername`` method, then + ``("unknown", 0)`` is returned. + + :return: + the address of the remote host, if known, as a ``(str, int)`` + tuple. + """ + gp = getattr(self.sock, "getpeername", None) + if gp is None: + return "unknown", 0 + return gp() + + def stop_thread(self): + self.active = False + self.packetizer.close() + # Keep trying to join() our main thread, quickly, until: + # * We join()ed successfully (self.is_alive() == False) + # * Or it looks like we've hit issue #520 (socket.recv hitting some + # race condition preventing it from timing out correctly), wherein + # our socket and packetizer are both closed (but where we'd + # otherwise be sitting forever on that recv()). + while ( + self.is_alive() + and self is not threading.current_thread() + and not self.sock._closed + and not self.packetizer.closed + ): + self.join(0.1) + + # internals... + + # TODO 4.0: make a public alias for this because multiple other classes + # already explicitly rely on it...or just rewrite logging :D + def _log(self, level, msg, *args): + if issubclass(type(msg), list): + for m in msg: + self.logger.log(level, m) + else: + self.logger.log(level, msg, *args) + + def _get_modulus_pack(self): + """used by KexGex to find primes for group exchange""" + return self._modulus_pack + + def _next_channel(self): + """you are holding the lock""" + chanid = self._channel_counter + while self._channels.get(chanid) is not None: + self._channel_counter = (self._channel_counter + 1) & 0xFFFFFF + chanid = self._channel_counter + self._channel_counter = (self._channel_counter + 1) & 0xFFFFFF + return chanid + + def _unlink_channel(self, chanid): + """used by a Channel to remove itself from the active channel list""" + self._channels.delete(chanid) + + def _send_message(self, data): + self.packetizer.send_message(data) + + def _send_user_message(self, data): + """ + send a message, but block if we're in key negotiation. this is used + for user-initiated requests. + """ + start = time.time() + while True: + self.clear_to_send.wait(0.1) + if not self.active: + self._log( + DEBUG, "Dropping user packet because connection is dead." + ) # noqa + return + self.clear_to_send_lock.acquire() + if self.clear_to_send.is_set(): + break + self.clear_to_send_lock.release() + if time.time() > start + self.clear_to_send_timeout: + raise SSHException( + "Key-exchange timed out waiting for key negotiation" + ) # noqa + try: + self._send_message(data) + finally: + self.clear_to_send_lock.release() + + def _set_K_H(self, k, h): + """ + Used by a kex obj to set the K (root key) and H (exchange hash). + """ + self.K = k + self.H = h + if self.session_id is None: + self.session_id = h + + def _expect_packet(self, *ptypes): + """ + Used by a kex obj to register the next packet type it expects to see. + """ + self._expected_packet = tuple(ptypes) + + def _verify_key(self, host_key, sig): + key = self._key_info[self.host_key_type](Message(host_key)) + if key is None: + raise SSHException("Unknown host key type") + if not key.verify_ssh_sig(self.H, Message(sig)): + raise SSHException( + "Signature verification ({}) failed.".format( + self.host_key_type + ) + ) # noqa + self.host_key = key + + def _compute_key(self, id, nbytes): + """id is 'A' - 'F' for the various keys used by ssh""" + m = Message() + m.add_mpint(self.K) + m.add_bytes(self.H) + m.add_byte(b(id)) + m.add_bytes(self.session_id) + # Fallback to SHA1 for kex engines that fail to specify a hex + # algorithm, or for e.g. transport tests that don't run kexinit. + hash_algo = getattr(self.kex_engine, "hash_algo", None) + hash_select_msg = "kex engine {} specified hash_algo {!r}".format( + self.kex_engine.__class__.__name__, hash_algo + ) + if hash_algo is None: + hash_algo = sha1 + hash_select_msg += ", falling back to sha1" + if not hasattr(self, "_logged_hash_selection"): + self._log(DEBUG, hash_select_msg) + setattr(self, "_logged_hash_selection", True) + out = sofar = hash_algo(m.asbytes()).digest() + while len(out) < nbytes: + m = Message() + m.add_mpint(self.K) + m.add_bytes(self.H) + m.add_bytes(sofar) + digest = hash_algo(m.asbytes()).digest() + out += digest + sofar += digest + return out[:nbytes] + + def _get_engine(self, name, key, iv=None, operation=None, aead=False): + if name not in self._cipher_info: + raise SSHException("Unknown cipher " + name) + info = self._cipher_info[name] + algorithm = info["class"](key) + # AEAD types (eg GCM) use their algorithm class /as/ the encryption + # engine (they expose the same encrypt/decrypt API as a CipherContext) + if aead: + return algorithm + # All others go through the Cipher class. + cipher = Cipher( + algorithm=algorithm, + # TODO: why is this getting tickled in aesgcm mode??? + mode=info["mode"](iv), + backend=default_backend(), + ) + if operation is self._ENCRYPT: + return cipher.encryptor() + else: + return cipher.decryptor() + + def _set_forward_agent_handler(self, handler): + if handler is None: + + def default_handler(channel): + self._queue_incoming_channel(channel) + + self._forward_agent_handler = default_handler + else: + self._forward_agent_handler = handler + + def _set_x11_handler(self, handler): + # only called if a channel has turned on x11 forwarding + if handler is None: + # by default, use the same mechanism as accept() + def default_handler(channel, src_addr_port): + self._queue_incoming_channel(channel) + + self._x11_handler = default_handler + else: + self._x11_handler = handler + + def _queue_incoming_channel(self, channel): + self.lock.acquire() + try: + self.server_accepts.append(channel) + self.server_accept_cv.notify() + finally: + self.lock.release() + + def _sanitize_window_size(self, window_size): + if window_size is None: + window_size = self.default_window_size + return clamp_value(MIN_WINDOW_SIZE, window_size, MAX_WINDOW_SIZE) + + def _sanitize_packet_size(self, max_packet_size): + if max_packet_size is None: + max_packet_size = self.default_max_packet_size + return clamp_value(MIN_PACKET_SIZE, max_packet_size, MAX_WINDOW_SIZE) + + def _ensure_authed(self, ptype, message): + """ + Checks message type against current auth state. + + If server mode, and auth has not succeeded, and the message is of a + post-auth type (channel open or global request) an appropriate error + response Message is crafted and returned to caller for sending. + + Otherwise (client mode, authed, or pre-auth message) returns None. + """ + if ( + not self.server_mode + or ptype <= HIGHEST_USERAUTH_MESSAGE_ID + or self.is_authenticated() + ): + return None + # WELP. We must be dealing with someone trying to do non-auth things + # without being authed. Tell them off, based on message class. + reply = Message() + # Global requests have no details, just failure. + if ptype == MSG_GLOBAL_REQUEST: + reply.add_byte(cMSG_REQUEST_FAILURE) + # Channel opens let us reject w/ a specific type + message. + elif ptype == MSG_CHANNEL_OPEN: + kind = message.get_text() # noqa + chanid = message.get_int() + reply.add_byte(cMSG_CHANNEL_OPEN_FAILURE) + reply.add_int(chanid) + reply.add_int(OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED) + reply.add_string("") + reply.add_string("en") + # NOTE: Post-open channel messages do not need checking; the above will + # reject attempts to open channels, meaning that even if a malicious + # user tries to send a MSG_CHANNEL_REQUEST, it will simply fall under + # the logic that handles unknown channel IDs (as the channel list will + # be empty.) + return reply + + def _enforce_strict_kex(self, ptype): + """ + Conditionally raise `MessageOrderError` during strict initial kex. + + This method should only be called inside code that handles non-KEXINIT + messages; it does not interrogate ``ptype`` besides using it to log + more accurately. + """ + if self.agreed_on_strict_kex and not self.initial_kex_done: + name = MSG_NAMES.get(ptype, f"msg {ptype}") + raise MessageOrderError( + f"In strict-kex mode, but was sent {name!r}!" + ) + + def run(self): + # (use the exposed "run" method, because if we specify a thread target + # of a private method, threading.Thread will keep a reference to it + # indefinitely, creating a GC cycle and not letting Transport ever be + # GC'd. it's a bug in Thread.) + + # Hold reference to 'sys' so we can test sys.modules to detect + # interpreter shutdown. + self.sys = sys + + # active=True occurs before the thread is launched, to avoid a race + _active_threads.append(self) + tid = hex(id(self) & xffffffff) + if self.server_mode: + self._log(DEBUG, "starting thread (server mode): {}".format(tid)) + else: + self._log(DEBUG, "starting thread (client mode): {}".format(tid)) + try: + try: + self.packetizer.write_all(b(self.local_version + "\r\n")) + self._log( + DEBUG, + "Local version/idstring: {}".format(self.local_version), + ) # noqa + self._check_banner() + # The above is actually very much part of the handshake, but + # sometimes the banner can be read but the machine is not + # responding, for example when the remote ssh daemon is loaded + # in to memory but we can not read from the disk/spawn a new + # shell. + # Make sure we can specify a timeout for the initial handshake. + # Reuse the banner timeout for now. + self.packetizer.start_handshake(self.handshake_timeout) + self._send_kex_init() + self._expect_packet(MSG_KEXINIT) + + while self.active: + if self.packetizer.need_rekey() and not self.in_kex: + self._send_kex_init() + try: + ptype, m = self.packetizer.read_message() + except NeedRekeyException: + continue + if ptype == MSG_IGNORE: + self._enforce_strict_kex(ptype) + continue + elif ptype == MSG_DISCONNECT: + self._parse_disconnect(m) + break + elif ptype == MSG_DEBUG: + self._enforce_strict_kex(ptype) + self._parse_debug(m) + continue + if len(self._expected_packet) > 0: + if ptype not in self._expected_packet: + exc_class = SSHException + if self.agreed_on_strict_kex: + exc_class = MessageOrderError + raise exc_class( + "Expecting packet from {!r}, got {:d}".format( + self._expected_packet, ptype + ) + ) # noqa + self._expected_packet = tuple() + # These message IDs indicate key exchange & will differ + # depending on exact exchange algorithm + if (ptype >= 30) and (ptype <= 41): + self.kex_engine.parse_next(ptype, m) + continue + + if ptype in self._handler_table: + error_msg = self._ensure_authed(ptype, m) + if error_msg: + self._send_message(error_msg) + else: + self._handler_table[ptype](m) + elif ptype in self._channel_handler_table: + chanid = m.get_int() + chan = self._channels.get(chanid) + if chan is not None: + self._channel_handler_table[ptype](chan, m) + elif chanid in self.channels_seen: + self._log( + DEBUG, + "Ignoring message for dead channel {:d}".format( # noqa + chanid + ), + ) + else: + self._log( + ERROR, + "Channel request for unknown channel {:d}".format( # noqa + chanid + ), + ) + break + elif ( + self.auth_handler is not None + and ptype in self.auth_handler._handler_table + ): + handler = self.auth_handler._handler_table[ptype] + handler(m) + if len(self._expected_packet) > 0: + continue + else: + # Respond with "I don't implement this particular + # message type" message (unless the message type was + # itself literally MSG_UNIMPLEMENTED, in which case, we + # just shut up to avoid causing a useless loop). + name = MSG_NAMES[ptype] + warning = "Oops, unhandled type {} ({!r})".format( + ptype, name + ) + self._log(WARNING, warning) + if ptype != MSG_UNIMPLEMENTED: + msg = Message() + msg.add_byte(cMSG_UNIMPLEMENTED) + msg.add_int(m.seqno) + self._send_message(msg) + self.packetizer.complete_handshake() + except SSHException as e: + self._log( + ERROR, + "Exception ({}): {}".format( + "server" if self.server_mode else "client", e + ), + ) + self._log(ERROR, util.tb_strings()) + self.saved_exception = e + except EOFError as e: + self._log(DEBUG, "EOF in transport thread") + self.saved_exception = e + except socket.error as e: + if type(e.args) is tuple: + if e.args: + emsg = "{} ({:d})".format(e.args[1], e.args[0]) + else: # empty tuple, e.g. socket.timeout + emsg = str(e) or repr(e) + else: + emsg = e.args + self._log(ERROR, "Socket exception: " + emsg) + self.saved_exception = e + except Exception as e: + self._log(ERROR, "Unknown exception: " + str(e)) + self._log(ERROR, util.tb_strings()) + self.saved_exception = e + _active_threads.remove(self) + for chan in list(self._channels.values()): + chan._unlink() + if self.active: + self.active = False + self.packetizer.close() + if self.completion_event is not None: + self.completion_event.set() + if self.auth_handler is not None: + self.auth_handler.abort() + for event in self.channel_events.values(): + event.set() + try: + self.lock.acquire() + self.server_accept_cv.notify() + finally: + self.lock.release() + self.sock.close() + except: + # Don't raise spurious 'NoneType has no attribute X' errors when we + # wake up during interpreter shutdown. Or rather -- raise + # everything *if* sys.modules (used as a convenient sentinel) + # appears to still exist. + if self.sys.modules is not None: + raise + + def _log_agreement(self, which, local, remote): + # Log useful, non-duplicative line re: an agreed-upon algorithm. + # Old code implied algorithms could be asymmetrical (different for + # inbound vs outbound) so we preserve that possibility. + msg = "{}: ".format(which) + if local == remote: + msg += local + else: + msg += "local={}, remote={}".format(local, remote) + self._log(DEBUG, msg) + + # protocol stages + + def _negotiate_keys(self, m): + # throws SSHException on anything unusual + self.clear_to_send_lock.acquire() + try: + self.clear_to_send.clear() + finally: + self.clear_to_send_lock.release() + if self.local_kex_init is None: + # remote side wants to renegotiate + self._send_kex_init() + self._parse_kex_init(m) + self.kex_engine.start_kex() + + def _check_banner(self): + # this is slow, but we only have to do it once + for i in range(100): + # give them 15 seconds for the first line, then just 2 seconds + # each additional line. (some sites have very high latency.) + if i == 0: + timeout = self.banner_timeout + else: + timeout = 2 + try: + buf = self.packetizer.readline(timeout) + except ProxyCommandFailure: + raise + except Exception as e: + raise SSHException( + "Error reading SSH protocol banner" + str(e) + ) + if buf[:4] == "SSH-": + break + self._log(DEBUG, "Banner: " + buf) + if buf[:4] != "SSH-": + raise SSHException('Indecipherable protocol version "' + buf + '"') + # save this server version string for later + self.remote_version = buf + self._log(DEBUG, "Remote version/idstring: {}".format(buf)) + # pull off any attached comment + # NOTE: comment used to be stored in a variable and then...never used. + # since 2003. ca 877cd974b8182d26fa76d566072917ea67b64e67 + i = buf.find(" ") + if i >= 0: + buf = buf[:i] + # parse out version string and make sure it matches + segs = buf.split("-", 2) + if len(segs) < 3: + raise SSHException("Invalid SSH banner") + version = segs[1] + client = segs[2] + if version != "1.99" and version != "2.0": + msg = "Incompatible version ({} instead of 2.0)" + raise IncompatiblePeer(msg.format(version)) + msg = "Connected (version {}, client {})".format(version, client) + self._log(INFO, msg) + + def _send_kex_init(self): + """ + announce to the other side that we'd like to negotiate keys, and what + kind of key negotiation we support. + """ + self.clear_to_send_lock.acquire() + try: + self.clear_to_send.clear() + finally: + self.clear_to_send_lock.release() + self.gss_kex_used = False + self.in_kex = True + kex_algos = list(self.preferred_kex) + if self.server_mode: + mp_required_prefix = "diffie-hellman-group-exchange-sha" + kex_mp = [k for k in kex_algos if k.startswith(mp_required_prefix)] + if (self._modulus_pack is None) and (len(kex_mp) > 0): + # can't do group-exchange if we don't have a pack of potential + # primes + pkex = [ + k + for k in self.get_security_options().kex + if not k.startswith(mp_required_prefix) + ] + self.get_security_options().kex = pkex + available_server_keys = list( + filter( + list(self.server_key_dict.keys()).__contains__, + # TODO: ensure tests will catch if somebody streamlines + # this by mistake - case is the admittedly silly one where + # the only calls to add_server_key() contain keys which + # were filtered out of the below via disabled_algorithms. + # If this is streamlined, we would then be allowing the + # disabled algorithm(s) for hostkey use + # TODO: honestly this prob just wants to get thrown out + # when we make kex configuration more straightforward + self.preferred_keys, + ) + ) + else: + available_server_keys = self.preferred_keys + # Signal support for MSG_EXT_INFO so server will send it to us. + # NOTE: doing this here handily means we don't even consider this + # value when agreeing on real kex algo to use (which is a common + # pitfall when adding this apparently). + kex_algos.append("ext-info-c") + + # Similar to ext-info, but used in both server modes, so done outside + # of above if/else. + if self.advertise_strict_kex: + which = "s" if self.server_mode else "c" + kex_algos.append(f"kex-strict-{which}-v00@openssh.com") + + m = Message() + m.add_byte(cMSG_KEXINIT) + m.add_bytes(os.urandom(16)) + m.add_list(kex_algos) + m.add_list(available_server_keys) + m.add_list(self.preferred_ciphers) + m.add_list(self.preferred_ciphers) + m.add_list(self.preferred_macs) + m.add_list(self.preferred_macs) + m.add_list(self.preferred_compression) + m.add_list(self.preferred_compression) + m.add_string(bytes()) + m.add_string(bytes()) + m.add_boolean(False) + m.add_int(0) + # save a copy for later (needed to compute a hash) + self.local_kex_init = self._latest_kex_init = m.asbytes() + self._send_message(m) + + def _really_parse_kex_init(self, m, ignore_first_byte=False): + parsed = {} + if ignore_first_byte: + m.get_byte() + m.get_bytes(16) # cookie, discarded + parsed["kex_algo_list"] = m.get_list() + parsed["server_key_algo_list"] = m.get_list() + parsed["client_encrypt_algo_list"] = m.get_list() + parsed["server_encrypt_algo_list"] = m.get_list() + parsed["client_mac_algo_list"] = m.get_list() + parsed["server_mac_algo_list"] = m.get_list() + parsed["client_compress_algo_list"] = m.get_list() + parsed["server_compress_algo_list"] = m.get_list() + parsed["client_lang_list"] = m.get_list() + parsed["server_lang_list"] = m.get_list() + parsed["kex_follows"] = m.get_boolean() + m.get_int() # unused + return parsed + + def _get_latest_kex_init(self): + return self._really_parse_kex_init( + Message(self._latest_kex_init), + ignore_first_byte=True, + ) + + def _parse_kex_init(self, m): + parsed = self._really_parse_kex_init(m) + kex_algo_list = parsed["kex_algo_list"] + server_key_algo_list = parsed["server_key_algo_list"] + client_encrypt_algo_list = parsed["client_encrypt_algo_list"] + server_encrypt_algo_list = parsed["server_encrypt_algo_list"] + client_mac_algo_list = parsed["client_mac_algo_list"] + server_mac_algo_list = parsed["server_mac_algo_list"] + client_compress_algo_list = parsed["client_compress_algo_list"] + server_compress_algo_list = parsed["server_compress_algo_list"] + client_lang_list = parsed["client_lang_list"] + server_lang_list = parsed["server_lang_list"] + kex_follows = parsed["kex_follows"] + + self._log(DEBUG, "=== Key exchange possibilities ===") + for prefix, value in ( + ("kex algos", kex_algo_list), + ("server key", server_key_algo_list), + # TODO: shouldn't these two lines say "cipher" to match usual + # terminology (including elsewhere in paramiko!)? + ("client encrypt", client_encrypt_algo_list), + ("server encrypt", server_encrypt_algo_list), + ("client mac", client_mac_algo_list), + ("server mac", server_mac_algo_list), + ("client compress", client_compress_algo_list), + ("server compress", server_compress_algo_list), + ("client lang", client_lang_list), + ("server lang", server_lang_list), + ): + if value == [""]: + value = [""] + value = ", ".join(value) + self._log(DEBUG, "{}: {}".format(prefix, value)) + self._log(DEBUG, "kex follows: {}".format(kex_follows)) + self._log(DEBUG, "=== Key exchange agreements ===") + + # Record, and strip out, ext-info and/or strict-kex non-algorithms + self._remote_ext_info = None + self._remote_strict_kex = None + to_pop = [] + for i, algo in enumerate(kex_algo_list): + if algo.startswith("ext-info-"): + self._remote_ext_info = algo + to_pop.insert(0, i) + elif algo.startswith("kex-strict-"): + # NOTE: this is what we are expecting from the /remote/ end. + which = "c" if self.server_mode else "s" + expected = f"kex-strict-{which}-v00@openssh.com" + # Set strict mode if agreed. + self.agreed_on_strict_kex = ( + algo == expected and self.advertise_strict_kex + ) + self._log( + DEBUG, f"Strict kex mode: {self.agreed_on_strict_kex}" + ) + to_pop.insert(0, i) + for i in to_pop: + kex_algo_list.pop(i) + + # CVE mitigation: expect zeroed-out seqno anytime we are performing kex + # init phase, if strict mode was negotiated. + if ( + self.agreed_on_strict_kex + and not self.initial_kex_done + and m.seqno != 0 + ): + raise MessageOrderError( + "In strict-kex mode, but KEXINIT was not the first packet!" + ) + + # as a server, we pick the first item in the client's list that we + # support. + # as a client, we pick the first item in our list that the server + # supports. + if self.server_mode: + agreed_kex = list( + filter(self.preferred_kex.__contains__, kex_algo_list) + ) + else: + agreed_kex = list( + filter(kex_algo_list.__contains__, self.preferred_kex) + ) + if len(agreed_kex) == 0: + # TODO: do an auth-overhaul style aggregate exception here? + # TODO: would let us streamline log output & show all failures up + # front + raise IncompatiblePeer( + "Incompatible ssh peer (no acceptable kex algorithm)" + ) # noqa + self.kex_engine = self._kex_info[agreed_kex[0]](self) + self._log(DEBUG, "Kex: {}".format(agreed_kex[0])) + + if self.server_mode: + available_server_keys = list( + filter( + list(self.server_key_dict.keys()).__contains__, + self.preferred_keys, + ) + ) + agreed_keys = list( + filter( + available_server_keys.__contains__, server_key_algo_list + ) + ) + else: + agreed_keys = list( + filter(server_key_algo_list.__contains__, self.preferred_keys) + ) + if len(agreed_keys) == 0: + raise IncompatiblePeer( + "Incompatible ssh peer (no acceptable host key)" + ) # noqa + self.host_key_type = agreed_keys[0] + if self.server_mode and (self.get_server_key() is None): + raise IncompatiblePeer( + "Incompatible ssh peer (can't match requested host key type)" + ) # noqa + self._log_agreement("HostKey", agreed_keys[0], agreed_keys[0]) + + if self.server_mode: + agreed_local_ciphers = list( + filter( + self.preferred_ciphers.__contains__, + server_encrypt_algo_list, + ) + ) + agreed_remote_ciphers = list( + filter( + self.preferred_ciphers.__contains__, + client_encrypt_algo_list, + ) + ) + else: + agreed_local_ciphers = list( + filter( + client_encrypt_algo_list.__contains__, + self.preferred_ciphers, + ) + ) + agreed_remote_ciphers = list( + filter( + server_encrypt_algo_list.__contains__, + self.preferred_ciphers, + ) + ) + if len(agreed_local_ciphers) == 0 or len(agreed_remote_ciphers) == 0: + raise IncompatiblePeer( + "Incompatible ssh server (no acceptable ciphers)" + ) # noqa + self.local_cipher = agreed_local_ciphers[0] + self.remote_cipher = agreed_remote_ciphers[0] + self._log_agreement( + "Cipher", local=self.local_cipher, remote=self.remote_cipher + ) + + if self.server_mode: + agreed_remote_macs = list( + filter(self.preferred_macs.__contains__, client_mac_algo_list) + ) + agreed_local_macs = list( + filter(self.preferred_macs.__contains__, server_mac_algo_list) + ) + else: + agreed_local_macs = list( + filter(client_mac_algo_list.__contains__, self.preferred_macs) + ) + agreed_remote_macs = list( + filter(server_mac_algo_list.__contains__, self.preferred_macs) + ) + if (len(agreed_local_macs) == 0) or (len(agreed_remote_macs) == 0): + raise IncompatiblePeer( + "Incompatible ssh server (no acceptable macs)" + ) + self.local_mac = agreed_local_macs[0] + self.remote_mac = agreed_remote_macs[0] + self._log_agreement( + "MAC", local=self.local_mac, remote=self.remote_mac + ) + + if self.server_mode: + agreed_remote_compression = list( + filter( + self.preferred_compression.__contains__, + client_compress_algo_list, + ) + ) + agreed_local_compression = list( + filter( + self.preferred_compression.__contains__, + server_compress_algo_list, + ) + ) + else: + agreed_local_compression = list( + filter( + client_compress_algo_list.__contains__, + self.preferred_compression, + ) + ) + agreed_remote_compression = list( + filter( + server_compress_algo_list.__contains__, + self.preferred_compression, + ) + ) + if ( + len(agreed_local_compression) == 0 + or len(agreed_remote_compression) == 0 + ): + msg = "Incompatible ssh server (no acceptable compression)" + msg += " {!r} {!r} {!r}" + raise IncompatiblePeer( + msg.format( + agreed_local_compression, + agreed_remote_compression, + self.preferred_compression, + ) + ) + self.local_compression = agreed_local_compression[0] + self.remote_compression = agreed_remote_compression[0] + self._log_agreement( + "Compression", + local=self.local_compression, + remote=self.remote_compression, + ) + self._log(DEBUG, "=== End of kex handshake ===") + + # save for computing hash later... + # now wait! openssh has a bug (and others might too) where there are + # actually some extra bytes (one NUL byte in openssh's case) added to + # the end of the packet but not parsed. turns out we need to throw + # away those bytes because they aren't part of the hash. + self.remote_kex_init = cMSG_KEXINIT + m.get_so_far() + + def _activate_inbound(self): + """switch on newly negotiated encryption parameters for + inbound traffic""" + info = self._cipher_info[self.remote_cipher] + aead = info.get("is_aead", False) + block_size = info["block-size"] + key_size = info["key-size"] + # Non-AEAD/GCM type ciphers' IV size is their block size. + iv_size = info.get("iv-size", block_size) + if self.server_mode: + iv_in = self._compute_key("A", iv_size) + key_in = self._compute_key("C", key_size) + else: + iv_in = self._compute_key("B", iv_size) + key_in = self._compute_key("D", key_size) + + engine = self._get_engine( + name=self.remote_cipher, + key=key_in, + iv=iv_in, + operation=self._DECRYPT, + aead=aead, + ) + etm = (not aead) and "etm@openssh.com" in self.remote_mac + mac_size = self._mac_info[self.remote_mac]["size"] + mac_engine = self._mac_info[self.remote_mac]["class"] + # initial mac keys are done in the hash's natural size (not the + # potentially truncated transmission size) + if self.server_mode: + mac_key = self._compute_key("E", mac_engine().digest_size) + else: + mac_key = self._compute_key("F", mac_engine().digest_size) + + self.packetizer.set_inbound_cipher( + block_engine=engine, + block_size=block_size, + mac_engine=None if aead else mac_engine, + mac_size=16 if aead else mac_size, + mac_key=None if aead else mac_key, + etm=etm, + aead=aead, + iv_in=iv_in if aead else None, + ) + + compress_in = self._compression_info[self.remote_compression][1] + if compress_in is not None and ( + self.remote_compression != "zlib@openssh.com" or self.authenticated + ): + self._log(DEBUG, "Switching on inbound compression ...") + self.packetizer.set_inbound_compressor(compress_in()) + # Reset inbound sequence number if strict mode. + if self.agreed_on_strict_kex: + self._log( + DEBUG, + "Resetting inbound seqno after NEWKEYS due to strict mode", + ) + self.packetizer.reset_seqno_in() + + def _activate_outbound(self): + """switch on newly negotiated encryption parameters for + outbound traffic""" + m = Message() + m.add_byte(cMSG_NEWKEYS) + self._send_message(m) + # Reset outbound sequence number if strict mode. + if self.agreed_on_strict_kex: + self._log( + DEBUG, + "Resetting outbound seqno after NEWKEYS due to strict mode", + ) + self.packetizer.reset_seqno_out() + info = self._cipher_info[self.local_cipher] + aead = info.get("is_aead", False) + block_size = info["block-size"] + key_size = info["key-size"] + # Non-AEAD/GCM type ciphers' IV size is their block size. + iv_size = info.get("iv-size", block_size) + if self.server_mode: + iv_out = self._compute_key("B", iv_size) + key_out = self._compute_key("D", key_size) + else: + iv_out = self._compute_key("A", iv_size) + key_out = self._compute_key("C", key_size) + + engine = self._get_engine( + name=self.local_cipher, + key=key_out, + iv=iv_out, + operation=self._ENCRYPT, + aead=aead, + ) + etm = (not aead) and "etm@openssh.com" in self.local_mac + mac_size = self._mac_info[self.local_mac]["size"] + mac_engine = self._mac_info[self.local_mac]["class"] + # initial mac keys are done in the hash's natural size (not the + # potentially truncated transmission size) + if self.server_mode: + mac_key = self._compute_key("F", mac_engine().digest_size) + else: + mac_key = self._compute_key("E", mac_engine().digest_size) + sdctr = self.local_cipher.endswith("-ctr") + + self.packetizer.set_outbound_cipher( + block_engine=engine, + block_size=block_size, + mac_engine=None if aead else mac_engine, + mac_size=16 if aead else mac_size, + mac_key=None if aead else mac_key, + sdctr=sdctr, + etm=etm, + aead=aead, + iv_out=iv_out if aead else None, + ) + + compress_out = self._compression_info[self.local_compression][0] + if compress_out is not None and ( + self.local_compression != "zlib@openssh.com" or self.authenticated + ): + self._log(DEBUG, "Switching on outbound compression ...") + self.packetizer.set_outbound_compressor(compress_out()) + if not self.packetizer.need_rekey(): + self.in_kex = False + # If client indicated extension support, send that packet immediately + if ( + self.server_mode + and self.server_sig_algs + and self._remote_ext_info == "ext-info-c" + ): + extensions = {"server-sig-algs": ",".join(self.preferred_pubkeys)} + m = Message() + m.add_byte(cMSG_EXT_INFO) + m.add_int(len(extensions)) + for name, value in sorted(extensions.items()): + m.add_string(name) + m.add_string(value) + self._send_message(m) + # we always expect to receive NEWKEYS now + self._expect_packet(MSG_NEWKEYS) + + def _auth_trigger(self): + self.authenticated = True + # delayed initiation of compression + if self.local_compression == "zlib@openssh.com": + compress_out = self._compression_info[self.local_compression][0] + self._log(DEBUG, "Switching on outbound compression ...") + self.packetizer.set_outbound_compressor(compress_out()) + if self.remote_compression == "zlib@openssh.com": + compress_in = self._compression_info[self.remote_compression][1] + self._log(DEBUG, "Switching on inbound compression ...") + self.packetizer.set_inbound_compressor(compress_in()) + + def _parse_ext_info(self, msg): + # Packet is a count followed by that many key-string to possibly-bytes + # pairs. + extensions = {} + for _ in range(msg.get_int()): + name = msg.get_text() + value = msg.get_string() + extensions[name] = value + self._log(DEBUG, "Got EXT_INFO: {}".format(extensions)) + # NOTE: this should work ok in cases where a server sends /two/ such + # messages; the RFC explicitly states a 2nd one should overwrite the + # 1st. + self.server_extensions = extensions + + def _parse_newkeys(self, m): + self._log(DEBUG, "Switch to new keys ...") + self._activate_inbound() + # can also free a bunch of stuff here + self.local_kex_init = self.remote_kex_init = None + self.K = None + self.kex_engine = None + if self.server_mode and (self.auth_handler is None): + # create auth handler for server mode + self.auth_handler = AuthHandler(self) + if not self.initial_kex_done: + # this was the first key exchange + # (also signal to packetizer as it sometimes wants to know this + # status as well, eg when seqnos rollover) + self.initial_kex_done = self.packetizer._initial_kex_done = True + # send an event? + if self.completion_event is not None: + self.completion_event.set() + # it's now okay to send data again (if this was a re-key) + if not self.packetizer.need_rekey(): + self.in_kex = False + self.clear_to_send_lock.acquire() + try: + self.clear_to_send.set() + finally: + self.clear_to_send_lock.release() + return + + def _parse_disconnect(self, m): + code = m.get_int() + desc = m.get_text() + self._log(INFO, "Disconnect (code {:d}): {}".format(code, desc)) + + def _parse_global_request(self, m): + kind = m.get_text() + self._log(DEBUG, 'Received global request "{}"'.format(kind)) + want_reply = m.get_boolean() + if not self.server_mode: + self._log( + DEBUG, + 'Rejecting "{}" global request from server.'.format(kind), + ) + ok = False + elif kind == "tcpip-forward": + address = m.get_text() + port = m.get_int() + ok = self.server_object.check_port_forward_request(address, port) + if ok: + ok = (ok,) + elif kind == "cancel-tcpip-forward": + address = m.get_text() + port = m.get_int() + self.server_object.cancel_port_forward_request(address, port) + ok = True + else: + ok = self.server_object.check_global_request(kind, m) + extra = () + if type(ok) is tuple: + extra = ok + ok = True + if want_reply: + msg = Message() + if ok: + msg.add_byte(cMSG_REQUEST_SUCCESS) + msg.add(*extra) + else: + msg.add_byte(cMSG_REQUEST_FAILURE) + self._send_message(msg) + + def _parse_request_success(self, m): + self._log(DEBUG, "Global request successful.") + self.global_response = m + if self.completion_event is not None: + self.completion_event.set() + + def _parse_request_failure(self, m): + self._log(DEBUG, "Global request denied.") + self.global_response = None + if self.completion_event is not None: + self.completion_event.set() + + def _parse_channel_open_success(self, m): + chanid = m.get_int() + server_chanid = m.get_int() + server_window_size = m.get_int() + server_max_packet_size = m.get_int() + chan = self._channels.get(chanid) + if chan is None: + self._log(WARNING, "Success for unrequested channel! [??]") + return + self.lock.acquire() + try: + chan._set_remote_channel( + server_chanid, server_window_size, server_max_packet_size + ) + self._log(DEBUG, "Secsh channel {:d} opened.".format(chanid)) + if chanid in self.channel_events: + self.channel_events[chanid].set() + del self.channel_events[chanid] + finally: + self.lock.release() + return + + def _parse_channel_open_failure(self, m): + chanid = m.get_int() + reason = m.get_int() + reason_str = m.get_text() + m.get_text() # ignored language + reason_text = CONNECTION_FAILED_CODE.get(reason, "(unknown code)") + self._log( + ERROR, + "Secsh channel {:d} open FAILED: {}: {}".format( + chanid, reason_str, reason_text + ), + ) + self.lock.acquire() + try: + self.saved_exception = ChannelException(reason, reason_text) + if chanid in self.channel_events: + self._channels.delete(chanid) + if chanid in self.channel_events: + self.channel_events[chanid].set() + del self.channel_events[chanid] + finally: + self.lock.release() + return + + def _parse_channel_open(self, m): + kind = m.get_text() + chanid = m.get_int() + initial_window_size = m.get_int() + max_packet_size = m.get_int() + reject = False + if ( + kind == "auth-agent@openssh.com" + and self._forward_agent_handler is not None + ): + self._log(DEBUG, "Incoming forward agent connection") + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + elif (kind == "x11") and (self._x11_handler is not None): + origin_addr = m.get_text() + origin_port = m.get_int() + self._log( + DEBUG, + "Incoming x11 connection from {}:{:d}".format( + origin_addr, origin_port + ), + ) + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + elif (kind == "forwarded-tcpip") and (self._tcp_handler is not None): + server_addr = m.get_text() + server_port = m.get_int() + origin_addr = m.get_text() + origin_port = m.get_int() + self._log( + DEBUG, + "Incoming tcp forwarded connection from {}:{:d}".format( + origin_addr, origin_port + ), + ) + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + elif not self.server_mode: + self._log( + DEBUG, + 'Rejecting "{}" channel request from server.'.format(kind), + ) + reject = True + reason = OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + else: + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + if kind == "direct-tcpip": + # handle direct-tcpip requests coming from the client + dest_addr = m.get_text() + dest_port = m.get_int() + origin_addr = m.get_text() + origin_port = m.get_int() + reason = self.server_object.check_channel_direct_tcpip_request( + my_chanid, + (origin_addr, origin_port), + (dest_addr, dest_port), + ) + else: + reason = self.server_object.check_channel_request( + kind, my_chanid + ) + if reason != OPEN_SUCCEEDED: + self._log( + DEBUG, + 'Rejecting "{}" channel request from client.'.format(kind), + ) + reject = True + if reject: + msg = Message() + msg.add_byte(cMSG_CHANNEL_OPEN_FAILURE) + msg.add_int(chanid) + msg.add_int(reason) + msg.add_string("") + msg.add_string("en") + self._send_message(msg) + return + + chan = Channel(my_chanid) + self.lock.acquire() + try: + self._channels.put(my_chanid, chan) + self.channels_seen[my_chanid] = True + chan._set_transport(self) + chan._set_window( + self.default_window_size, self.default_max_packet_size + ) + chan._set_remote_channel( + chanid, initial_window_size, max_packet_size + ) + finally: + self.lock.release() + m = Message() + m.add_byte(cMSG_CHANNEL_OPEN_SUCCESS) + m.add_int(chanid) + m.add_int(my_chanid) + m.add_int(self.default_window_size) + m.add_int(self.default_max_packet_size) + self._send_message(m) + self._log( + DEBUG, "Secsh channel {:d} ({}) opened.".format(my_chanid, kind) + ) + if kind == "auth-agent@openssh.com": + self._forward_agent_handler(chan) + elif kind == "x11": + self._x11_handler(chan, (origin_addr, origin_port)) + elif kind == "forwarded-tcpip": + chan.origin_addr = (origin_addr, origin_port) + self._tcp_handler( + chan, (origin_addr, origin_port), (server_addr, server_port) + ) + else: + self._queue_incoming_channel(chan) + + def _parse_debug(self, m): + m.get_boolean() # always_display + msg = m.get_string() + m.get_string() # language + self._log(DEBUG, "Debug msg: {}".format(util.safe_string(msg))) + + def _get_subsystem_handler(self, name): + try: + self.lock.acquire() + if name not in self.subsystem_table: + return None, [], {} + return self.subsystem_table[name] + finally: + self.lock.release() + + _channel_handler_table = { + MSG_CHANNEL_SUCCESS: Channel._request_success, + MSG_CHANNEL_FAILURE: Channel._request_failed, + MSG_CHANNEL_DATA: Channel._feed, + MSG_CHANNEL_EXTENDED_DATA: Channel._feed_extended, + MSG_CHANNEL_WINDOW_ADJUST: Channel._window_adjust, + MSG_CHANNEL_REQUEST: Channel._handle_request, + MSG_CHANNEL_EOF: Channel._handle_eof, + MSG_CHANNEL_CLOSE: Channel._handle_close, + } + + +# TODO 4.0: drop this, we barely use it ourselves, it badly replicates the +# Transport-internal algorithm management, AND does so in a way which doesn't +# honor newer things like disabled_algorithms! +class SecurityOptions: + """ + Simple object containing the security preferences of an ssh transport. + These are tuples of acceptable ciphers, digests, key types, and key + exchange algorithms, listed in order of preference. + + Changing the contents and/or order of these fields affects the underlying + `.Transport` (but only if you change them before starting the session). + If you try to add an algorithm that paramiko doesn't recognize, + ``ValueError`` will be raised. If you try to assign something besides a + tuple to one of the fields, ``TypeError`` will be raised. + """ + + __slots__ = "_transport" + + def __init__(self, transport): + self._transport = transport + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + """ + return "".format(self._transport) + + def _set(self, name, orig, x): + if type(x) is list: + x = tuple(x) + if type(x) is not tuple: + raise TypeError("expected tuple or list") + possible = list(getattr(self._transport, orig).keys()) + forbidden = [n for n in x if n not in possible] + if len(forbidden) > 0: + raise ValueError("unknown cipher") + setattr(self._transport, name, x) + + @property + def ciphers(self): + """Symmetric encryption ciphers""" + return self._transport._preferred_ciphers + + @ciphers.setter + def ciphers(self, x): + self._set("_preferred_ciphers", "_cipher_info", x) + + @property + def digests(self): + """Digest (one-way hash) algorithms""" + return self._transport._preferred_macs + + @digests.setter + def digests(self, x): + self._set("_preferred_macs", "_mac_info", x) + + @property + def key_types(self): + """Public-key algorithms""" + return self._transport._preferred_keys + + @key_types.setter + def key_types(self, x): + self._set("_preferred_keys", "_key_info", x) + + @property + def kex(self): + """Key exchange algorithms""" + return self._transport._preferred_kex + + @kex.setter + def kex(self, x): + self._set("_preferred_kex", "_kex_info", x) + + @property + def compression(self): + """Compression algorithms""" + return self._transport._preferred_compression + + @compression.setter + def compression(self, x): + self._set("_preferred_compression", "_compression_info", x) + + +class ChannelMap: + def __init__(self): + # (id -> Channel) + self._map = weakref.WeakValueDictionary() + self._lock = threading.Lock() + + def put(self, chanid, chan): + self._lock.acquire() + try: + self._map[chanid] = chan + finally: + self._lock.release() + + def get(self, chanid): + self._lock.acquire() + try: + return self._map.get(chanid, None) + finally: + self._lock.release() + + def delete(self, chanid): + self._lock.acquire() + try: + try: + del self._map[chanid] + except KeyError: + pass + finally: + self._lock.release() + + def values(self): + self._lock.acquire() + try: + return list(self._map.values()) + finally: + self._lock.release() + + def __len__(self): + self._lock.acquire() + try: + return len(self._map) + finally: + self._lock.release() + + +class ServiceRequestingTransport(Transport): + """ + Transport, but also handling service requests, like it oughtta! + + .. versionadded:: 3.2 + """ + + # NOTE: this purposefully duplicates some of the parent class in order to + # modernize, refactor, etc. The intent is that eventually we will collapse + # this one onto the parent in a backwards incompatible release. + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._service_userauth_accepted = False + self._handler_table[MSG_SERVICE_ACCEPT] = self._parse_service_accept + + def _parse_service_accept(self, m): + service = m.get_text() + # Short-circuit for any service name not ssh-userauth. + # NOTE: it's technically possible for 'service name' in + # SERVICE_REQUEST/ACCEPT messages to be "ssh-connection" -- + # but I don't see evidence of Paramiko ever initiating or expecting to + # receive one of these. We /do/ see the 'service name' field in + # MSG_USERAUTH_REQUEST/ACCEPT/FAILURE set to this string, but that is a + # different set of handlers, so...! + if service != "ssh-userauth": + # TODO 4.0: consider erroring here (with an ability to opt out?) + # instead as it probably means something went Very Wrong. + self._log( + DEBUG, 'Service request "{}" accepted (?)'.format(service) + ) + return + # Record that we saw a service-userauth acceptance, meaning we are free + # to submit auth requests. + self._service_userauth_accepted = True + self._log(DEBUG, "MSG_SERVICE_ACCEPT received; auth may begin") + + def ensure_session(self): + # Make sure we're not trying to auth on a not-yet-open or + # already-closed transport session; that's our responsibility, not that + # of AuthHandler. + if (not self.active) or (not self.initial_kex_done): + # TODO: better error message? this can happen in many places, eg + # user error (authing before connecting) or developer error (some + # improperly handled pre/mid auth shutdown didn't become fatal + # enough). The latter is much more common & should ideally be fixed + # by terminating things harder? + raise SSHException("No existing session") + # Also make sure we've actually been told we are allowed to auth. + if self._service_userauth_accepted: + return + # Or request to do so, otherwise. + m = Message() + m.add_byte(cMSG_SERVICE_REQUEST) + m.add_string("ssh-userauth") + self._log(DEBUG, "Sending MSG_SERVICE_REQUEST: ssh-userauth") + self._send_message(m) + # Now we wait to hear back; the user is expecting a blocking-style auth + # request so there's no point giving control back anywhere. + while not self._service_userauth_accepted: + # TODO: feels like we're missing an AuthHandler Event like + # 'self.auth_event' which is set when AuthHandler shuts down in + # ways good AND bad. Transport only seems to have completion_event + # which is unclear re: intent, eg it's set by newkeys which always + # happens on connection, so it'll always be set by the time we get + # here. + # NOTE: this copies the timing of event.wait() in + # AuthHandler.wait_for_response, re: 1/10 of a second. Could + # presumably be smaller, but seems unlikely this period is going to + # be "too long" for any code doing ssh networking... + time.sleep(0.1) + self.auth_handler = self.get_auth_handler() + + def get_auth_handler(self): + # NOTE: using new sibling subclass instead of classic AuthHandler + return AuthOnlyHandler(self) + + def auth_none(self, username): + # TODO 4.0: merge to parent, preserving (most of) docstring + self.ensure_session() + return self.auth_handler.auth_none(username) + + def auth_password(self, username, password, fallback=True): + # TODO 4.0: merge to parent, preserving (most of) docstring + self.ensure_session() + try: + return self.auth_handler.auth_password(username, password) + except BadAuthenticationType as e: + # if password auth isn't allowed, but keyboard-interactive *is*, + # try to fudge it + if not fallback or ("keyboard-interactive" not in e.allowed_types): + raise + try: + + def handler(title, instructions, fields): + if len(fields) > 1: + raise SSHException("Fallback authentication failed.") + if len(fields) == 0: + # for some reason, at least on os x, a 2nd request will + # be made with zero fields requested. maybe it's just + # to try to fake out automated scripting of the exact + # type we're doing here. *shrug* :) + return [] + return [password] + + return self.auth_interactive(username, handler) + except SSHException: + # attempt to fudge failed; just raise the original exception + raise e + + def auth_publickey(self, username, key): + # TODO 4.0: merge to parent, preserving (most of) docstring + self.ensure_session() + return self.auth_handler.auth_publickey(username, key) + + def auth_interactive(self, username, handler, submethods=""): + # TODO 4.0: merge to parent, preserving (most of) docstring + self.ensure_session() + return self.auth_handler.auth_interactive( + username, handler, submethods + ) + + def auth_interactive_dumb(self, username, handler=None, submethods=""): + # TODO 4.0: merge to parent, preserving (most of) docstring + # NOTE: legacy impl omitted equiv of ensure_session since it just wraps + # another call to an auth method. however we reinstate it for + # consistency reasons. + self.ensure_session() + if not handler: + + def handler(title, instructions, prompt_list): + answers = [] + if title: + print(title.strip()) + if instructions: + print(instructions.strip()) + for prompt, show_input in prompt_list: + print(prompt.strip(), end=" ") + answers.append(input()) + return answers + + return self.auth_interactive(username, handler, submethods) + + def auth_gssapi_with_mic(self, username, gss_host, gss_deleg_creds): + # TODO 4.0: merge to parent, preserving (most of) docstring + self.ensure_session() + self.auth_handler = self.get_auth_handler() + return self.auth_handler.auth_gssapi_with_mic( + username, gss_host, gss_deleg_creds + ) + + def auth_gssapi_keyex(self, username): + # TODO 4.0: merge to parent, preserving (most of) docstring + self.ensure_session() + self.auth_handler = self.get_auth_handler() + return self.auth_handler.auth_gssapi_keyex(username) diff --git a/lib/paramiko/util.py b/lib/paramiko/util.py new file mode 100644 index 0000000..c23e498 --- /dev/null +++ b/lib/paramiko/util.py @@ -0,0 +1,336 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Useful functions used by the rest of paramiko. +""" + + +import sys +import struct +import traceback +import threading +import logging + +from paramiko.common import ( + DEBUG, + zero_byte, + xffffffff, + max_byte, + byte_ord, + byte_chr, +) +from paramiko.config import SSHConfig + + +def inflate_long(s, always_positive=False): + """turns a normalized byte string into a long-int + (adapted from Crypto.Util.number)""" + out = 0 + negative = 0 + if not always_positive and (len(s) > 0) and (byte_ord(s[0]) >= 0x80): + negative = 1 + if len(s) % 4: + filler = zero_byte + if negative: + filler = max_byte + # never convert this to ``s +=`` because this is a string, not a number + # noinspection PyAugmentAssignment + s = filler * (4 - len(s) % 4) + s + for i in range(0, len(s), 4): + out = (out << 32) + struct.unpack(">I", s[i : i + 4])[0] + if negative: + out -= 1 << (8 * len(s)) + return out + + +def deflate_long(n, add_sign_padding=True): + """turns a long-int into a normalized byte string + (adapted from Crypto.Util.number)""" + # after much testing, this algorithm was deemed to be the fastest + s = bytes() + n = int(n) + while (n != 0) and (n != -1): + s = struct.pack(">I", n & xffffffff) + s + n >>= 32 + # strip off leading zeros, FFs + for i in enumerate(s): + if (n == 0) and (i[1] != 0): + break + if (n == -1) and (i[1] != 0xFF): + break + else: + # degenerate case, n was either 0 or -1 + i = (0,) + if n == 0: + s = zero_byte + else: + s = max_byte + s = s[i[0] :] + if add_sign_padding: + if (n == 0) and (byte_ord(s[0]) >= 0x80): + s = zero_byte + s + if (n == -1) and (byte_ord(s[0]) < 0x80): + s = max_byte + s + return s + + +def format_binary(data, prefix=""): + x = 0 + out = [] + while len(data) > x + 16: + out.append(format_binary_line(data[x : x + 16])) + x += 16 + if x < len(data): + out.append(format_binary_line(data[x:])) + return [prefix + line for line in out] + + +def format_binary_line(data): + left = " ".join(["{:02X}".format(byte_ord(c)) for c in data]) + right = "".join( + [".{:c}..".format(byte_ord(c))[(byte_ord(c) + 63) // 95] for c in data] + ) + return "{:50s} {}".format(left, right) + + +def safe_string(s): + out = b"" + for c in s: + i = byte_ord(c) + if 32 <= i <= 127: + out += byte_chr(i) + else: + out += b("%{:02X}".format(i)) + return out + + +def bit_length(n): + try: + return n.bit_length() + except AttributeError: + norm = deflate_long(n, False) + hbyte = byte_ord(norm[0]) + if hbyte == 0: + return 1 + bitlen = len(norm) * 8 + while not (hbyte & 0x80): + hbyte <<= 1 + bitlen -= 1 + return bitlen + + +def tb_strings(): + return "".join(traceback.format_exception(*sys.exc_info())).split("\n") + + +def generate_key_bytes(hash_alg, salt, key, nbytes): + """ + Given a password, passphrase, or other human-source key, scramble it + through a secure hash into some keyworthy bytes. This specific algorithm + is used for encrypting/decrypting private key files. + + :param function hash_alg: A function which creates a new hash object, such + as ``hashlib.sha256``. + :param salt: data to salt the hash with. + :type bytes salt: Hash salt bytes. + :param str key: human-entered password or passphrase. + :param int nbytes: number of bytes to generate. + :return: Key data, as `bytes`. + """ + keydata = bytes() + digest = bytes() + if len(salt) > 8: + salt = salt[:8] + while nbytes > 0: + hash_obj = hash_alg() + if len(digest) > 0: + hash_obj.update(digest) + hash_obj.update(b(key)) + hash_obj.update(salt) + digest = hash_obj.digest() + size = min(nbytes, len(digest)) + keydata += digest[:size] + nbytes -= size + return keydata + + +def load_host_keys(filename): + """ + Read a file of known SSH host keys, in the format used by openssh, and + return a compound dict of ``hostname -> keytype ->`` `PKey + `. The hostname may be an IP address or DNS name. + + This type of file unfortunately doesn't exist on Windows, but on posix, + it will usually be stored in ``os.path.expanduser("~/.ssh/known_hosts")``. + + Since 1.5.3, this is just a wrapper around `.HostKeys`. + + :param str filename: name of the file to read host keys from + :return: + nested dict of `.PKey` objects, indexed by hostname and then keytype + """ + from paramiko.hostkeys import HostKeys + + return HostKeys(filename) + + +def parse_ssh_config(file_obj): + """ + Provided only as a backward-compatible wrapper around `.SSHConfig`. + + .. deprecated:: 2.7 + Use `SSHConfig.from_file` instead. + """ + config = SSHConfig() + config.parse(file_obj) + return config + + +def lookup_ssh_host_config(hostname, config): + """ + Provided only as a backward-compatible wrapper around `.SSHConfig`. + """ + return config.lookup(hostname) + + +def mod_inverse(x, m): + # it's crazy how small Python can make this function. + u1, u2, u3 = 1, 0, m + v1, v2, v3 = 0, 1, x + + while v3 > 0: + q = u3 // v3 + u1, v1 = v1, u1 - v1 * q + u2, v2 = v2, u2 - v2 * q + u3, v3 = v3, u3 - v3 * q + if u2 < 0: + u2 += m + return u2 + + +_g_thread_data = threading.local() +_g_thread_counter = 0 +_g_thread_lock = threading.Lock() + + +def get_thread_id(): + global _g_thread_data, _g_thread_counter, _g_thread_lock # noqa + try: + return _g_thread_data.id + except AttributeError: + with _g_thread_lock: + _g_thread_counter += 1 + _g_thread_data.id = _g_thread_counter + return _g_thread_data.id + + +def log_to_file(filename, level=DEBUG): + """send paramiko logs to a logfile, + if they're not already going somewhere""" + logger = logging.getLogger("paramiko") + if len(logger.handlers) > 0: + return + logger.setLevel(level) + f = open(filename, "a") + handler = logging.StreamHandler(f) + frm = "%(levelname)-.3s [%(asctime)s.%(msecs)03d] thr=%(_threadid)-3d" + frm += " %(name)s: %(message)s" + handler.setFormatter(logging.Formatter(frm, "%Y%m%d-%H:%M:%S")) + logger.addHandler(handler) + + +# make only one filter object, so it doesn't get applied more than once +class PFilter: + def filter(self, record): + record._threadid = get_thread_id() + return True + + +_pfilter = PFilter() + + +def get_logger(name): + logger = logging.getLogger(name) + logger.addFilter(_pfilter) + return logger + + +def constant_time_bytes_eq(a, b): + if len(a) != len(b): + return False + res = 0 + # noinspection PyUnresolvedReferences + for i in range(len(a)): # noqa: F821 + res |= byte_ord(a[i]) ^ byte_ord(b[i]) + return res == 0 + + +class ClosingContextManager: + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + +def clamp_value(minimum, val, maximum): + return max(minimum, min(val, maximum)) + + +def asbytes(s): + """ + Coerce to bytes if possible or return unchanged. + """ + try: + # Attempt to run through our version of b(), which does the Right Thing + # for unicode strings vs bytestrings, and raises TypeError if it's not + # one of those types. + return b(s) + except TypeError: + try: + # If it wasn't a string/byte/buffer-ish object, try calling an + # asbytes() method, which many of our internal classes implement. + return s.asbytes() + except AttributeError: + # Finally, just do nothing & assume this object is sufficiently + # byte-y or buffer-y that everything will work out (or that callers + # are capable of handling whatever it is.) + return s + + +# TODO: clean this up / force callers to assume bytes OR unicode +def b(s, encoding="utf8"): + """cast unicode or bytes to bytes""" + if isinstance(s, bytes): + return s + elif isinstance(s, str): + return s.encode(encoding) + else: + raise TypeError(f"Expected unicode or bytes, got {type(s)}") + + +# TODO: clean this up / force callers to assume bytes OR unicode +def u(s, encoding="utf8"): + """cast bytes or unicode to unicode""" + if isinstance(s, bytes): + return s.decode(encoding) + elif isinstance(s, str): + return s + else: + raise TypeError(f"Expected unicode or bytes, got {type(s)}") diff --git a/lib/paramiko/win_openssh.py b/lib/paramiko/win_openssh.py new file mode 100644 index 0000000..614b589 --- /dev/null +++ b/lib/paramiko/win_openssh.py @@ -0,0 +1,56 @@ +# Copyright (C) 2021 Lew Gordon +# Copyright (C) 2022 Patrick Spendrin +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os.path +import time + +PIPE_NAME = r"\\.\pipe\openssh-ssh-agent" + + +def can_talk_to_agent(): + # use os.listdir() instead of os.path.exists(), because os.path.exists() + # uses CreateFileW() API and the pipe cannot be reopen unless the server + # calls DisconnectNamedPipe(). + dir_, name = os.path.split(PIPE_NAME) + name = name.lower() + return any(name == n.lower() for n in os.listdir(dir_)) + + +class OpenSSHAgentConnection: + def __init__(self): + while True: + try: + self._pipe = os.open(PIPE_NAME, os.O_RDWR | os.O_BINARY) + except OSError as e: + # retry when errno 22 which means that the server has not + # called DisconnectNamedPipe() yet. + if e.errno != 22: + raise + else: + break + time.sleep(0.1) + + def send(self, data): + return os.write(self._pipe, data) + + def recv(self, n): + return os.read(self._pipe, n) + + def close(self): + return os.close(self._pipe) diff --git a/lib/paramiko/win_pageant.py b/lib/paramiko/win_pageant.py new file mode 100644 index 0000000..c927de6 --- /dev/null +++ b/lib/paramiko/win_pageant.py @@ -0,0 +1,138 @@ +# Copyright (C) 2005 John Arbash-Meinel +# Modified up by: Todd Whiteman +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" +Functions for communicating with Pageant, the basic windows ssh agent program. +""" + +import array +import ctypes.wintypes +import platform +import struct +from paramiko.common import zero_byte +from paramiko.util import b + +import _thread as thread + +from . import _winapi + + +_AGENT_COPYDATA_ID = 0x804E50BA +_AGENT_MAX_MSGLEN = 8192 +# Note: The WM_COPYDATA value is pulled from win32con, as a workaround +# so we do not have to import this huge library just for this one variable. +win32con_WM_COPYDATA = 74 + + +def _get_pageant_window_object(): + return ctypes.windll.user32.FindWindowA(b"Pageant", b"Pageant") + + +def can_talk_to_agent(): + """ + Check to see if there is a "Pageant" agent we can talk to. + + This checks both if we have the required libraries (win32all or ctypes) + and if there is a Pageant currently running. + """ + return bool(_get_pageant_window_object()) + + +if platform.architecture()[0] == "64bit": + ULONG_PTR = ctypes.c_uint64 +else: + ULONG_PTR = ctypes.c_uint32 + + +class COPYDATASTRUCT(ctypes.Structure): + """ + ctypes implementation of + http://msdn.microsoft.com/en-us/library/windows/desktop/ms649010%28v=vs.85%29.aspx + """ + + _fields_ = [ + ("num_data", ULONG_PTR), + ("data_size", ctypes.wintypes.DWORD), + ("data_loc", ctypes.c_void_p), + ] + + +def _query_pageant(msg): + """ + Communication with the Pageant process is done through a shared + memory-mapped file. + """ + hwnd = _get_pageant_window_object() + if not hwnd: + # Raise a failure to connect exception, pageant isn't running anymore! + return None + + # create a name for the mmap + map_name = f"PageantRequest{thread.get_ident():08x}" + + pymap = _winapi.MemoryMap( + map_name, _AGENT_MAX_MSGLEN, _winapi.get_security_attributes_for_user() + ) + with pymap: + pymap.write(msg) + # Create an array buffer containing the mapped filename + char_buffer = array.array("b", b(map_name) + zero_byte) # noqa + char_buffer_address, char_buffer_size = char_buffer.buffer_info() + # Create a string to use for the SendMessage function call + cds = COPYDATASTRUCT( + _AGENT_COPYDATA_ID, char_buffer_size, char_buffer_address + ) + + response = ctypes.windll.user32.SendMessageA( + hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds) + ) + + if response > 0: + pymap.seek(0) + datalen = pymap.read(4) + retlen = struct.unpack(">I", datalen)[0] + return datalen + pymap.read(retlen) + return None + + +class PageantConnection: + """ + Mock "connection" to an agent which roughly approximates the behavior of + a unix local-domain socket (as used by Agent). Requests are sent to the + pageant daemon via special Windows magick, and responses are buffered back + for subsequent reads. + """ + + def __init__(self): + self._response = None + + def send(self, data): + self._response = _query_pageant(data) + + def recv(self, n): + if self._response is None: + return "" + ret = self._response[:n] + self._response = self._response[n:] + if self._response == "": + self._response = None + return ret + + def close(self): + pass diff --git a/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER b/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/prompt_toolkit-3.0.52.dist-info/METADATA b/lib/prompt_toolkit-3.0.52.dist-info/METADATA new file mode 100644 index 0000000..7fbbd56 --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/METADATA @@ -0,0 +1,172 @@ +Metadata-Version: 2.4 +Name: prompt_toolkit +Version: 3.0.52 +Summary: Library for building powerful interactive command lines in Python +Author: Jonathan Slenders +Project-URL: Homepage, https://github.com/prompt-toolkit/python-prompt-toolkit +Project-URL: Documentation, https://python-prompt-toolkit.readthedocs.io/en/stable/ +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +License-File: LICENSE +License-File: AUTHORS.rst +Requires-Dist: wcwidth +Dynamic: license-file + +Python Prompt Toolkit +===================== + +|AppVeyor| |PyPI| |RTD| |License| |Codecov| + +.. image :: /docs/images/logo_400px.png + +``prompt_toolkit`` *is a library for building powerful interactive command line applications in Python.* + +Read the `documentation on readthedocs +`_. + + +Gallery +******* + +`ptpython `_ is an interactive +Python Shell, build on top of ``prompt_toolkit``. + +.. image :: /docs/images/ptpython.png + +`More examples `_ + + +prompt_toolkit features +*********************** + +``prompt_toolkit`` could be a replacement for `GNU readline +`_, but it can be much +more than that. + +Some features: + +- **Pure Python**. +- Syntax highlighting of the input while typing. (For instance, with a Pygments lexer.) +- Multi-line input editing. +- Advanced code completion. +- Both Emacs and Vi key bindings. (Similar to readline.) +- Even some advanced Vi functionality, like named registers and digraphs. +- Reverse and forward incremental search. +- Works well with Unicode double width characters. (Chinese input.) +- Selecting text for copy/paste. (Both Emacs and Vi style.) +- Support for `bracketed paste `_. +- Mouse support for cursor positioning and scrolling. +- Auto suggestions. (Like `fish shell `_.) +- Multiple input buffers. +- No global state. +- Lightweight, the only dependencies are Pygments and wcwidth. +- Runs on Linux, OS X, FreeBSD, OpenBSD and Windows systems. +- And much more... + +Feel free to create tickets for bugs and feature requests, and create pull +requests if you have nice patches that you would like to share with others. + + +Installation +************ + +:: + + pip install prompt_toolkit + +For Conda, do: + +:: + + conda install -c https://conda.anaconda.org/conda-forge prompt_toolkit + + +About Windows support +********************* + +``prompt_toolkit`` is cross platform, and everything that you build on top +should run fine on both Unix and Windows systems. Windows support is best on +recent Windows 10 builds, for which the command line window supports vt100 +escape sequences. (If not supported, we fall back to using Win32 APIs for color +and cursor movements). + +It's worth noting that the implementation is a "best effort of what is +possible". Both Unix and Windows terminals have their limitations. But in +general, the Unix experience will still be a little better. + + +Getting started +*************** + +The most simple example of the library would look like this: + +.. code:: python + + from prompt_toolkit import prompt + + if __name__ == '__main__': + answer = prompt('Give me some input: ') + print('You said: %s' % answer) + +For more complex examples, have a look in the ``examples`` directory. All +examples are chosen to demonstrate only one thing. Also, don't be afraid to +look at the source code. The implementation of the ``prompt`` function could be +a good start. + +Philosophy +********** + +The source code of ``prompt_toolkit`` should be **readable**, **concise** and +**efficient**. We prefer short functions focusing each on one task and for which +the input and output types are clearly specified. We mostly prefer composition +over inheritance, because inheritance can result in too much functionality in +the same object. We prefer immutable objects where possible (objects don't +change after initialization). Reusability is important. We absolutely refrain +from having a changing global state, it should be possible to have multiple +independent instances of the same code in the same process. The architecture +should be layered: the lower levels operate on primitive operations and data +structures giving -- when correctly combined -- all the possible flexibility; +while at the higher level, there should be a simpler API, ready-to-use and +sufficient for most use cases. Thinking about algorithms and efficiency is +important, but avoid premature optimization. + + +`Projects using prompt_toolkit `_ +*********************************************** + +Special thanks to +***************** + +- `Pygments `_: Syntax highlighter. +- `wcwidth `_: Determine columns needed for a wide characters. + +.. |PyPI| image:: https://img.shields.io/pypi/v/prompt_toolkit.svg + :target: https://pypi.python.org/pypi/prompt-toolkit/ + :alt: Latest Version + +.. |AppVeyor| image:: https://ci.appveyor.com/api/projects/status/32r7s2skrgm9ubva?svg=true + :target: https://ci.appveyor.com/project/prompt-toolkit/python-prompt-toolkit/ + +.. |RTD| image:: https://readthedocs.org/projects/python-prompt-toolkit/badge/ + :target: https://python-prompt-toolkit.readthedocs.io/en/master/ + +.. |License| image:: https://img.shields.io/github/license/prompt-toolkit/python-prompt-toolkit.svg + :target: https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/LICENSE + +.. |Codecov| image:: https://codecov.io/gh/prompt-toolkit/python-prompt-toolkit/branch/master/graphs/badge.svg?style=flat + :target: https://codecov.io/gh/prompt-toolkit/python-prompt-toolkit/ + diff --git a/lib/prompt_toolkit-3.0.52.dist-info/RECORD b/lib/prompt_toolkit-3.0.52.dist-info/RECORD new file mode 100644 index 0000000..88e587b --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/RECORD @@ -0,0 +1,298 @@ +prompt_toolkit-3.0.52.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +prompt_toolkit-3.0.52.dist-info/METADATA,sha256=gNS9QKahcd_SXDMrrr7h_YciNC8TPbU4Xi1XGaXaLik,6414 +prompt_toolkit-3.0.52.dist-info/RECORD,, +prompt_toolkit-3.0.52.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91 +prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst,sha256=09xixryENmWElauJrqN1Eef6k5HSgmVyOcnPuA29QuU,148 +prompt_toolkit-3.0.52.dist-info/licenses/LICENSE,sha256=MDV02b3YXHV9YCUBeUK_F7ru3yd49ivX9CXQfYgPTEo,1493 +prompt_toolkit-3.0.52.dist-info/top_level.txt,sha256=5rJXrEGx6st4KkmhOPR6l0ITDbV53x_Xy6MurOukXfA,15 +prompt_toolkit/__init__.py,sha256=zWTvKCSL-Qb_hO8opATHynv2gNdm0YMCA9x3obLJUH4,1376 +prompt_toolkit/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc,, +prompt_toolkit/__pycache__/buffer.cpython-314.pyc,, +prompt_toolkit/__pycache__/cache.cpython-314.pyc,, +prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc,, +prompt_toolkit/__pycache__/data_structures.cpython-314.pyc,, +prompt_toolkit/__pycache__/document.cpython-314.pyc,, +prompt_toolkit/__pycache__/enums.cpython-314.pyc,, +prompt_toolkit/__pycache__/history.cpython-314.pyc,, +prompt_toolkit/__pycache__/keys.cpython-314.pyc,, +prompt_toolkit/__pycache__/log.cpython-314.pyc,, +prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc,, +prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc,, +prompt_toolkit/__pycache__/renderer.cpython-314.pyc,, +prompt_toolkit/__pycache__/search.cpython-314.pyc,, +prompt_toolkit/__pycache__/selection.cpython-314.pyc,, +prompt_toolkit/__pycache__/token.cpython-314.pyc,, +prompt_toolkit/__pycache__/utils.cpython-314.pyc,, +prompt_toolkit/__pycache__/validation.cpython-314.pyc,, +prompt_toolkit/__pycache__/win32_types.cpython-314.pyc,, +prompt_toolkit/application/__init__.py,sha256=rat9iPhYTmo7nd2BU8xZSU_-AfJpjnnBmxe9y3TQivM,657 +prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/application/__pycache__/application.cpython-314.pyc,, +prompt_toolkit/application/__pycache__/current.cpython-314.pyc,, +prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc,, +prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc,, +prompt_toolkit/application/application.py,sha256=iTjvCjKjtPss4_L8MYidBFFQLumfHLgrkFQSGqE4O3c,63242 +prompt_toolkit/application/current.py,sha256=IyZmMpzOWEISG_o06Bj2vzgdDTxI7uAQnbTHFJGuL0s,6209 +prompt_toolkit/application/dummy.py,sha256=BCfThUgz5Eb5fWJSKBVeJaA5kwksw8jJQtN6g61xMXM,1619 +prompt_toolkit/application/run_in_terminal.py,sha256=qgTfpG3qGP4wRy9l_7zU8P7s59CARIvagulTyPn_oEg,3767 +prompt_toolkit/auto_suggest.py,sha256=qSiaxlaKjLyNEJ8bJN0641gqsIzZ3LB2cOyq88xBQb4,5798 +prompt_toolkit/buffer.py,sha256=VkAbKTJV7441aO4Ei-ozqc8IBlNPEnLGAt5t42tB6jg,74513 +prompt_toolkit/cache.py,sha256=Lo3ewsEIgn-LQBYNni79w74u5LSvvuVYF0e6giEArQg,3823 +prompt_toolkit/clipboard/__init__.py,sha256=yK0LonIfEZRyoXqcgLdh8kjOhechjO-Ej2-C1g3VegQ,439 +prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc,, +prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc,, +prompt_toolkit/clipboard/base.py,sha256=rucEv1kKfvZUj6bwGRz04uSSTZie7rvnKUnyVXb2vv4,2515 +prompt_toolkit/clipboard/in_memory.py,sha256=U_iY6UUevkKMfTvir_XMsD0qwuLVKuoTeRdjkZW-A6I,1060 +prompt_toolkit/clipboard/pyperclip.py,sha256=H9HOlyGW0XItvx_Ji64zBQkiQPfLb6DFAw5J5irzK28,1160 +prompt_toolkit/completion/__init__.py,sha256=8Hm2yJ1nqBkaC-R9ugELgjhU32U308V89F6bJG0QDYo,992 +prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/completion/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc,, +prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc,, +prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc,, +prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc,, +prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc,, +prompt_toolkit/completion/base.py,sha256=T7212aScNaGMaSrDIwsJIXeXLIM_eVCIcScNcDPZYwI,16103 +prompt_toolkit/completion/deduplicate.py,sha256=QibqYI23GPjsbyxaxiNoqAbKawzHmfYOlxeW2HPFbTE,1436 +prompt_toolkit/completion/filesystem.py,sha256=Z_RR72e14bVavdWnbxECw23qCt_TWTY9R6DpVqW7vxE,3949 +prompt_toolkit/completion/fuzzy_completer.py,sha256=RnREvA7y6nC7LKGqZUEvtuSm8eXVQYheJTsnhUvRbmM,7738 +prompt_toolkit/completion/nested.py,sha256=ig2Qy4dLyDvOT6O8Qb-iRZLWzlT11S5tVQ3GFZmpm-U,3844 +prompt_toolkit/completion/word_completer.py,sha256=VF1S7YCxYqI3pKmVXJaD82eMW1ZMq8_zAAIS1XKGU5M,3435 +prompt_toolkit/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/contrib/completers/__init__.py,sha256=qJB_xNFGbhfiDv_zUaox9mkQEGqBYqP_jfByQDb93hA,103 +prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc,, +prompt_toolkit/contrib/completers/system.py,sha256=0Hc2dziheEx2qNog4YOl-4Tu8Fg5Dx2xjNURTx09BDg,2057 +prompt_toolkit/contrib/regular_languages/__init__.py,sha256=cgMQkakD4FbvLUozDGucRRFOk8yScfcKfqOMpCtvAPo,3279 +prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc,, +prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc,, +prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc,, +prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc,, +prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc,, +prompt_toolkit/contrib/regular_languages/compiler.py,sha256=3tnUJCE2jCcVI63vcpI0kG4KfuqIatSQRb8-F5UCgsI,21948 +prompt_toolkit/contrib/regular_languages/completion.py,sha256=jESF35RaYWj_rnT-OZc_zC9QZXYvPao4JZ8wx7yS3KM,3468 +prompt_toolkit/contrib/regular_languages/lexer.py,sha256=DBgyek9LkfJv6hz24eOaVM--w9Qaw4zIMWusMvGHBts,3415 +prompt_toolkit/contrib/regular_languages/regex_parser.py,sha256=zWGJfQSjomvdj2rD7MPpn2pWOUR7VMv4su5iAV0jzM4,7732 +prompt_toolkit/contrib/regular_languages/validation.py,sha256=4k5wxgUFc_KTOW5PmmZOrWb-Z-HjX8fjjKqul-oR8uc,2059 +prompt_toolkit/contrib/ssh/__init__.py,sha256=UcRG2wc28EEKtFEudoIXz_DFzWKKQjAVSv6cf-ufPiM,180 +prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc,, +prompt_toolkit/contrib/ssh/server.py,sha256=81McNn6r0Cbu9SPceH7fa5QirAnteHmNh1Gk4dFpgvI,6130 +prompt_toolkit/contrib/telnet/__init__.py,sha256=NyUfsmJdafGiUxD9gzYQNlVdHu_ILDH7F57VJw8efUM,104 +prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc,, +prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc,, +prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc,, +prompt_toolkit/contrib/telnet/log.py,sha256=LcFRDyRxoRKSZsVRVpBOrEgsEt_LQLyUHKtgVZklopI,167 +prompt_toolkit/contrib/telnet/protocol.py,sha256=2i-JYfaAse-uFWtNdVEoP_Q-OMbkl3YbUfv_wvaaS3k,5584 +prompt_toolkit/contrib/telnet/server.py,sha256=S2UGX30O0aYjYV_oy4hnJM7f7xllGxXA-WUnlKqLNTw,13461 +prompt_toolkit/cursor_shapes.py,sha256=k5g5yJONGl1ITgy29KX9yzspJvIJ6Jbbwd7WkYC9Z-4,3721 +prompt_toolkit/data_structures.py,sha256=w0BZy6Fpx4se-kAI9Kj8Q7lAKLln8U_Em_ncpqnC1xY,212 +prompt_toolkit/document.py,sha256=s83zyjuwE8pvEdFN8y46pRYa_YRveVBkf9As3xlYU-w,40547 +prompt_toolkit/enums.py,sha256=F3q9JmH9vhpMLA2OKKN7RrNQu_YDlNWoPU-0qsTUuAs,358 +prompt_toolkit/eventloop/__init__.py,sha256=pxSkV_zybeoj6Ff3lgNHhbD5ENmBW9mk_XkiyeRL_OY,730 +prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc,, +prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc,, +prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc,, +prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc,, +prompt_toolkit/eventloop/async_generator.py,sha256=nozLJR4z2MJKV7Qi0hsknA2mb1Jcp7XJx-AdUEDhDhw,3933 +prompt_toolkit/eventloop/inputhook.py,sha256=LDElZtmg-kLQiItMS8CFPxtLzxV8QzohWHsWUvw3h00,6130 +prompt_toolkit/eventloop/utils.py,sha256=VhYmsDZmRwVXnEPBF_C2LpiW-ranPn6EIXWIuMa6XaU,3200 +prompt_toolkit/eventloop/win32.py,sha256=wrLJVOtOw_tqVOeK6ttNF47Sk2oX342dLN1pxKBLCL4,2286 +prompt_toolkit/filters/__init__.py,sha256=2YSVwf4EnLf1VOXYmb8Dr0WoA93XGGO0iCUIr14KGXQ,1990 +prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/filters/__pycache__/app.cpython-314.pyc,, +prompt_toolkit/filters/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc,, +prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc,, +prompt_toolkit/filters/app.py,sha256=GAQzU3An_pUIIV0ja4RV7Wfl0CbLI-U1HUMOItrCKNI,10374 +prompt_toolkit/filters/base.py,sha256=asrgKE-gzYlRLrS4w3kMFimvZtXQ9pk252Vs5ShVeeM,6855 +prompt_toolkit/filters/cli.py,sha256=QGV7JT7-BUXpPXNzBLUcNH3GI69ugFZCDV1nylOjq78,1867 +prompt_toolkit/filters/utils.py,sha256=4nOjHPEd451Pj3qpfg40fq3XSnt1kmq3WoAbhu2NV-8,859 +prompt_toolkit/formatted_text/__init__.py,sha256=aQtNhxOhIa_HmvlNOQ2RGGpplg-KX3sYFJWiXgNfQxY,1509 +prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc,, +prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc,, +prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc,, +prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc,, +prompt_toolkit/formatted_text/ansi.py,sha256=9jajRZ4bIZ4rRDoSNMyDKF0ctErcEwoyFbvh96ownXU,9824 +prompt_toolkit/formatted_text/base.py,sha256=X3y5QIPH2IS9LesYzXneELtT4zGpik8gd-UQVh6I2bE,5162 +prompt_toolkit/formatted_text/html.py,sha256=-88VwuuCLRNkzEgK8FJKOHT9NDh939BxH8vGivvILdU,4374 +prompt_toolkit/formatted_text/pygments.py,sha256=sK-eFFzOnD2sgadVLgNkW-xOuTw_uIf8_z06DZ4CA8g,780 +prompt_toolkit/formatted_text/utils.py,sha256=r6tPtwo6dqvqf9gqZ7ARyvtNUjDDq6QZqrTWg6EMFuQ,3044 +prompt_toolkit/history.py,sha256=S9W9SgL83QftMQANdjdjBMm-yGmeM51_qCRRC1H4Mr8,9441 +prompt_toolkit/input/__init__.py,sha256=7g6kwNanG4Ml12FFdj9E1ivChpXWcfRUMUJzmTQMS7U,273 +prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/win32.cpython-314.pyc,, +prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc,, +prompt_toolkit/input/ansi_escape_sequences.py,sha256=h7714SZgs2z80PZRVxsCfHHJjtEUmWlToVCBtFFvfR4,13663 +prompt_toolkit/input/base.py,sha256=nqdM61RerkAnTBxQCqxxIHErKvaEXUerPIn--gr6NLs,4030 +prompt_toolkit/input/defaults.py,sha256=a-vczSh7kngFArLhFsJ2CXNdkx5WQlzilxHLdzGDkFw,2500 +prompt_toolkit/input/posix_pipe.py,sha256=B_JS2-FB6Sk0da9gSH0NnhcUCkp3bw0m1-ogMOHmmcE,3158 +prompt_toolkit/input/posix_utils.py,sha256=ySaEGnt_IwG5nzxcpILgEXC60mbrIAbC3ZZ6kuE9zCw,3973 +prompt_toolkit/input/typeahead.py,sha256=mAaf5_XKTLpao1kw9ORIrhGGEz9gvu4oc-iZKKMQz3k,2545 +prompt_toolkit/input/vt100.py,sha256=soxxSLU7fwp6yn77j5gCYUZroEp7KBKm4a3Zn4vRAsk,10514 +prompt_toolkit/input/vt100_parser.py,sha256=qDrNbsnPukZblfyjgfjCvzMv8xQKRz0M84UvUWq7P44,8407 +prompt_toolkit/input/win32.py,sha256=gF_IXqZhFKeAbKWYGkMKRBG3kF4yqDgDzWG2tX073IM,31410 +prompt_toolkit/input/win32_pipe.py,sha256=OvjKHN5xfEoGHLygWwayyeB0RolHL6YHLNeOMK-54LU,4700 +prompt_toolkit/key_binding/__init__.py,sha256=IZWqJLBjQaQMfo0SJTjqJKQH0TZcSNa2Cdln-M4z8JI,447 +prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc,, +prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc,, +prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc,, +prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc,, +prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc,, +prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc,, +prompt_toolkit/key_binding/bindings/auto_suggest.py,sha256=4PrJVgIK_Nt2o3RtVtuRm2aFPGrackhuMCBVNtjPj7M,1855 +prompt_toolkit/key_binding/bindings/basic.py,sha256=Fp9mj-RYZlGmAU9UV9wIIEnlxELN7NJ0qakMVH7MuRU,7229 +prompt_toolkit/key_binding/bindings/completion.py,sha256=6nR3WfGe7FDsjq1xTsDazeajkV9KBLpCYQi3klujdLU,6903 +prompt_toolkit/key_binding/bindings/cpr.py,sha256=181XQNZ0-sgL-vV2B67aRitTFHadogvMUh6LWVMUTV0,786 +prompt_toolkit/key_binding/bindings/emacs.py,sha256=trIZUu8e5kJGSaq6Ndb-Exz4NdHV9SjUsfsw_UM8c6o,19634 +prompt_toolkit/key_binding/bindings/focus.py,sha256=LIP4InccUUvD7I4NZrqtY9WjVfO_wJLyrVcoxAw92uU,507 +prompt_toolkit/key_binding/bindings/mouse.py,sha256=6JPr0BqzFfLEVb7Ek_WO0CejUcwq0jIrrNwvSGkHeus,18586 +prompt_toolkit/key_binding/bindings/named_commands.py,sha256=jdkqQ-ltNYC2BgIW1QdG7Qx4mWIod2Ps6C2TpL6NJ-Y,18407 +prompt_toolkit/key_binding/bindings/open_in_editor.py,sha256=bgVmeDmVtHsgzJnc59b-dOZ-nO6WydBYI_7aOWMpe5c,1356 +prompt_toolkit/key_binding/bindings/page_navigation.py,sha256=RPLUEZuZvekErPazex7pK0c6LxeV9LshewBHp012iMI,2392 +prompt_toolkit/key_binding/bindings/scroll.py,sha256=hQeQ0m2AStUKjVNDXfa9DTMw3WS5qzW1n3gU0fkfWFk,5613 +prompt_toolkit/key_binding/bindings/search.py,sha256=rU6VYra1IDzN6mG4mrbGivrZ-hjy_kZcjsKqmdVJKAE,2632 +prompt_toolkit/key_binding/bindings/vi.py,sha256=TSglqzPZU9VMernOvn09GVxObFXpXuyCSiH9i1MpIIo,75602 +prompt_toolkit/key_binding/defaults.py,sha256=JZJTshyBV39cWH2AT7xDP9AXOiyXQpjaI-ckePTi7os,1975 +prompt_toolkit/key_binding/digraphs.py,sha256=rZvh9AdY5Te6bSlIHRQNskJYVIONYahYuu-w9Pti5yM,32785 +prompt_toolkit/key_binding/emacs_state.py,sha256=ZJBWcLTzgtRkUW9UiDuI-SRrnlLsxu3IrTOK0_UQt5Y,884 +prompt_toolkit/key_binding/key_bindings.py,sha256=HEbemzMuSdO4fhHc_p7em02dd-rt7ZqxGzs-Fpu6LcY,20933 +prompt_toolkit/key_binding/key_processor.py,sha256=0WLK4dcU8klL2Xna_RKxOpsW7t8ld67Y9Xmto3U-n0E,17555 +prompt_toolkit/key_binding/vi_state.py,sha256=GAlH1xPvYoJnRUM9s3bTMlWh5UnIZ40fFtYInKMC2x0,3337 +prompt_toolkit/keys.py,sha256=nDkIqJbm_dRsVjArp7oItGKIFAAnSxcSniSwc1O-BYA,4916 +prompt_toolkit/layout/__init__.py,sha256=gNbniLmlvkWwPE6Kg2ykyZJRTOKsWnHbwUjyO-VFDP8,3603 +prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc,, +prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc,, +prompt_toolkit/layout/containers.py,sha256=BpR90-WNGPBcoYyCCSorAn2eHTz-1aoeQQO7QGqbuYU,100179 +prompt_toolkit/layout/controls.py,sha256=9h6425oGeBwLO85MBNdHSh6XsrtEay5JwuIX-fuzsVI,35993 +prompt_toolkit/layout/dimension.py,sha256=GOd3W6bxtH9tPaGo31brUwF94K2UOz-ldB_6rearjSI,6948 +prompt_toolkit/layout/dummy.py,sha256=8zB3MwDDd4RpI880WUKhv719tTzy5bXS9gm9zdkBZ10,1047 +prompt_toolkit/layout/layout.py,sha256=VXqWAoL3EviGn4CxtOrFJekMALvl9xff1bTSnE-gXF8,13960 +prompt_toolkit/layout/margins.py,sha256=bt-IvD03uQvmLVYvGZLqPLluR6kUlBRBAGJwCc8F7II,10375 +prompt_toolkit/layout/menus.py,sha256=B4H2oCPF48gLy9cB0vkdGIoH_8gGyj95TDHtfxXRVSw,27195 +prompt_toolkit/layout/mouse_handlers.py,sha256=lwbGSdpn6_pcK7HQWJ6IvHsxTf1_ahBew4pkmtU6zUM,1589 +prompt_toolkit/layout/processors.py,sha256=0VE4UIGRzyXvDO4XqCB7LXNG9WkSxLz7FW7toOvHDSE,34296 +prompt_toolkit/layout/screen.py,sha256=2PWdPDkQxtJrMSv9oqdZrWa7ChCnC7J4SvfVIithi5E,10113 +prompt_toolkit/layout/scrollable_pane.py,sha256=JQtPfafU61RJt3MzGW2wsw96o1sjJH0g2DSVyO7J6qA,19264 +prompt_toolkit/layout/utils.py,sha256=qot9clyeG3FoplfAS2O6QxmnnA_PDln4-dUJ5Hu76fQ,2371 +prompt_toolkit/lexers/__init__.py,sha256=QldV9b8J2Kb9Exyv2fDss-YRzP07z2FYAhwPN4coWn8,409 +prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc,, +prompt_toolkit/lexers/base.py,sha256=XdyKLj4rD25CVCqSCfElWE3ppBN1LGQ9fRLPi1oYfl0,2350 +prompt_toolkit/lexers/pygments.py,sha256=it89LjsltZpzlQJPb95GX4GdMu7gq1J1QzWC29lCQi4,11922 +prompt_toolkit/log.py,sha256=6typpL_jnewdUc3j2OoplVLwnP9dSMOkIsJy_sgR9IY,153 +prompt_toolkit/mouse_events.py,sha256=4mUHJbG8WrrQznw7z_jsOrdmldC5ZMRM4gDDUy51pyk,2473 +prompt_toolkit/output/__init__.py,sha256=GVlT-U_W0EuIP-c1Qjyp0DN6Fl2PsCEhFzjUMRHsGWI,280 +prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/win32.cpython-314.pyc,, +prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc,, +prompt_toolkit/output/base.py,sha256=o74Vok7cXLxgHoAaqKHQAGcNZILn5B5g6Z0pUXU6x7s,8348 +prompt_toolkit/output/color_depth.py,sha256=KEFTlxCYTqOvA-VDx4wUb8G6HaYD5Hbf5GKmPZwssCs,1569 +prompt_toolkit/output/conemu.py,sha256=_w2IEFR-mXsaMFINgZITiJNRCS9QowLUxeskPEpz2GE,1865 +prompt_toolkit/output/defaults.py,sha256=72RecTuugrjvfZinbvsFRYDwMcczE9Zw3ttmmiG0Ivg,3689 +prompt_toolkit/output/flush_stdout.py,sha256=ReT0j0IwVJEcth7VJj2zE6UcY0OVz5Ut1rpANnbCyYQ,3236 +prompt_toolkit/output/plain_text.py,sha256=VnjoDmy0pKQoubXXQJQ_MljoDYi1FcLdNZB2KN_TQIs,3296 +prompt_toolkit/output/vt100.py,sha256=a_vswyv0w0AyWZoI5O2a13y9q0Fj3cxdK2wqwi2zS0U,23474 +prompt_toolkit/output/win32.py,sha256=_aM4-pJq91l2mFkpBtRHksQ3xaUjw_Jj0yVJdle1Vbo,22639 +prompt_toolkit/output/windows10.py,sha256=yf0i1xAs-mbqOCwq25K78hkJjju1jXZ5b0e-w9aSBBA,4362 +prompt_toolkit/patch_stdout.py,sha256=8gEaQdqykdBczlvp3FrOjDlEG02yeXoYKrDAGqj48Wg,9477 +prompt_toolkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +prompt_toolkit/renderer.py,sha256=h4r7bShanQyvh9nrSQfxvHZUPT6ZPUH1kD5Nbeu2RwY,29398 +prompt_toolkit/search.py,sha256=6Go_LtBeBlIMkdUCqb-WFCBKLchd70kgtccqP5dyv08,6951 +prompt_toolkit/selection.py,sha256=P6zQOahBqqt1YZmfQ2-V9iJjOo4cxl0bdmU_-0jezJI,1274 +prompt_toolkit/shortcuts/__init__.py,sha256=ENuc0MPbDtxMnPNhYKUeYDzp4_J_ycQ-aQDfgg7imRw,1020 +prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc,, +prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc,, +prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc,, +prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc,, +prompt_toolkit/shortcuts/choice_input.py,sha256=Jzs6y2x6BLo5Otetx3szDKks7ikLwqlx_gYhXYP7HWY,10523 +prompt_toolkit/shortcuts/dialogs.py,sha256=gFibLlbaii8ijuurk9TpbNi5fMTHu99T6m1wfFilbE8,9007 +prompt_toolkit/shortcuts/progress_bar/__init__.py,sha256=QeAssmFBDPCC5VRoObAp4UkebwETP3qS7-na4acstWM,540 +prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc,, +prompt_toolkit/shortcuts/progress_bar/base.py,sha256=_cqp7coZMFDc7ZoAUL1iz3fL1Dt5hw3hi1HEfBvUpK8,14402 +prompt_toolkit/shortcuts/progress_bar/formatters.py,sha256=VfRADwUm8op-DzoM51UrKI8pSa1T1LAz5q9VMUW2siI,11739 +prompt_toolkit/shortcuts/prompt.py,sha256=JpGwxOW7nd0maObh8H4njxJ_PLALJozCiW_xCYBf8ug,60747 +prompt_toolkit/shortcuts/utils.py,sha256=NNjBY0Brkcb13Gxhh7Yc72_YpDFsQbkIlm7ZXvW3rK0,6950 +prompt_toolkit/styles/__init__.py,sha256=7N1NNE1gTQo5mjT9f7mRwRodkrBoNpT9pmqWK-lrSeY,1640 +prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/styles/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc,, +prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc,, +prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc,, +prompt_toolkit/styles/__pycache__/style.cpython-314.pyc,, +prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc,, +prompt_toolkit/styles/base.py,sha256=J0Q7q4Nyrxig6KVLMnxZ-Jh3HlbPSKRPDSgtHjGMx_U,5064 +prompt_toolkit/styles/defaults.py,sha256=TRnP1PeuauYa_Ru1PpJ_ImsfaldvLE1JjmPV8tvfJjs,8699 +prompt_toolkit/styles/named_colors.py,sha256=yZ30oKB-fCRk6RMASYg8q3Uz2zgdfy_YNbuQWYpyYas,4367 +prompt_toolkit/styles/pygments.py,sha256=yWJEcvYCFo1e2EN9IF5HWpxHQ104J0HOJg1LUsSA9oM,1974 +prompt_toolkit/styles/style.py,sha256=GXYGYLJolUiErIHmuxk5VuPgwehoV2l8Cvfdy9_GKc0,13263 +prompt_toolkit/styles/style_transformation.py,sha256=cGaOo-jqhP79QoEHLQxrOZo9QMrxWxtXgfXKsHlx1Jg,12427 +prompt_toolkit/token.py,sha256=do3EnxLrCDVbq47MzJ2vqSYps-CjVKWNCWzCZgdf5Jo,121 +prompt_toolkit/utils.py,sha256=7O8hILpI2VZb0KoC7J-5z1S2aXICf_kwtmRq5xdfDTg,8631 +prompt_toolkit/validation.py,sha256=XTdmExMgaqj-Whym9yYyQxOAaKce97KYyyGXwCxMr-A,5807 +prompt_toolkit/widgets/__init__.py,sha256=RZXj6UzZWFuxOQXc1TwHLIwwZYJU-YBAaV4oLrC2dCA,1218 +prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc,, +prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc,, +prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc,, +prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc,, +prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc,, +prompt_toolkit/widgets/base.py,sha256=MEqDogz8BCH7KEQEH0L6w9BndCFhsiaBhg0ZFm91VQo,35546 +prompt_toolkit/widgets/dialogs.py,sha256=K2ACcf0rKXwpBQGQcjSTq2aNeSInGmklzZRPnhdtZTc,3380 +prompt_toolkit/widgets/menus.py,sha256=SeX-llaTpF1pVak2lw37mAP0SFDONIRZT5oq23mARg8,13419 +prompt_toolkit/widgets/toolbars.py,sha256=MoxOxaa8Yi3nJvH4G8OCwlNuwx3XWUJ07J0a7D17_w0,12178 +prompt_toolkit/win32_types.py,sha256=3xVjabRA3Q-RN2x3DLqTOrstuYj4_uCq6w2i8t6LZ6E,5551 diff --git a/lib/prompt_toolkit-3.0.52.dist-info/WHEEL b/lib/prompt_toolkit-3.0.52.dist-info/WHEEL new file mode 100644 index 0000000..e7fa31b --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst b/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst new file mode 100644 index 0000000..f7c8f60 --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst @@ -0,0 +1,11 @@ +Authors +======= + +Creator +------- +Jonathan Slenders + +Contributors +------------ + +- Amjith Ramanujam diff --git a/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE b/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE new file mode 100644 index 0000000..e1720e0 --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014, Jonathan Slenders +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +* Neither the name of the {organization} nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt b/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt new file mode 100644 index 0000000..29392df --- /dev/null +++ b/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt @@ -0,0 +1 @@ +prompt_toolkit diff --git a/lib/prompt_toolkit/__init__.py b/lib/prompt_toolkit/__init__.py new file mode 100644 index 0000000..7a0c4ce --- /dev/null +++ b/lib/prompt_toolkit/__init__.py @@ -0,0 +1,54 @@ +""" +prompt_toolkit +============== + +Author: Jonathan Slenders + +Description: prompt_toolkit is a Library for building powerful interactive + command lines in Python. It can be a replacement for GNU + Readline, but it can be much more than that. + +See the examples directory to learn about the usage. + +Probably, to get started, you might also want to have a look at +`prompt_toolkit.shortcuts.prompt`. +""" + +from __future__ import annotations + +import re +from importlib import metadata + +# note: this is a bit more lax than the actual pep 440 to allow for a/b/rc/dev without a number +pep440 = re.compile( + r"^([1-9]\d*!)?(0|[1-9]\d*)(\.(0|[1-9]\d*))*((a|b|rc)(0|[1-9]\d*)?)?(\.post(0|[1-9]\d*))?(\.dev(0|[1-9]\d*)?)?$", + re.UNICODE, +) +from .application import Application +from .formatted_text import ANSI, HTML +from .shortcuts import PromptSession, choice, print_formatted_text, prompt + +# Don't forget to update in `docs/conf.py`! +__version__ = metadata.version("prompt_toolkit") + +assert pep440.match(__version__) + +# Version tuple. +VERSION = tuple(int(v.rstrip("abrc")) for v in __version__.split(".")[:3]) + + +__all__ = [ + # Application. + "Application", + # Shortcuts. + "prompt", + "choice", + "PromptSession", + "print_formatted_text", + # Formatted text. + "HTML", + "ANSI", + # Version info. + "__version__", + "VERSION", +] diff --git a/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..cf739b5 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc new file mode 100644 index 0000000..a9029a1 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc new file mode 100644 index 0000000..351cd67 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc new file mode 100644 index 0000000..68fe919 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc new file mode 100644 index 0000000..bbd84c5 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc new file mode 100644 index 0000000..0104bd0 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc new file mode 100644 index 0000000..553df62 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc new file mode 100644 index 0000000..e46c824 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc new file mode 100644 index 0000000..6499fbd Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc new file mode 100644 index 0000000..d77839f Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc new file mode 100644 index 0000000..66462db Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc new file mode 100644 index 0000000..73714ce Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc new file mode 100644 index 0000000..e8d5487 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc new file mode 100644 index 0000000..9637555 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc new file mode 100644 index 0000000..c414716 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc new file mode 100644 index 0000000..aebfa03 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc new file mode 100644 index 0000000..0424c3f Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..c7e2f90 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc new file mode 100644 index 0000000..24bc1c3 Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc b/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc new file mode 100644 index 0000000..a3ef6ab Binary files /dev/null and b/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/application/__init__.py b/lib/prompt_toolkit/application/__init__.py new file mode 100644 index 0000000..569d8c0 --- /dev/null +++ b/lib/prompt_toolkit/application/__init__.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from .application import Application +from .current import ( + AppSession, + create_app_session, + create_app_session_from_tty, + get_app, + get_app_or_none, + get_app_session, + set_app, +) +from .dummy import DummyApplication +from .run_in_terminal import in_terminal, run_in_terminal + +__all__ = [ + # Application. + "Application", + # Current. + "AppSession", + "get_app_session", + "create_app_session", + "create_app_session_from_tty", + "get_app", + "get_app_or_none", + "set_app", + # Dummy. + "DummyApplication", + # Run_in_terminal + "in_terminal", + "run_in_terminal", +] diff --git a/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..4ce237e Binary files /dev/null and b/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc b/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc new file mode 100644 index 0000000..0173714 Binary files /dev/null and b/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc b/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc new file mode 100644 index 0000000..54d0463 Binary files /dev/null and b/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc b/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc new file mode 100644 index 0000000..489f5e6 Binary files /dev/null and b/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc b/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc new file mode 100644 index 0000000..a80fc2c Binary files /dev/null and b/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/application/application.py b/lib/prompt_toolkit/application/application.py new file mode 100644 index 0000000..f27ada1 --- /dev/null +++ b/lib/prompt_toolkit/application/application.py @@ -0,0 +1,1630 @@ +from __future__ import annotations + +import asyncio +import contextvars +import os +import re +import signal +import sys +import threading +import time +from asyncio import ( + AbstractEventLoop, + Future, + Task, + ensure_future, + get_running_loop, + sleep, +) +from contextlib import ExitStack, contextmanager +from subprocess import Popen +from traceback import format_tb +from typing import ( + Any, + Callable, + Coroutine, + Generator, + Generic, + Hashable, + Iterable, + Iterator, + TypeVar, + cast, + overload, +) + +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.cache import SimpleCache +from prompt_toolkit.clipboard import Clipboard, InMemoryClipboard +from prompt_toolkit.cursor_shapes import AnyCursorShapeConfig, to_cursor_shape_config +from prompt_toolkit.data_structures import Size +from prompt_toolkit.enums import EditingMode +from prompt_toolkit.eventloop import ( + InputHook, + get_traceback_from_context, + new_eventloop_with_inputhook, + run_in_executor_with_context, +) +from prompt_toolkit.eventloop.utils import call_soon_threadsafe +from prompt_toolkit.filters import Condition, Filter, FilterOrBool, to_filter +from prompt_toolkit.formatted_text import AnyFormattedText +from prompt_toolkit.input.base import Input +from prompt_toolkit.input.typeahead import get_typeahead, store_typeahead +from prompt_toolkit.key_binding.bindings.page_navigation import ( + load_page_navigation_bindings, +) +from prompt_toolkit.key_binding.defaults import load_key_bindings +from prompt_toolkit.key_binding.emacs_state import EmacsState +from prompt_toolkit.key_binding.key_bindings import ( + Binding, + ConditionalKeyBindings, + GlobalOnlyKeyBindings, + KeyBindings, + KeyBindingsBase, + KeysTuple, + merge_key_bindings, +) +from prompt_toolkit.key_binding.key_processor import KeyPressEvent, KeyProcessor +from prompt_toolkit.key_binding.vi_state import ViState +from prompt_toolkit.keys import Keys +from prompt_toolkit.layout.containers import Container, Window +from prompt_toolkit.layout.controls import BufferControl, UIControl +from prompt_toolkit.layout.dummy import create_dummy_layout +from prompt_toolkit.layout.layout import Layout, walk +from prompt_toolkit.output import ColorDepth, Output +from prompt_toolkit.renderer import Renderer, print_formatted_text +from prompt_toolkit.search import SearchState +from prompt_toolkit.styles import ( + BaseStyle, + DummyStyle, + DummyStyleTransformation, + DynamicStyle, + StyleTransformation, + default_pygments_style, + default_ui_style, + merge_styles, +) +from prompt_toolkit.utils import Event, in_main_thread + +from .current import get_app_session, set_app +from .run_in_terminal import in_terminal, run_in_terminal + +__all__ = [ + "Application", +] + + +E = KeyPressEvent +_AppResult = TypeVar("_AppResult") +ApplicationEventHandler = Callable[["Application[_AppResult]"], None] + +_SIGWINCH = getattr(signal, "SIGWINCH", None) +_SIGTSTP = getattr(signal, "SIGTSTP", None) + + +class Application(Generic[_AppResult]): + """ + The main Application class! + This glues everything together. + + :param layout: A :class:`~prompt_toolkit.layout.Layout` instance. + :param key_bindings: + :class:`~prompt_toolkit.key_binding.KeyBindingsBase` instance for + the key bindings. + :param clipboard: :class:`~prompt_toolkit.clipboard.Clipboard` to use. + :param full_screen: When True, run the application on the alternate screen buffer. + :param color_depth: Any :class:`~.ColorDepth` value, a callable that + returns a :class:`~.ColorDepth` or `None` for default. + :param erase_when_done: (bool) Clear the application output when it finishes. + :param reverse_vi_search_direction: Normally, in Vi mode, a '/' searches + forward and a '?' searches backward. In Readline mode, this is usually + reversed. + :param min_redraw_interval: Number of seconds to wait between redraws. Use + this for applications where `invalidate` is called a lot. This could cause + a lot of terminal output, which some terminals are not able to process. + + `None` means that every `invalidate` will be scheduled right away + (which is usually fine). + + When one `invalidate` is called, but a scheduled redraw of a previous + `invalidate` call has not been executed yet, nothing will happen in any + case. + + :param max_render_postpone_time: When there is high CPU (a lot of other + scheduled calls), postpone the rendering max x seconds. '0' means: + don't postpone. '.5' means: try to draw at least twice a second. + + :param refresh_interval: Automatically invalidate the UI every so many + seconds. When `None` (the default), only invalidate when `invalidate` + has been called. + + :param terminal_size_polling_interval: Poll the terminal size every so many + seconds. Useful if the applications runs in a thread other then then + main thread where SIGWINCH can't be handled, or on Windows. + + Filters: + + :param mouse_support: (:class:`~prompt_toolkit.filters.Filter` or + boolean). When True, enable mouse support. + :param paste_mode: :class:`~prompt_toolkit.filters.Filter` or boolean. + :param editing_mode: :class:`~prompt_toolkit.enums.EditingMode`. + + :param enable_page_navigation_bindings: When `True`, enable the page + navigation key bindings. These include both Emacs and Vi bindings like + page-up, page-down and so on to scroll through pages. Mostly useful for + creating an editor or other full screen applications. Probably, you + don't want this for the implementation of a REPL. By default, this is + enabled if `full_screen` is set. + + Callbacks (all of these should accept an + :class:`~prompt_toolkit.application.Application` object as input.) + + :param on_reset: Called during reset. + :param on_invalidate: Called when the UI has been invalidated. + :param before_render: Called right before rendering. + :param after_render: Called right after rendering. + + I/O: + (Note that the preferred way to change the input/output is by creating an + `AppSession` with the required input/output objects. If you need multiple + applications running at the same time, you have to create a separate + `AppSession` using a `with create_app_session():` block. + + :param input: :class:`~prompt_toolkit.input.Input` instance. + :param output: :class:`~prompt_toolkit.output.Output` instance. (Probably + Vt100_Output or Win32Output.) + + Usage: + + app = Application(...) + app.run() + + # Or + await app.run_async() + """ + + def __init__( + self, + layout: Layout | None = None, + style: BaseStyle | None = None, + include_default_pygments_style: FilterOrBool = True, + style_transformation: StyleTransformation | None = None, + key_bindings: KeyBindingsBase | None = None, + clipboard: Clipboard | None = None, + full_screen: bool = False, + color_depth: (ColorDepth | Callable[[], ColorDepth | None] | None) = None, + mouse_support: FilterOrBool = False, + enable_page_navigation_bindings: None + | (FilterOrBool) = None, # Can be None, True or False. + paste_mode: FilterOrBool = False, + editing_mode: EditingMode = EditingMode.EMACS, + erase_when_done: bool = False, + reverse_vi_search_direction: FilterOrBool = False, + min_redraw_interval: float | int | None = None, + max_render_postpone_time: float | int | None = 0.01, + refresh_interval: float | None = None, + terminal_size_polling_interval: float | None = 0.5, + cursor: AnyCursorShapeConfig = None, + on_reset: ApplicationEventHandler[_AppResult] | None = None, + on_invalidate: ApplicationEventHandler[_AppResult] | None = None, + before_render: ApplicationEventHandler[_AppResult] | None = None, + after_render: ApplicationEventHandler[_AppResult] | None = None, + # I/O. + input: Input | None = None, + output: Output | None = None, + ) -> None: + # If `enable_page_navigation_bindings` is not specified, enable it in + # case of full screen applications only. This can be overridden by the user. + if enable_page_navigation_bindings is None: + enable_page_navigation_bindings = Condition(lambda: self.full_screen) + + paste_mode = to_filter(paste_mode) + mouse_support = to_filter(mouse_support) + reverse_vi_search_direction = to_filter(reverse_vi_search_direction) + enable_page_navigation_bindings = to_filter(enable_page_navigation_bindings) + include_default_pygments_style = to_filter(include_default_pygments_style) + + if layout is None: + layout = create_dummy_layout() + + if style_transformation is None: + style_transformation = DummyStyleTransformation() + + self.style = style + self.style_transformation = style_transformation + + # Key bindings. + self.key_bindings = key_bindings + self._default_bindings = load_key_bindings() + self._page_navigation_bindings = load_page_navigation_bindings() + + self.layout = layout + self.clipboard = clipboard or InMemoryClipboard() + self.full_screen: bool = full_screen + self._color_depth = color_depth + self.mouse_support = mouse_support + + self.paste_mode = paste_mode + self.editing_mode = editing_mode + self.erase_when_done = erase_when_done + self.reverse_vi_search_direction = reverse_vi_search_direction + self.enable_page_navigation_bindings = enable_page_navigation_bindings + self.min_redraw_interval = min_redraw_interval + self.max_render_postpone_time = max_render_postpone_time + self.refresh_interval = refresh_interval + self.terminal_size_polling_interval = terminal_size_polling_interval + + self.cursor = to_cursor_shape_config(cursor) + + # Events. + self.on_invalidate = Event(self, on_invalidate) + self.on_reset = Event(self, on_reset) + self.before_render = Event(self, before_render) + self.after_render = Event(self, after_render) + + # I/O. + session = get_app_session() + self.output = output or session.output + self.input = input or session.input + + # List of 'extra' functions to execute before a Application.run. + self.pre_run_callables: list[Callable[[], None]] = [] + + self._is_running = False + self.future: Future[_AppResult] | None = None + self.loop: AbstractEventLoop | None = None + self._loop_thread: threading.Thread | None = None + self.context: contextvars.Context | None = None + + #: Quoted insert. This flag is set if we go into quoted insert mode. + self.quoted_insert = False + + #: Vi state. (For Vi key bindings.) + self.vi_state = ViState() + self.emacs_state = EmacsState() + + #: When to flush the input (For flushing escape keys.) This is important + #: on terminals that use vt100 input. We can't distinguish the escape + #: key from for instance the left-arrow key, if we don't know what follows + #: after "\x1b". This little timer will consider "\x1b" to be escape if + #: nothing did follow in this time span. + #: This seems to work like the `ttimeoutlen` option in Vim. + self.ttimeoutlen = 0.5 # Seconds. + + #: Like Vim's `timeoutlen` option. This can be `None` or a float. For + #: instance, suppose that we have a key binding AB and a second key + #: binding A. If the uses presses A and then waits, we don't handle + #: this binding yet (unless it was marked 'eager'), because we don't + #: know what will follow. This timeout is the maximum amount of time + #: that we wait until we call the handlers anyway. Pass `None` to + #: disable this timeout. + self.timeoutlen = 1.0 + + #: The `Renderer` instance. + # Make sure that the same stdout is used, when a custom renderer has been passed. + self._merged_style = self._create_merged_style(include_default_pygments_style) + + self.renderer = Renderer( + self._merged_style, + self.output, + full_screen=full_screen, + mouse_support=mouse_support, + cpr_not_supported_callback=self.cpr_not_supported_callback, + ) + + #: Render counter. This one is increased every time the UI is rendered. + #: It can be used as a key for caching certain information during one + #: rendering. + self.render_counter = 0 + + # Invalidate flag. When 'True', a repaint has been scheduled. + self._invalidated = False + self._invalidate_events: list[ + Event[object] + ] = [] # Collection of 'invalidate' Event objects. + self._last_redraw_time = 0.0 # Unix timestamp of last redraw. Used when + # `min_redraw_interval` is given. + + #: The `InputProcessor` instance. + self.key_processor = KeyProcessor(_CombinedRegistry(self)) + + # If `run_in_terminal` was called. This will point to a `Future` what will be + # set at the point when the previous run finishes. + self._running_in_terminal = False + self._running_in_terminal_f: Future[None] | None = None + + # Trigger initialize callback. + self.reset() + + def _create_merged_style(self, include_default_pygments_style: Filter) -> BaseStyle: + """ + Create a `Style` object that merges the default UI style, the default + pygments style, and the custom user style. + """ + dummy_style = DummyStyle() + pygments_style = default_pygments_style() + + @DynamicStyle + def conditional_pygments_style() -> BaseStyle: + if include_default_pygments_style(): + return pygments_style + else: + return dummy_style + + return merge_styles( + [ + default_ui_style(), + conditional_pygments_style, + DynamicStyle(lambda: self.style), + ] + ) + + @property + def color_depth(self) -> ColorDepth: + """ + The active :class:`.ColorDepth`. + + The current value is determined as follows: + + - If a color depth was given explicitly to this application, use that + value. + - Otherwise, fall back to the color depth that is reported by the + :class:`.Output` implementation. If the :class:`.Output` class was + created using `output.defaults.create_output`, then this value is + coming from the $PROMPT_TOOLKIT_COLOR_DEPTH environment variable. + """ + depth = self._color_depth + + if callable(depth): + depth = depth() + + if depth is None: + depth = self.output.get_default_color_depth() + + return depth + + @property + def current_buffer(self) -> Buffer: + """ + The currently focused :class:`~.Buffer`. + + (This returns a dummy :class:`.Buffer` when none of the actual buffers + has the focus. In this case, it's really not practical to check for + `None` values or catch exceptions every time.) + """ + return self.layout.current_buffer or Buffer( + name="dummy-buffer" + ) # Dummy buffer. + + @property + def current_search_state(self) -> SearchState: + """ + Return the current :class:`.SearchState`. (The one for the focused + :class:`.BufferControl`.) + """ + ui_control = self.layout.current_control + if isinstance(ui_control, BufferControl): + return ui_control.search_state + else: + return SearchState() # Dummy search state. (Don't return None!) + + def reset(self) -> None: + """ + Reset everything, for reading the next input. + """ + # Notice that we don't reset the buffers. (This happens just before + # returning, and when we have multiple buffers, we clearly want the + # content in the other buffers to remain unchanged between several + # calls of `run`. (And the same is true for the focus stack.) + + self.exit_style = "" + + self._background_tasks: set[Task[None]] = set() + + self.renderer.reset() + self.key_processor.reset() + self.layout.reset() + self.vi_state.reset() + self.emacs_state.reset() + + # Trigger reset event. + self.on_reset.fire() + + # Make sure that we have a 'focusable' widget focused. + # (The `Layout` class can't determine this.) + layout = self.layout + + if not layout.current_control.is_focusable(): + for w in layout.find_all_windows(): + if w.content.is_focusable(): + layout.current_window = w + break + + def invalidate(self) -> None: + """ + Thread safe way of sending a repaint trigger to the input event loop. + """ + if not self._is_running: + # Don't schedule a redraw if we're not running. + # Otherwise, `get_running_loop()` in `call_soon_threadsafe` can fail. + # See: https://github.com/dbcli/mycli/issues/797 + return + + # `invalidate()` called if we don't have a loop yet (not running?), or + # after the event loop was closed. + if self.loop is None or self.loop.is_closed(): + return + + # Never schedule a second redraw, when a previous one has not yet been + # executed. (This should protect against other threads calling + # 'invalidate' many times, resulting in 100% CPU.) + if self._invalidated: + return + else: + self._invalidated = True + + # Trigger event. + self.loop.call_soon_threadsafe(self.on_invalidate.fire) + + def redraw() -> None: + self._invalidated = False + self._redraw() + + def schedule_redraw() -> None: + call_soon_threadsafe( + redraw, max_postpone_time=self.max_render_postpone_time, loop=self.loop + ) + + if self.min_redraw_interval: + # When a minimum redraw interval is set, wait minimum this amount + # of time between redraws. + diff = time.time() - self._last_redraw_time + if diff < self.min_redraw_interval: + + async def redraw_in_future() -> None: + await sleep(cast(float, self.min_redraw_interval) - diff) + schedule_redraw() + + self.loop.call_soon_threadsafe( + lambda: self.create_background_task(redraw_in_future()) + ) + else: + schedule_redraw() + else: + schedule_redraw() + + @property + def invalidated(self) -> bool: + "True when a redraw operation has been scheduled." + return self._invalidated + + def _redraw(self, render_as_done: bool = False) -> None: + """ + Render the command line again. (Not thread safe!) (From other threads, + or if unsure, use :meth:`.Application.invalidate`.) + + :param render_as_done: make sure to put the cursor after the UI. + """ + + def run_in_context() -> None: + # Only draw when no sub application was started. + if self._is_running and not self._running_in_terminal: + if self.min_redraw_interval: + self._last_redraw_time = time.time() + + # Render + self.render_counter += 1 + self.before_render.fire() + + if render_as_done: + if self.erase_when_done: + self.renderer.erase() + else: + # Draw in 'done' state and reset renderer. + self.renderer.render(self, self.layout, is_done=render_as_done) + else: + self.renderer.render(self, self.layout) + + self.layout.update_parents_relations() + + # Fire render event. + self.after_render.fire() + + self._update_invalidate_events() + + # NOTE: We want to make sure this Application is the active one. The + # invalidate function is often called from a context where this + # application is not the active one. (Like the + # `PromptSession._auto_refresh_context`). + # We copy the context in case the context was already active, to + # prevent RuntimeErrors. (The rendering is not supposed to change + # any context variables.) + if self.context is not None: + self.context.copy().run(run_in_context) + + def _start_auto_refresh_task(self) -> None: + """ + Start a while/true loop in the background for automatic invalidation of + the UI. + """ + if self.refresh_interval is not None and self.refresh_interval != 0: + + async def auto_refresh(refresh_interval: float) -> None: + while True: + await sleep(refresh_interval) + self.invalidate() + + self.create_background_task(auto_refresh(self.refresh_interval)) + + def _update_invalidate_events(self) -> None: + """ + Make sure to attach 'invalidate' handlers to all invalidate events in + the UI. + """ + # Remove all the original event handlers. (Components can be removed + # from the UI.) + for ev in self._invalidate_events: + ev -= self._invalidate_handler + + # Gather all new events. + # (All controls are able to invalidate themselves.) + def gather_events() -> Iterable[Event[object]]: + for c in self.layout.find_all_controls(): + yield from c.get_invalidate_events() + + self._invalidate_events = list(gather_events()) + + for ev in self._invalidate_events: + ev += self._invalidate_handler + + def _invalidate_handler(self, sender: object) -> None: + """ + Handler for invalidate events coming from UIControls. + + (This handles the difference in signature between event handler and + `self.invalidate`. It also needs to be a method -not a nested + function-, so that we can remove it again .) + """ + self.invalidate() + + def _on_resize(self) -> None: + """ + When the window size changes, we erase the current output and request + again the cursor position. When the CPR answer arrives, the output is + drawn again. + """ + # Erase, request position (when cursor is at the start position) + # and redraw again. -- The order is important. + self.renderer.erase(leave_alternate_screen=False) + self._request_absolute_cursor_position() + self._redraw() + + def _pre_run(self, pre_run: Callable[[], None] | None = None) -> None: + """ + Called during `run`. + + `self.future` should be set to the new future at the point where this + is called in order to avoid data races. `pre_run` can be used to set a + `threading.Event` to synchronize with UI termination code, running in + another thread that would call `Application.exit`. (See the progress + bar code for an example.) + """ + if pre_run: + pre_run() + + # Process registered "pre_run_callables" and clear list. + for c in self.pre_run_callables: + c() + del self.pre_run_callables[:] + + async def run_async( + self, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + slow_callback_duration: float = 0.5, + ) -> _AppResult: + """ + Run the prompt_toolkit :class:`~prompt_toolkit.application.Application` + until :meth:`~prompt_toolkit.application.Application.exit` has been + called. Return the value that was passed to + :meth:`~prompt_toolkit.application.Application.exit`. + + This is the main entry point for a prompt_toolkit + :class:`~prompt_toolkit.application.Application` and usually the only + place where the event loop is actually running. + + :param pre_run: Optional callable, which is called right after the + "reset" of the application. + :param set_exception_handler: When set, in case of an exception, go out + of the alternate screen and hide the application, display the + exception, and wait for the user to press ENTER. + :param handle_sigint: Handle SIGINT signal if possible. This will call + the `` key binding when a SIGINT is received. (This only + works in the main thread.) + :param slow_callback_duration: Display warnings if code scheduled in + the asyncio event loop takes more time than this. The asyncio + default of `0.1` is sometimes not sufficient on a slow system, + because exceptionally, the drawing of the app, which happens in the + event loop, can take a bit longer from time to time. + """ + assert not self._is_running, "Application is already running." + + if not in_main_thread() or sys.platform == "win32": + # Handling signals in other threads is not supported. + # Also on Windows, `add_signal_handler(signal.SIGINT, ...)` raises + # `NotImplementedError`. + # See: https://github.com/prompt-toolkit/python-prompt-toolkit/issues/1553 + handle_sigint = False + + async def _run_async(f: asyncio.Future[_AppResult]) -> _AppResult: + context = contextvars.copy_context() + self.context = context + + # Counter for cancelling 'flush' timeouts. Every time when a key is + # pressed, we start a 'flush' timer for flushing our escape key. But + # when any subsequent input is received, a new timer is started and + # the current timer will be ignored. + flush_task: asyncio.Task[None] | None = None + + # Reset. + # (`self.future` needs to be set when `pre_run` is called.) + self.reset() + self._pre_run(pre_run) + + # Feed type ahead input first. + self.key_processor.feed_multiple(get_typeahead(self.input)) + self.key_processor.process_keys() + + def read_from_input() -> None: + nonlocal flush_task + + # Ignore when we aren't running anymore. This callback will + # removed from the loop next time. (It could be that it was + # still in the 'tasks' list of the loop.) + # Except: if we need to process incoming CPRs. + if not self._is_running and not self.renderer.waiting_for_cpr: + return + + # Get keys from the input object. + keys = self.input.read_keys() + + # Feed to key processor. + self.key_processor.feed_multiple(keys) + self.key_processor.process_keys() + + # Quit when the input stream was closed. + if self.input.closed: + if not f.done(): + f.set_exception(EOFError) + else: + # Automatically flush keys. + if flush_task: + flush_task.cancel() + flush_task = self.create_background_task(auto_flush_input()) + + def read_from_input_in_context() -> None: + # Ensure that key bindings callbacks are always executed in the + # current context. This is important when key bindings are + # accessing contextvars. (These callbacks are currently being + # called from a different context. Underneath, + # `loop.add_reader` is used to register the stdin FD.) + # (We copy the context to avoid a `RuntimeError` in case the + # context is already active.) + context.copy().run(read_from_input) + + async def auto_flush_input() -> None: + # Flush input after timeout. + # (Used for flushing the enter key.) + # This sleep can be cancelled, in that case we won't flush yet. + await sleep(self.ttimeoutlen) + flush_input() + + def flush_input() -> None: + if not self.is_done: + # Get keys, and feed to key processor. + keys = self.input.flush_keys() + self.key_processor.feed_multiple(keys) + self.key_processor.process_keys() + + if self.input.closed: + f.set_exception(EOFError) + + # Enter raw mode, attach input and attach WINCH event handler. + with self.input.raw_mode(), self.input.attach( + read_from_input_in_context + ), attach_winch_signal_handler(self._on_resize): + # Draw UI. + self._request_absolute_cursor_position() + self._redraw() + self._start_auto_refresh_task() + + self.create_background_task(self._poll_output_size()) + + # Wait for UI to finish. + try: + result = await f + finally: + # In any case, when the application finishes. + # (Successful, or because of an error.) + try: + self._redraw(render_as_done=True) + finally: + # _redraw has a good chance to fail if it calls widgets + # with bad code. Make sure to reset the renderer + # anyway. + self.renderer.reset() + + # Unset `is_running`, this ensures that possibly + # scheduled draws won't paint during the following + # yield. + self._is_running = False + + # Detach event handlers for invalidate events. + # (Important when a UIControl is embedded in multiple + # applications, like ptterm in pymux. An invalidate + # should not trigger a repaint in terminated + # applications.) + for ev in self._invalidate_events: + ev -= self._invalidate_handler + self._invalidate_events = [] + + # Wait for CPR responses. + if self.output.responds_to_cpr: + await self.renderer.wait_for_cpr_responses() + + # Wait for the run-in-terminals to terminate. + previous_run_in_terminal_f = self._running_in_terminal_f + + if previous_run_in_terminal_f: + await previous_run_in_terminal_f + + # Store unprocessed input as typeahead for next time. + store_typeahead(self.input, self.key_processor.empty_queue()) + + return result + + @contextmanager + def set_loop() -> Iterator[AbstractEventLoop]: + loop = get_running_loop() + self.loop = loop + self._loop_thread = threading.current_thread() + + try: + yield loop + finally: + self.loop = None + self._loop_thread = None + + @contextmanager + def set_is_running() -> Iterator[None]: + self._is_running = True + try: + yield + finally: + self._is_running = False + + @contextmanager + def set_handle_sigint(loop: AbstractEventLoop) -> Iterator[None]: + if handle_sigint: + with _restore_sigint_from_ctypes(): + # save sigint handlers (python and os level) + # See: https://github.com/prompt-toolkit/python-prompt-toolkit/issues/1576 + loop.add_signal_handler( + signal.SIGINT, + lambda *_: loop.call_soon_threadsafe( + self.key_processor.send_sigint + ), + ) + try: + yield + finally: + loop.remove_signal_handler(signal.SIGINT) + else: + yield + + @contextmanager + def set_exception_handler_ctx(loop: AbstractEventLoop) -> Iterator[None]: + if set_exception_handler: + previous_exc_handler = loop.get_exception_handler() + loop.set_exception_handler(self._handle_exception) + try: + yield + finally: + loop.set_exception_handler(previous_exc_handler) + + else: + yield + + @contextmanager + def set_callback_duration(loop: AbstractEventLoop) -> Iterator[None]: + # Set slow_callback_duration. + original_slow_callback_duration = loop.slow_callback_duration + loop.slow_callback_duration = slow_callback_duration + try: + yield + finally: + # Reset slow_callback_duration. + loop.slow_callback_duration = original_slow_callback_duration + + @contextmanager + def create_future( + loop: AbstractEventLoop, + ) -> Iterator[asyncio.Future[_AppResult]]: + f = loop.create_future() + self.future = f # XXX: make sure to set this before calling '_redraw'. + + try: + yield f + finally: + # Also remove the Future again. (This brings the + # application back to its initial state, where it also + # doesn't have a Future.) + self.future = None + + with ExitStack() as stack: + stack.enter_context(set_is_running()) + + # Make sure to set `_invalidated` to `False` to begin with, + # otherwise we're not going to paint anything. This can happen if + # this application had run before on a different event loop, and a + # paint was scheduled using `call_soon_threadsafe` with + # `max_postpone_time`. + self._invalidated = False + + loop = stack.enter_context(set_loop()) + + stack.enter_context(set_handle_sigint(loop)) + stack.enter_context(set_exception_handler_ctx(loop)) + stack.enter_context(set_callback_duration(loop)) + stack.enter_context(set_app(self)) + stack.enter_context(self._enable_breakpointhook()) + + f = stack.enter_context(create_future(loop)) + + try: + return await _run_async(f) + finally: + # Wait for the background tasks to be done. This needs to + # go in the finally! If `_run_async` raises + # `KeyboardInterrupt`, we still want to wait for the + # background tasks. + await self.cancel_and_wait_for_background_tasks() + + # The `ExitStack` above is defined in typeshed in a way that it can + # swallow exceptions. Without next line, mypy would think that there's + # a possibility we don't return here. See: + # https://github.com/python/mypy/issues/7726 + assert False, "unreachable" + + def run( + self, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + in_thread: bool = False, + inputhook: InputHook | None = None, + ) -> _AppResult: + """ + A blocking 'run' call that waits until the UI is finished. + + This will run the application in a fresh asyncio event loop. + + :param pre_run: Optional callable, which is called right after the + "reset" of the application. + :param set_exception_handler: When set, in case of an exception, go out + of the alternate screen and hide the application, display the + exception, and wait for the user to press ENTER. + :param in_thread: When true, run the application in a background + thread, and block the current thread until the application + terminates. This is useful if we need to be sure the application + won't use the current event loop (asyncio does not support nested + event loops). A new event loop will be created in this background + thread, and that loop will also be closed when the background + thread terminates. When this is used, it's especially important to + make sure that all asyncio background tasks are managed through + `get_appp().create_background_task()`, so that unfinished tasks are + properly cancelled before the event loop is closed. This is used + for instance in ptpython. + :param handle_sigint: Handle SIGINT signal. Call the key binding for + `Keys.SIGINT`. (This only works in the main thread.) + """ + if in_thread: + result: _AppResult + exception: BaseException | None = None + + def run_in_thread() -> None: + nonlocal result, exception + try: + result = self.run( + pre_run=pre_run, + set_exception_handler=set_exception_handler, + # Signal handling only works in the main thread. + handle_sigint=False, + inputhook=inputhook, + ) + except BaseException as e: + exception = e + + thread = threading.Thread(target=run_in_thread) + thread.start() + thread.join() + + if exception is not None: + raise exception + return result + + coro = self.run_async( + pre_run=pre_run, + set_exception_handler=set_exception_handler, + handle_sigint=handle_sigint, + ) + + def _called_from_ipython() -> bool: + try: + return ( + sys.modules["IPython"].version_info < (8, 18, 0, "") + and "IPython/terminal/interactiveshell.py" + in sys._getframe(3).f_code.co_filename + ) + except BaseException: + return False + + if inputhook is not None: + # Create new event loop with given input hook and run the app. + # In Python 3.12, we can use asyncio.run(loop_factory=...) + # For now, use `run_until_complete()`. + loop = new_eventloop_with_inputhook(inputhook) + result = loop.run_until_complete(coro) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + return result + + elif _called_from_ipython(): + # workaround to make input hooks work for IPython until + # https://github.com/ipython/ipython/pull/14241 is merged. + # IPython was setting the input hook by installing an event loop + # previously. + try: + # See whether a loop was installed already. If so, use that. + # That's required for the input hooks to work, they are + # installed using `set_event_loop`. + loop = asyncio.get_event_loop() + except RuntimeError: + # No loop installed. Run like usual. + return asyncio.run(coro) + else: + # Use existing loop. + return loop.run_until_complete(coro) + + else: + # No loop installed. Run like usual. + return asyncio.run(coro) + + def _handle_exception( + self, loop: AbstractEventLoop, context: dict[str, Any] + ) -> None: + """ + Handler for event loop exceptions. + This will print the exception, using run_in_terminal. + """ + # For Python 2: we have to get traceback at this point, because + # we're still in the 'except:' block of the event loop where the + # traceback is still available. Moving this code in the + # 'print_exception' coroutine will loose the exception. + tb = get_traceback_from_context(context) + formatted_tb = "".join(format_tb(tb)) + + async def in_term() -> None: + async with in_terminal(): + # Print output. Similar to 'loop.default_exception_handler', + # but don't use logger. (This works better on Python 2.) + print("\nUnhandled exception in event loop:") + print(formatted_tb) + print("Exception {}".format(context.get("exception"))) + + await _do_wait_for_enter("Press ENTER to continue...") + + ensure_future(in_term()) + + @contextmanager + def _enable_breakpointhook(self) -> Generator[None, None, None]: + """ + Install our custom breakpointhook for the duration of this context + manager. (We will only install the hook if no other custom hook was + set.) + """ + if sys.breakpointhook == sys.__breakpointhook__: + sys.breakpointhook = self._breakpointhook + + try: + yield + finally: + sys.breakpointhook = sys.__breakpointhook__ + else: + yield + + def _breakpointhook(self, *a: object, **kw: object) -> None: + """ + Breakpointhook which uses PDB, but ensures that the application is + hidden and input echoing is restored during each debugger dispatch. + + This can be called from any thread. In any case, the application's + event loop will be blocked while the PDB input is displayed. The event + will continue after leaving the debugger. + """ + app = self + # Inline import on purpose. We don't want to import pdb, if not needed. + import pdb + from types import FrameType + + TraceDispatch = Callable[[FrameType, str, Any], Any] + + @contextmanager + def hide_app_from_eventloop_thread() -> Generator[None, None, None]: + """Stop application if `__breakpointhook__` is called from within + the App's event loop.""" + # Hide application. + app.renderer.erase() + + # Detach input and dispatch to debugger. + with app.input.detach(): + with app.input.cooked_mode(): + yield + + # Note: we don't render the application again here, because + # there's a good chance that there's a breakpoint on the next + # line. This paint/erase cycle would move the PDB prompt back + # to the middle of the screen. + + @contextmanager + def hide_app_from_other_thread() -> Generator[None, None, None]: + """Stop application if `__breakpointhook__` is called from a + thread other than the App's event loop.""" + ready = threading.Event() + done = threading.Event() + + async def in_loop() -> None: + # from .run_in_terminal import in_terminal + # async with in_terminal(): + # ready.set() + # await asyncio.get_running_loop().run_in_executor(None, done.wait) + # return + + # Hide application. + app.renderer.erase() + + # Detach input and dispatch to debugger. + with app.input.detach(): + with app.input.cooked_mode(): + ready.set() + # Here we block the App's event loop thread until the + # debugger resumes. We could have used `with + # run_in_terminal.in_terminal():` like the commented + # code above, but it seems to work better if we + # completely stop the main event loop while debugging. + done.wait() + + self.create_background_task(in_loop()) + ready.wait() + try: + yield + finally: + done.set() + + class CustomPdb(pdb.Pdb): + def trace_dispatch( + self, frame: FrameType, event: str, arg: Any + ) -> TraceDispatch: + if app._loop_thread is None: + return super().trace_dispatch(frame, event, arg) + + if app._loop_thread == threading.current_thread(): + with hide_app_from_eventloop_thread(): + return super().trace_dispatch(frame, event, arg) + + with hide_app_from_other_thread(): + return super().trace_dispatch(frame, event, arg) + + frame = sys._getframe().f_back + CustomPdb(stdout=sys.__stdout__).set_trace(frame) + + def create_background_task( + self, coroutine: Coroutine[Any, Any, None] + ) -> asyncio.Task[None]: + """ + Start a background task (coroutine) for the running application. When + the `Application` terminates, unfinished background tasks will be + cancelled. + + Given that we still support Python versions before 3.11, we can't use + task groups (and exception groups), because of that, these background + tasks are not allowed to raise exceptions. If they do, we'll call the + default exception handler from the event loop. + + If at some point, we have Python 3.11 as the minimum supported Python + version, then we can use a `TaskGroup` (with the lifetime of + `Application.run_async()`, and run run the background tasks in there. + + This is not threadsafe. + """ + loop = self.loop or get_running_loop() + task: asyncio.Task[None] = loop.create_task(coroutine) + self._background_tasks.add(task) + + task.add_done_callback(self._on_background_task_done) + return task + + def _on_background_task_done(self, task: asyncio.Task[None]) -> None: + """ + Called when a background task completes. Remove it from + `_background_tasks`, and handle exceptions if any. + """ + self._background_tasks.discard(task) + + if task.cancelled(): + return + + exc = task.exception() + if exc is not None: + get_running_loop().call_exception_handler( + { + "message": f"prompt_toolkit.Application background task {task!r} " + "raised an unexpected exception.", + "exception": exc, + "task": task, + } + ) + + async def cancel_and_wait_for_background_tasks(self) -> None: + """ + Cancel all background tasks, and wait for the cancellation to complete. + If any of the background tasks raised an exception, this will also + propagate the exception. + + (If we had nurseries like Trio, this would be the `__aexit__` of a + nursery.) + """ + for task in self._background_tasks: + task.cancel() + + # Wait until the cancellation of the background tasks completes. + # `asyncio.wait()` does not propagate exceptions raised within any of + # these tasks, which is what we want. Otherwise, we can't distinguish + # between a `CancelledError` raised in this task because it got + # cancelled, and a `CancelledError` raised on this `await` checkpoint, + # because *we* got cancelled during the teardown of the application. + # (If we get cancelled here, then it's important to not suppress the + # `CancelledError`, and have it propagate.) + # NOTE: Currently, if we get cancelled at this point then we can't wait + # for the cancellation to complete (in the future, we should be + # using anyio or Python's 3.11 TaskGroup.) + # Also, if we had exception groups, we could propagate an + # `ExceptionGroup` if something went wrong here. Right now, we + # don't propagate exceptions, but have them printed in + # `_on_background_task_done`. + if len(self._background_tasks) > 0: + await asyncio.wait( + self._background_tasks, timeout=None, return_when=asyncio.ALL_COMPLETED + ) + + async def _poll_output_size(self) -> None: + """ + Coroutine for polling the terminal dimensions. + + Useful for situations where `attach_winch_signal_handler` is not sufficient: + - If we are not running in the main thread. + - On Windows. + """ + size: Size | None = None + interval = self.terminal_size_polling_interval + + if interval is None: + return + + while True: + await asyncio.sleep(interval) + new_size = self.output.get_size() + + if size is not None and new_size != size: + self._on_resize() + size = new_size + + def cpr_not_supported_callback(self) -> None: + """ + Called when we don't receive the cursor position response in time. + """ + if not self.output.responds_to_cpr: + return # We know about this already. + + def in_terminal() -> None: + self.output.write( + "WARNING: your terminal doesn't support cursor position requests (CPR).\r\n" + ) + self.output.flush() + + run_in_terminal(in_terminal) + + @overload + def exit(self) -> None: + "Exit without arguments." + + @overload + def exit(self, *, result: _AppResult, style: str = "") -> None: + "Exit with `_AppResult`." + + @overload + def exit( + self, *, exception: BaseException | type[BaseException], style: str = "" + ) -> None: + "Exit with exception." + + def exit( + self, + result: _AppResult | None = None, + exception: BaseException | type[BaseException] | None = None, + style: str = "", + ) -> None: + """ + Exit application. + + .. note:: + + If `Application.exit` is called before `Application.run()` is + called, then the `Application` won't exit (because the + `Application.future` doesn't correspond to the current run). Use a + `pre_run` hook and an event to synchronize the closing if there's a + chance this can happen. + + :param result: Set this result for the application. + :param exception: Set this exception as the result for an application. For + a prompt, this is often `EOFError` or `KeyboardInterrupt`. + :param style: Apply this style on the whole content when quitting, + often this is 'class:exiting' for a prompt. (Used when + `erase_when_done` is not set.) + """ + assert result is None or exception is None + + if self.future is None: + raise Exception("Application is not running. Application.exit() failed.") + + if self.future.done(): + raise Exception("Return value already set. Application.exit() failed.") + + self.exit_style = style + + if exception is not None: + self.future.set_exception(exception) + else: + self.future.set_result(cast(_AppResult, result)) + + def _request_absolute_cursor_position(self) -> None: + """ + Send CPR request. + """ + # Note: only do this if the input queue is not empty, and a return + # value has not been set. Otherwise, we won't be able to read the + # response anyway. + if not self.key_processor.input_queue and not self.is_done: + self.renderer.request_absolute_cursor_position() + + async def run_system_command( + self, + command: str, + wait_for_enter: bool = True, + display_before_text: AnyFormattedText = "", + wait_text: str = "Press ENTER to continue...", + ) -> None: + """ + Run system command (While hiding the prompt. When finished, all the + output will scroll above the prompt.) + + :param command: Shell command to be executed. + :param wait_for_enter: FWait for the user to press enter, when the + command is finished. + :param display_before_text: If given, text to be displayed before the + command executes. + :return: A `Future` object. + """ + async with in_terminal(): + # Try to use the same input/output file descriptors as the one, + # used to run this application. + try: + input_fd = self.input.fileno() + except AttributeError: + input_fd = sys.stdin.fileno() + try: + output_fd = self.output.fileno() + except AttributeError: + output_fd = sys.stdout.fileno() + + # Run sub process. + def run_command() -> None: + self.print_text(display_before_text) + p = Popen(command, shell=True, stdin=input_fd, stdout=output_fd) + p.wait() + + await run_in_executor_with_context(run_command) + + # Wait for the user to press enter. + if wait_for_enter: + await _do_wait_for_enter(wait_text) + + def suspend_to_background(self, suspend_group: bool = True) -> None: + """ + (Not thread safe -- to be called from inside the key bindings.) + Suspend process. + + :param suspend_group: When true, suspend the whole process group. + (This is the default, and probably what you want.) + """ + # Only suspend when the operating system supports it. + # (Not on Windows.) + if _SIGTSTP is not None: + + def run() -> None: + signal = cast(int, _SIGTSTP) + # Send `SIGTSTP` to own process. + # This will cause it to suspend. + + # Usually we want the whole process group to be suspended. This + # handles the case when input is piped from another process. + if suspend_group: + os.kill(0, signal) + else: + os.kill(os.getpid(), signal) + + run_in_terminal(run) + + def print_text( + self, text: AnyFormattedText, style: BaseStyle | None = None + ) -> None: + """ + Print a list of (style_str, text) tuples to the output. + (When the UI is running, this method has to be called through + `run_in_terminal`, otherwise it will destroy the UI.) + + :param text: List of ``(style_str, text)`` tuples. + :param style: Style class to use. Defaults to the active style in the CLI. + """ + print_formatted_text( + output=self.output, + formatted_text=text, + style=style or self._merged_style, + color_depth=self.color_depth, + style_transformation=self.style_transformation, + ) + + @property + def is_running(self) -> bool: + "`True` when the application is currently active/running." + return self._is_running + + @property + def is_done(self) -> bool: + if self.future: + return self.future.done() + return False + + def get_used_style_strings(self) -> list[str]: + """ + Return a list of used style strings. This is helpful for debugging, and + for writing a new `Style`. + """ + attrs_for_style = self.renderer._attrs_for_style + + if attrs_for_style: + return sorted( + re.sub(r"\s+", " ", style_str).strip() + for style_str in attrs_for_style.keys() + ) + + return [] + + +class _CombinedRegistry(KeyBindingsBase): + """ + The `KeyBindings` of key bindings for a `Application`. + This merges the global key bindings with the one of the current user + control. + """ + + def __init__(self, app: Application[_AppResult]) -> None: + self.app = app + self._cache: SimpleCache[ + tuple[Window, frozenset[UIControl]], KeyBindingsBase + ] = SimpleCache() + + @property + def _version(self) -> Hashable: + """Not needed - this object is not going to be wrapped in another + KeyBindings object.""" + raise NotImplementedError + + @property + def bindings(self) -> list[Binding]: + """Not needed - this object is not going to be wrapped in another + KeyBindings object.""" + raise NotImplementedError + + def _create_key_bindings( + self, current_window: Window, other_controls: list[UIControl] + ) -> KeyBindingsBase: + """ + Create a `KeyBindings` object that merges the `KeyBindings` from the + `UIControl` with all the parent controls and the global key bindings. + """ + key_bindings = [] + collected_containers = set() + + # Collect key bindings from currently focused control and all parent + # controls. Don't include key bindings of container parent controls. + container: Container = current_window + while True: + collected_containers.add(container) + kb = container.get_key_bindings() + if kb is not None: + key_bindings.append(kb) + + if container.is_modal(): + break + + parent = self.app.layout.get_parent(container) + if parent is None: + break + else: + container = parent + + # Include global bindings (starting at the top-model container). + for c in walk(container): + if c not in collected_containers: + kb = c.get_key_bindings() + if kb is not None: + key_bindings.append(GlobalOnlyKeyBindings(kb)) + + # Add App key bindings + if self.app.key_bindings: + key_bindings.append(self.app.key_bindings) + + # Add mouse bindings. + key_bindings.append( + ConditionalKeyBindings( + self.app._page_navigation_bindings, + self.app.enable_page_navigation_bindings, + ) + ) + key_bindings.append(self.app._default_bindings) + + # Reverse this list. The current control's key bindings should come + # last. They need priority. + key_bindings = key_bindings[::-1] + + return merge_key_bindings(key_bindings) + + @property + def _key_bindings(self) -> KeyBindingsBase: + current_window = self.app.layout.current_window + other_controls = list(self.app.layout.find_all_controls()) + key = current_window, frozenset(other_controls) + + return self._cache.get( + key, lambda: self._create_key_bindings(current_window, other_controls) + ) + + def get_bindings_for_keys(self, keys: KeysTuple) -> list[Binding]: + return self._key_bindings.get_bindings_for_keys(keys) + + def get_bindings_starting_with_keys(self, keys: KeysTuple) -> list[Binding]: + return self._key_bindings.get_bindings_starting_with_keys(keys) + + +async def _do_wait_for_enter(wait_text: AnyFormattedText) -> None: + """ + Create a sub application to wait for the enter key press. + This has two advantages over using 'input'/'raw_input': + - This will share the same input/output I/O. + - This doesn't block the event loop. + """ + from prompt_toolkit.shortcuts import PromptSession + + key_bindings = KeyBindings() + + @key_bindings.add("enter") + def _ok(event: E) -> None: + event.app.exit() + + @key_bindings.add(Keys.Any) + def _ignore(event: E) -> None: + "Disallow typing." + pass + + session: PromptSession[None] = PromptSession( + message=wait_text, key_bindings=key_bindings + ) + try: + await session.app.run_async() + except KeyboardInterrupt: + pass # Control-c pressed. Don't propagate this error. + + +@contextmanager +def attach_winch_signal_handler( + handler: Callable[[], None], +) -> Generator[None, None, None]: + """ + Attach the given callback as a WINCH signal handler within the context + manager. Restore the original signal handler when done. + + The `Application.run` method will register SIGWINCH, so that it will + properly repaint when the terminal window resizes. However, using + `run_in_terminal`, we can temporarily send an application to the + background, and run an other app in between, which will then overwrite the + SIGWINCH. This is why it's important to restore the handler when the app + terminates. + """ + # The tricky part here is that signals are registered in the Unix event + # loop with a wakeup fd, but another application could have registered + # signals using signal.signal directly. For now, the implementation is + # hard-coded for the `asyncio.unix_events._UnixSelectorEventLoop`. + + # No WINCH? Then don't do anything. + sigwinch = getattr(signal, "SIGWINCH", None) + if sigwinch is None or not in_main_thread(): + yield + return + + # Keep track of the previous handler. + # (Only UnixSelectorEventloop has `_signal_handlers`.) + loop = get_running_loop() + previous_winch_handler = getattr(loop, "_signal_handlers", {}).get(sigwinch) + + try: + loop.add_signal_handler(sigwinch, handler) + yield + finally: + # Restore the previous signal handler. + loop.remove_signal_handler(sigwinch) + if previous_winch_handler is not None: + loop.add_signal_handler( + sigwinch, + previous_winch_handler._callback, + *previous_winch_handler._args, + ) + + +@contextmanager +def _restore_sigint_from_ctypes() -> Generator[None, None, None]: + # The following functions are part of the stable ABI since python 3.2 + # See: https://docs.python.org/3/c-api/sys.html#c.PyOS_getsig + # Inline import: these are not available on Pypy. + try: + from ctypes import c_int, c_void_p, pythonapi + except ImportError: + have_ctypes_signal = False + else: + # GraalPy has the functions, but they don't work + have_ctypes_signal = sys.implementation.name != "graalpy" + + if have_ctypes_signal: + # PyOS_sighandler_t PyOS_getsig(int i) + pythonapi.PyOS_getsig.restype = c_void_p + pythonapi.PyOS_getsig.argtypes = (c_int,) + + # PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) + pythonapi.PyOS_setsig.restype = c_void_p + pythonapi.PyOS_setsig.argtypes = ( + c_int, + c_void_p, + ) + + sigint = signal.getsignal(signal.SIGINT) + if have_ctypes_signal: + sigint_os = pythonapi.PyOS_getsig(signal.SIGINT) + + try: + yield + finally: + if sigint is not None: + signal.signal(signal.SIGINT, sigint) + if have_ctypes_signal: + pythonapi.PyOS_setsig(signal.SIGINT, sigint_os) diff --git a/lib/prompt_toolkit/application/current.py b/lib/prompt_toolkit/application/current.py new file mode 100644 index 0000000..f7032fe --- /dev/null +++ b/lib/prompt_toolkit/application/current.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar +from typing import TYPE_CHECKING, Any, Generator + +if TYPE_CHECKING: + from prompt_toolkit.input.base import Input + from prompt_toolkit.output.base import Output + + from .application import Application + +__all__ = [ + "AppSession", + "get_app_session", + "get_app", + "get_app_or_none", + "set_app", + "create_app_session", + "create_app_session_from_tty", +] + + +class AppSession: + """ + An AppSession is an interactive session, usually connected to one terminal. + Within one such session, interaction with many applications can happen, one + after the other. + + The input/output device is not supposed to change during one session. + + Warning: Always use the `create_app_session` function to create an + instance, so that it gets activated correctly. + + :param input: Use this as a default input for all applications + running in this session, unless an input is passed to the `Application` + explicitly. + :param output: Use this as a default output. + """ + + def __init__( + self, input: Input | None = None, output: Output | None = None + ) -> None: + self._input = input + self._output = output + + # The application will be set dynamically by the `set_app` context + # manager. This is called in the application itself. + self.app: Application[Any] | None = None + + def __repr__(self) -> str: + return f"AppSession(app={self.app!r})" + + @property + def input(self) -> Input: + if self._input is None: + from prompt_toolkit.input.defaults import create_input + + self._input = create_input() + return self._input + + @property + def output(self) -> Output: + if self._output is None: + from prompt_toolkit.output.defaults import create_output + + self._output = create_output() + return self._output + + +_current_app_session: ContextVar[AppSession] = ContextVar( + "_current_app_session", default=AppSession() +) + + +def get_app_session() -> AppSession: + return _current_app_session.get() + + +def get_app() -> Application[Any]: + """ + Get the current active (running) Application. + An :class:`.Application` is active during the + :meth:`.Application.run_async` call. + + We assume that there can only be one :class:`.Application` active at the + same time. There is only one terminal window, with only one stdin and + stdout. This makes the code significantly easier than passing around the + :class:`.Application` everywhere. + + If no :class:`.Application` is running, then return by default a + :class:`.DummyApplication`. For practical reasons, we prefer to not raise + an exception. This way, we don't have to check all over the place whether + an actual `Application` was returned. + + (For applications like pymux where we can have more than one `Application`, + we'll use a work-around to handle that.) + """ + session = _current_app_session.get() + if session.app is not None: + return session.app + + from .dummy import DummyApplication + + return DummyApplication() + + +def get_app_or_none() -> Application[Any] | None: + """ + Get the current active (running) Application, or return `None` if no + application is running. + """ + session = _current_app_session.get() + return session.app + + +@contextmanager +def set_app(app: Application[Any]) -> Generator[None, None, None]: + """ + Context manager that sets the given :class:`.Application` active in an + `AppSession`. + + This should only be called by the `Application` itself. + The application will automatically be active while its running. If you want + the application to be active in other threads/coroutines, where that's not + the case, use `contextvars.copy_context()`, or use `Application.context` to + run it in the appropriate context. + """ + session = _current_app_session.get() + + previous_app = session.app + session.app = app + try: + yield + finally: + session.app = previous_app + + +@contextmanager +def create_app_session( + input: Input | None = None, output: Output | None = None +) -> Generator[AppSession, None, None]: + """ + Create a separate AppSession. + + This is useful if there can be multiple individual ``AppSession``'s going + on. Like in the case of a Telnet/SSH server. + """ + # If no input/output is specified, fall back to the current input/output, + # if there was one that was set/created for the current session. + # (Note that we check `_input`/`_output` and not `input`/`output`. This is + # because we don't want to accidentally create a new input/output objects + # here and store it in the "parent" `AppSession`. Especially, when + # combining pytest's `capsys` fixture and `create_app_session`, sys.stdin + # and sys.stderr are patched for every test, so we don't want to leak + # those outputs object across `AppSession`s.) + if input is None: + input = get_app_session()._input + if output is None: + output = get_app_session()._output + + # Create new `AppSession` and activate. + session = AppSession(input=input, output=output) + + token = _current_app_session.set(session) + try: + yield session + finally: + _current_app_session.reset(token) + + +@contextmanager +def create_app_session_from_tty() -> Generator[AppSession, None, None]: + """ + Create `AppSession` that always prefers the TTY input/output. + + Even if `sys.stdin` and `sys.stdout` are connected to input/output pipes, + this will still use the terminal for interaction (because `sys.stderr` is + still connected to the terminal). + + Usage:: + + from prompt_toolkit.shortcuts import prompt + + with create_app_session_from_tty(): + prompt('>') + """ + from prompt_toolkit.input.defaults import create_input + from prompt_toolkit.output.defaults import create_output + + input = create_input(always_prefer_tty=True) + output = create_output(always_prefer_tty=True) + + with create_app_session(input=input, output=output) as app_session: + yield app_session diff --git a/lib/prompt_toolkit/application/dummy.py b/lib/prompt_toolkit/application/dummy.py new file mode 100644 index 0000000..43819e1 --- /dev/null +++ b/lib/prompt_toolkit/application/dummy.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Callable + +from prompt_toolkit.eventloop import InputHook +from prompt_toolkit.formatted_text import AnyFormattedText +from prompt_toolkit.input import DummyInput +from prompt_toolkit.output import DummyOutput + +from .application import Application + +__all__ = [ + "DummyApplication", +] + + +class DummyApplication(Application[None]): + """ + When no :class:`.Application` is running, + :func:`.get_app` will run an instance of this :class:`.DummyApplication` instead. + """ + + def __init__(self) -> None: + super().__init__(output=DummyOutput(), input=DummyInput()) + + def run( + self, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + in_thread: bool = False, + inputhook: InputHook | None = None, + ) -> None: + raise NotImplementedError("A DummyApplication is not supposed to run.") + + async def run_async( + self, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + slow_callback_duration: float = 0.5, + ) -> None: + raise NotImplementedError("A DummyApplication is not supposed to run.") + + async def run_system_command( + self, + command: str, + wait_for_enter: bool = True, + display_before_text: AnyFormattedText = "", + wait_text: str = "", + ) -> None: + raise NotImplementedError + + def suspend_to_background(self, suspend_group: bool = True) -> None: + raise NotImplementedError diff --git a/lib/prompt_toolkit/application/run_in_terminal.py b/lib/prompt_toolkit/application/run_in_terminal.py new file mode 100644 index 0000000..1f5e18e --- /dev/null +++ b/lib/prompt_toolkit/application/run_in_terminal.py @@ -0,0 +1,117 @@ +""" +Tools for running functions on the terminal above the current application or prompt. +""" + +from __future__ import annotations + +from asyncio import Future, ensure_future +from contextlib import asynccontextmanager +from typing import AsyncGenerator, Awaitable, Callable, TypeVar + +from prompt_toolkit.eventloop import run_in_executor_with_context + +from .current import get_app_or_none + +__all__ = [ + "run_in_terminal", + "in_terminal", +] + +_T = TypeVar("_T") + + +def run_in_terminal( + func: Callable[[], _T], render_cli_done: bool = False, in_executor: bool = False +) -> Awaitable[_T]: + """ + Run function on the terminal above the current application or prompt. + + What this does is first hiding the prompt, then running this callable + (which can safely output to the terminal), and then again rendering the + prompt which causes the output of this function to scroll above the + prompt. + + ``func`` is supposed to be a synchronous function. If you need an + asynchronous version of this function, use the ``in_terminal`` context + manager directly. + + :param func: The callable to execute. + :param render_cli_done: When True, render the interface in the + 'Done' state first, then execute the function. If False, + erase the interface first. + :param in_executor: When True, run in executor. (Use this for long + blocking functions, when you don't want to block the event loop.) + + :returns: A `Future`. + """ + + async def run() -> _T: + async with in_terminal(render_cli_done=render_cli_done): + if in_executor: + return await run_in_executor_with_context(func) + else: + return func() + + return ensure_future(run()) + + +@asynccontextmanager +async def in_terminal(render_cli_done: bool = False) -> AsyncGenerator[None, None]: + """ + Asynchronous context manager that suspends the current application and runs + the body in the terminal. + + .. code:: + + async def f(): + async with in_terminal(): + call_some_function() + await call_some_async_function() + """ + app = get_app_or_none() + if app is None or not app._is_running: + yield + return + + # When a previous `run_in_terminal` call was in progress. Wait for that + # to finish, before starting this one. Chain to previous call. + previous_run_in_terminal_f = app._running_in_terminal_f + new_run_in_terminal_f: Future[None] = Future() + app._running_in_terminal_f = new_run_in_terminal_f + + # Wait for the previous `run_in_terminal` to finish. + if previous_run_in_terminal_f is not None: + await previous_run_in_terminal_f + + # Wait for all CPRs to arrive. We don't want to detach the input until + # all cursor position responses have been arrived. Otherwise, the tty + # will echo its input and can show stuff like ^[[39;1R. + if app.output.responds_to_cpr: + await app.renderer.wait_for_cpr_responses() + + # Draw interface in 'done' state, or erase. + if render_cli_done: + app._redraw(render_as_done=True) + else: + app.renderer.erase() + + # Disable rendering. + app._running_in_terminal = True + + # Detach input. + try: + with app.input.detach(): + with app.input.cooked_mode(): + yield + finally: + # Redraw interface again. + try: + app._running_in_terminal = False + app.renderer.reset() + app._request_absolute_cursor_position() + app._redraw() + finally: + # (Check for `.done()`, because it can be that this future was + # cancelled.) + if not new_run_in_terminal_f.done(): + new_run_in_terminal_f.set_result(None) diff --git a/lib/prompt_toolkit/auto_suggest.py b/lib/prompt_toolkit/auto_suggest.py new file mode 100644 index 0000000..73213ba --- /dev/null +++ b/lib/prompt_toolkit/auto_suggest.py @@ -0,0 +1,177 @@ +""" +`Fish-style `_ like auto-suggestion. + +While a user types input in a certain buffer, suggestions are generated +(asynchronously.) Usually, they are displayed after the input. When the cursor +presses the right arrow and the cursor is at the end of the input, the +suggestion will be inserted. + +If you want the auto suggestions to be asynchronous (in a background thread), +because they take too much time, and could potentially block the event loop, +then wrap the :class:`.AutoSuggest` instance into a +:class:`.ThreadedAutoSuggest`. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Callable + +from prompt_toolkit.eventloop import run_in_executor_with_context + +from .document import Document +from .filters import Filter, to_filter + +if TYPE_CHECKING: + from .buffer import Buffer + +__all__ = [ + "Suggestion", + "AutoSuggest", + "ThreadedAutoSuggest", + "DummyAutoSuggest", + "AutoSuggestFromHistory", + "ConditionalAutoSuggest", + "DynamicAutoSuggest", +] + + +class Suggestion: + """ + Suggestion returned by an auto-suggest algorithm. + + :param text: The suggestion text. + """ + + def __init__(self, text: str) -> None: + self.text = text + + def __repr__(self) -> str: + return f"Suggestion({self.text})" + + +class AutoSuggest(metaclass=ABCMeta): + """ + Base class for auto suggestion implementations. + """ + + @abstractmethod + def get_suggestion(self, buffer: Buffer, document: Document) -> Suggestion | None: + """ + Return `None` or a :class:`.Suggestion` instance. + + We receive both :class:`~prompt_toolkit.buffer.Buffer` and + :class:`~prompt_toolkit.document.Document`. The reason is that auto + suggestions are retrieved asynchronously. (Like completions.) The + buffer text could be changed in the meantime, but ``document`` contains + the buffer document like it was at the start of the auto suggestion + call. So, from here, don't access ``buffer.text``, but use + ``document.text`` instead. + + :param buffer: The :class:`~prompt_toolkit.buffer.Buffer` instance. + :param document: The :class:`~prompt_toolkit.document.Document` instance. + """ + + async def get_suggestion_async( + self, buff: Buffer, document: Document + ) -> Suggestion | None: + """ + Return a :class:`.Future` which is set when the suggestions are ready. + This function can be overloaded in order to provide an asynchronous + implementation. + """ + return self.get_suggestion(buff, document) + + +class ThreadedAutoSuggest(AutoSuggest): + """ + Wrapper that runs auto suggestions in a thread. + (Use this to prevent the user interface from becoming unresponsive if the + generation of suggestions takes too much time.) + """ + + def __init__(self, auto_suggest: AutoSuggest) -> None: + self.auto_suggest = auto_suggest + + def get_suggestion(self, buff: Buffer, document: Document) -> Suggestion | None: + return self.auto_suggest.get_suggestion(buff, document) + + async def get_suggestion_async( + self, buff: Buffer, document: Document + ) -> Suggestion | None: + """ + Run the `get_suggestion` function in a thread. + """ + + def run_get_suggestion_thread() -> Suggestion | None: + return self.get_suggestion(buff, document) + + return await run_in_executor_with_context(run_get_suggestion_thread) + + +class DummyAutoSuggest(AutoSuggest): + """ + AutoSuggest class that doesn't return any suggestion. + """ + + def get_suggestion(self, buffer: Buffer, document: Document) -> Suggestion | None: + return None # No suggestion + + +class AutoSuggestFromHistory(AutoSuggest): + """ + Give suggestions based on the lines in the history. + """ + + def get_suggestion(self, buffer: Buffer, document: Document) -> Suggestion | None: + history = buffer.history + + # Consider only the last line for the suggestion. + text = document.text.rsplit("\n", 1)[-1] + + # Only create a suggestion when this is not an empty line. + if text.strip(): + # Find first matching line in history. + for string in reversed(list(history.get_strings())): + for line in reversed(string.splitlines()): + if line.startswith(text): + return Suggestion(line[len(text) :]) + + return None + + +class ConditionalAutoSuggest(AutoSuggest): + """ + Auto suggest that can be turned on and of according to a certain condition. + """ + + def __init__(self, auto_suggest: AutoSuggest, filter: bool | Filter) -> None: + self.auto_suggest = auto_suggest + self.filter = to_filter(filter) + + def get_suggestion(self, buffer: Buffer, document: Document) -> Suggestion | None: + if self.filter(): + return self.auto_suggest.get_suggestion(buffer, document) + + return None + + +class DynamicAutoSuggest(AutoSuggest): + """ + Validator class that can dynamically returns any Validator. + + :param get_validator: Callable that returns a :class:`.Validator` instance. + """ + + def __init__(self, get_auto_suggest: Callable[[], AutoSuggest | None]) -> None: + self.get_auto_suggest = get_auto_suggest + + def get_suggestion(self, buff: Buffer, document: Document) -> Suggestion | None: + auto_suggest = self.get_auto_suggest() or DummyAutoSuggest() + return auto_suggest.get_suggestion(buff, document) + + async def get_suggestion_async( + self, buff: Buffer, document: Document + ) -> Suggestion | None: + auto_suggest = self.get_auto_suggest() or DummyAutoSuggest() + return await auto_suggest.get_suggestion_async(buff, document) diff --git a/lib/prompt_toolkit/buffer.py b/lib/prompt_toolkit/buffer.py new file mode 100644 index 0000000..f5847d4 --- /dev/null +++ b/lib/prompt_toolkit/buffer.py @@ -0,0 +1,2029 @@ +""" +Data structures for the Buffer. +It holds the text, cursor position, history, etc... +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import re +import shlex +import shutil +import subprocess +import tempfile +from collections import deque +from enum import Enum +from functools import wraps +from typing import Any, Callable, Coroutine, Iterable, TypeVar, cast + +from .application.current import get_app +from .application.run_in_terminal import run_in_terminal +from .auto_suggest import AutoSuggest, Suggestion +from .cache import FastDictCache +from .clipboard import ClipboardData +from .completion import ( + CompleteEvent, + Completer, + Completion, + DummyCompleter, + get_common_complete_suffix, +) +from .document import Document +from .eventloop import aclosing +from .filters import FilterOrBool, to_filter +from .history import History, InMemoryHistory +from .search import SearchDirection, SearchState +from .selection import PasteMode, SelectionState, SelectionType +from .utils import Event, to_str +from .validation import ValidationError, Validator + +__all__ = [ + "EditReadOnlyBuffer", + "Buffer", + "CompletionState", + "indent", + "unindent", + "reshape_text", +] + +logger = logging.getLogger(__name__) + + +class EditReadOnlyBuffer(Exception): + "Attempt editing of read-only :class:`.Buffer`." + + +class ValidationState(Enum): + "The validation state of a buffer. This is set after the validation." + + VALID = "VALID" + INVALID = "INVALID" + UNKNOWN = "UNKNOWN" + + +class CompletionState: + """ + Immutable class that contains a completion state. + """ + + def __init__( + self, + original_document: Document, + completions: list[Completion] | None = None, + complete_index: int | None = None, + ) -> None: + #: Document as it was when the completion started. + self.original_document = original_document + + #: List of all the current Completion instances which are possible at + #: this point. + self.completions = completions or [] + + #: Position in the `completions` array. + #: This can be `None` to indicate "no completion", the original text. + self.complete_index = complete_index # Position in the `_completions` array. + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.original_document!r}, <{len(self.completions)!r}> completions, index={self.complete_index!r})" + + def go_to_index(self, index: int | None) -> None: + """ + Create a new :class:`.CompletionState` object with the new index. + + When `index` is `None` deselect the completion. + """ + if self.completions: + assert index is None or 0 <= index < len(self.completions) + self.complete_index = index + + def new_text_and_position(self) -> tuple[str, int]: + """ + Return (new_text, new_cursor_position) for this completion. + """ + if self.complete_index is None: + return self.original_document.text, self.original_document.cursor_position + else: + original_text_before_cursor = self.original_document.text_before_cursor + original_text_after_cursor = self.original_document.text_after_cursor + + c = self.completions[self.complete_index] + if c.start_position == 0: + before = original_text_before_cursor + else: + before = original_text_before_cursor[: c.start_position] + + new_text = before + c.text + original_text_after_cursor + new_cursor_position = len(before) + len(c.text) + return new_text, new_cursor_position + + @property + def current_completion(self) -> Completion | None: + """ + Return the current completion, or return `None` when no completion is + selected. + """ + if self.complete_index is not None: + return self.completions[self.complete_index] + return None + + +_QUOTED_WORDS_RE = re.compile(r"""(\s+|".*?"|'.*?')""") + + +class YankNthArgState: + """ + For yank-last-arg/yank-nth-arg: Keep track of where we are in the history. + """ + + def __init__( + self, history_position: int = 0, n: int = -1, previous_inserted_word: str = "" + ) -> None: + self.history_position = history_position + self.previous_inserted_word = previous_inserted_word + self.n = n + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(history_position={self.history_position!r}, n={self.n!r}, previous_inserted_word={self.previous_inserted_word!r})" + + +BufferEventHandler = Callable[["Buffer"], None] +BufferAcceptHandler = Callable[["Buffer"], bool] + + +class Buffer: + """ + The core data structure that holds the text and cursor position of the + current input line and implements all text manipulations on top of it. It + also implements the history, undo stack and the completion state. + + :param completer: :class:`~prompt_toolkit.completion.Completer` instance. + :param history: :class:`~prompt_toolkit.history.History` instance. + :param tempfile_suffix: The tempfile suffix (extension) to be used for the + "open in editor" function. For a Python REPL, this would be ".py", so + that the editor knows the syntax highlighting to use. This can also be + a callable that returns a string. + :param tempfile: For more advanced tempfile situations where you need + control over the subdirectories and filename. For a Git Commit Message, + this would be ".git/COMMIT_EDITMSG", so that the editor knows the syntax + highlighting to use. This can also be a callable that returns a string. + :param name: Name for this buffer. E.g. DEFAULT_BUFFER. This is mostly + useful for key bindings where we sometimes prefer to refer to a buffer + by their name instead of by reference. + :param accept_handler: Called when the buffer input is accepted. (Usually + when the user presses `enter`.) The accept handler receives this + `Buffer` as input and should return True when the buffer text should be + kept instead of calling reset. + + In case of a `PromptSession` for instance, we want to keep the text, + because we will exit the application, and only reset it during the next + run. + :param max_number_of_completions: Never display more than this number of + completions, even when the completer can produce more (limited by + default to 10k for performance). + + Events: + + :param on_text_changed: When the buffer text changes. (Callable or None.) + :param on_text_insert: When new text is inserted. (Callable or None.) + :param on_cursor_position_changed: When the cursor moves. (Callable or None.) + :param on_completions_changed: When the completions were changed. (Callable or None.) + :param on_suggestion_set: When an auto-suggestion text has been set. (Callable or None.) + + Filters: + + :param complete_while_typing: :class:`~prompt_toolkit.filters.Filter` + or `bool`. Decide whether or not to do asynchronous autocompleting while + typing. + :param validate_while_typing: :class:`~prompt_toolkit.filters.Filter` + or `bool`. Decide whether or not to do asynchronous validation while + typing. + :param enable_history_search: :class:`~prompt_toolkit.filters.Filter` or + `bool` to indicate when up-arrow partial string matching is enabled. It + is advised to not enable this at the same time as + `complete_while_typing`, because when there is an autocompletion found, + the up arrows usually browse through the completions, rather than + through the history. + :param read_only: :class:`~prompt_toolkit.filters.Filter`. When True, + changes will not be allowed. + :param multiline: :class:`~prompt_toolkit.filters.Filter` or `bool`. When + not set, pressing `Enter` will call the `accept_handler`. Otherwise, + pressing `Esc-Enter` is required. + """ + + def __init__( + self, + completer: Completer | None = None, + auto_suggest: AutoSuggest | None = None, + history: History | None = None, + validator: Validator | None = None, + tempfile_suffix: str | Callable[[], str] = "", + tempfile: str | Callable[[], str] = "", + name: str = "", + complete_while_typing: FilterOrBool = False, + validate_while_typing: FilterOrBool = False, + enable_history_search: FilterOrBool = False, + document: Document | None = None, + accept_handler: BufferAcceptHandler | None = None, + read_only: FilterOrBool = False, + multiline: FilterOrBool = True, + max_number_of_completions: int = 10000, + on_text_changed: BufferEventHandler | None = None, + on_text_insert: BufferEventHandler | None = None, + on_cursor_position_changed: BufferEventHandler | None = None, + on_completions_changed: BufferEventHandler | None = None, + on_suggestion_set: BufferEventHandler | None = None, + ) -> None: + # Accept both filters and booleans as input. + enable_history_search = to_filter(enable_history_search) + complete_while_typing = to_filter(complete_while_typing) + validate_while_typing = to_filter(validate_while_typing) + read_only = to_filter(read_only) + multiline = to_filter(multiline) + + self.completer = completer or DummyCompleter() + self.auto_suggest = auto_suggest + self.validator = validator + self.tempfile_suffix = tempfile_suffix + self.tempfile = tempfile + self.name = name + self.accept_handler = accept_handler + + # Filters. (Usually, used by the key bindings to drive the buffer.) + self.complete_while_typing = complete_while_typing + self.validate_while_typing = validate_while_typing + self.enable_history_search = enable_history_search + self.read_only = read_only + self.multiline = multiline + self.max_number_of_completions = max_number_of_completions + + # Text width. (For wrapping, used by the Vi 'gq' operator.) + self.text_width = 0 + + #: The command buffer history. + # Note that we shouldn't use a lazy 'or' here. bool(history) could be + # False when empty. + self.history = InMemoryHistory() if history is None else history + + self.__cursor_position = 0 + + # Events + self.on_text_changed: Event[Buffer] = Event(self, on_text_changed) + self.on_text_insert: Event[Buffer] = Event(self, on_text_insert) + self.on_cursor_position_changed: Event[Buffer] = Event( + self, on_cursor_position_changed + ) + self.on_completions_changed: Event[Buffer] = Event(self, on_completions_changed) + self.on_suggestion_set: Event[Buffer] = Event(self, on_suggestion_set) + + # Document cache. (Avoid creating new Document instances.) + self._document_cache: FastDictCache[ + tuple[str, int, SelectionState | None], Document + ] = FastDictCache(Document, size=10) + + # Create completer / auto suggestion / validation coroutines. + self._async_suggester = self._create_auto_suggest_coroutine() + self._async_completer = self._create_completer_coroutine() + self._async_validator = self._create_auto_validate_coroutine() + + # Asyncio task for populating the history. + self._load_history_task: asyncio.Future[None] | None = None + + # Reset other attributes. + self.reset(document=document) + + def __repr__(self) -> str: + if len(self.text) < 15: + text = self.text + else: + text = self.text[:12] + "..." + + return f"" + + def reset( + self, document: Document | None = None, append_to_history: bool = False + ) -> None: + """ + :param append_to_history: Append current input to history first. + """ + if append_to_history: + self.append_to_history() + + document = document or Document() + + self.__cursor_position = document.cursor_position + + # `ValidationError` instance. (Will be set when the input is wrong.) + self.validation_error: ValidationError | None = None + self.validation_state: ValidationState | None = ValidationState.UNKNOWN + + # State of the selection. + self.selection_state: SelectionState | None = None + + # Multiple cursor mode. (When we press 'I' or 'A' in visual-block mode, + # we can insert text on multiple lines at once. This is implemented by + # using multiple cursors.) + self.multiple_cursor_positions: list[int] = [] + + # When doing consecutive up/down movements, prefer to stay at this column. + self.preferred_column: int | None = None + + # State of complete browser + # For interactive completion through Ctrl-N/Ctrl-P. + self.complete_state: CompletionState | None = None + + # State of Emacs yank-nth-arg completion. + self.yank_nth_arg_state: YankNthArgState | None = None # for yank-nth-arg. + + # Remember the document that we had *right before* the last paste + # operation. This is used for rotating through the kill ring. + self.document_before_paste: Document | None = None + + # Current suggestion. + self.suggestion: Suggestion | None = None + + # The history search text. (Used for filtering the history when we + # browse through it.) + self.history_search_text: str | None = None + + # Undo/redo stacks (stack of `(text, cursor_position)`). + self._undo_stack: list[tuple[str, int]] = [] + self._redo_stack: list[tuple[str, int]] = [] + + # Cancel history loader. If history loading was still ongoing. + # Cancel the `_load_history_task`, so that next repaint of the + # `BufferControl` we will repopulate it. + if self._load_history_task is not None: + self._load_history_task.cancel() + self._load_history_task = None + + #: The working lines. Similar to history, except that this can be + #: modified. The user can press arrow_up and edit previous entries. + #: Ctrl-C should reset this, and copy the whole history back in here. + #: Enter should process the current command and append to the real + #: history. + self._working_lines: deque[str] = deque([document.text]) + self.__working_index = 0 + + def load_history_if_not_yet_loaded(self) -> None: + """ + Create task for populating the buffer history (if not yet done). + + Note:: + + This needs to be called from within the event loop of the + application, because history loading is async, and we need to be + sure the right event loop is active. Therefor, we call this method + in the `BufferControl.create_content`. + + There are situations where prompt_toolkit applications are created + in one thread, but will later run in a different thread (Ptpython + is one example. The REPL runs in a separate thread, in order to + prevent interfering with a potential different event loop in the + main thread. The REPL UI however is still created in the main + thread.) We could decide to not support creating prompt_toolkit + objects in one thread and running the application in a different + thread, but history loading is the only place where it matters, and + this solves it. + """ + if self._load_history_task is None: + + async def load_history() -> None: + async for item in self.history.load(): + self._working_lines.appendleft(item) + self.__working_index += 1 + + self._load_history_task = get_app().create_background_task(load_history()) + + def load_history_done(f: asyncio.Future[None]) -> None: + """ + Handle `load_history` result when either done, cancelled, or + when an exception was raised. + """ + try: + f.result() + except asyncio.CancelledError: + # Ignore cancellation. But handle it, so that we don't get + # this traceback. + pass + except GeneratorExit: + # Probably not needed, but we had situations where + # `GeneratorExit` was raised in `load_history` during + # cancellation. + pass + except BaseException: + # Log error if something goes wrong. (We don't have a + # caller to which we can propagate this exception.) + logger.exception("Loading history failed") + + self._load_history_task.add_done_callback(load_history_done) + + # + + def _set_text(self, value: str) -> bool: + """set text at current working_index. Return whether it changed.""" + working_index = self.working_index + working_lines = self._working_lines + + original_value = working_lines[working_index] + working_lines[working_index] = value + + # Return True when this text has been changed. + if len(value) != len(original_value): + # For Python 2, it seems that when two strings have a different + # length and one is a prefix of the other, Python still scans + # character by character to see whether the strings are different. + # (Some benchmarking showed significant differences for big + # documents. >100,000 of lines.) + return True + elif value != original_value: + return True + return False + + def _set_cursor_position(self, value: int) -> bool: + """Set cursor position. Return whether it changed.""" + original_position = self.__cursor_position + self.__cursor_position = max(0, value) + + return self.__cursor_position != original_position + + @property + def text(self) -> str: + return self._working_lines[self.working_index] + + @text.setter + def text(self, value: str) -> None: + """ + Setting text. (When doing this, make sure that the cursor_position is + valid for this text. text/cursor_position should be consistent at any time, + otherwise set a Document instead.) + """ + # Ensure cursor position remains within the size of the text. + if self.cursor_position > len(value): + self.cursor_position = len(value) + + # Don't allow editing of read-only buffers. + if self.read_only(): + raise EditReadOnlyBuffer() + + changed = self._set_text(value) + + if changed: + self._text_changed() + + # Reset history search text. + # (Note that this doesn't need to happen when working_index + # changes, which is when we traverse the history. That's why we + # don't do this in `self._text_changed`.) + self.history_search_text = None + + @property + def cursor_position(self) -> int: + return self.__cursor_position + + @cursor_position.setter + def cursor_position(self, value: int) -> None: + """ + Setting cursor position. + """ + assert isinstance(value, int) + + # Ensure cursor position is within the size of the text. + if value > len(self.text): + value = len(self.text) + if value < 0: + value = 0 + + changed = self._set_cursor_position(value) + + if changed: + self._cursor_position_changed() + + @property + def working_index(self) -> int: + return self.__working_index + + @working_index.setter + def working_index(self, value: int) -> None: + if self.__working_index != value: + self.__working_index = value + # Make sure to reset the cursor position, otherwise we end up in + # situations where the cursor position is out of the bounds of the + # text. + self.cursor_position = 0 + self._text_changed() + + def _text_changed(self) -> None: + # Remove any validation errors and complete state. + self.validation_error = None + self.validation_state = ValidationState.UNKNOWN + self.complete_state = None + self.yank_nth_arg_state = None + self.document_before_paste = None + self.selection_state = None + self.suggestion = None + self.preferred_column = None + + # fire 'on_text_changed' event. + self.on_text_changed.fire() + + # Input validation. + # (This happens on all change events, unlike auto completion, also when + # deleting text.) + if self.validator and self.validate_while_typing(): + get_app().create_background_task(self._async_validator()) + + def _cursor_position_changed(self) -> None: + # Remove any complete state. + # (Input validation should only be undone when the cursor position + # changes.) + self.complete_state = None + self.yank_nth_arg_state = None + self.document_before_paste = None + + # Unset preferred_column. (Will be set after the cursor movement, if + # required.) + self.preferred_column = None + + # Note that the cursor position can change if we have a selection the + # new position of the cursor determines the end of the selection. + + # fire 'on_cursor_position_changed' event. + self.on_cursor_position_changed.fire() + + @property + def document(self) -> Document: + """ + Return :class:`~prompt_toolkit.document.Document` instance from the + current text, cursor position and selection state. + """ + return self._document_cache[ + self.text, self.cursor_position, self.selection_state + ] + + @document.setter + def document(self, value: Document) -> None: + """ + Set :class:`~prompt_toolkit.document.Document` instance. + + This will set both the text and cursor position at the same time, but + atomically. (Change events will be triggered only after both have been set.) + """ + self.set_document(value) + + def set_document(self, value: Document, bypass_readonly: bool = False) -> None: + """ + Set :class:`~prompt_toolkit.document.Document` instance. Like the + ``document`` property, but accept an ``bypass_readonly`` argument. + + :param bypass_readonly: When True, don't raise an + :class:`.EditReadOnlyBuffer` exception, even + when the buffer is read-only. + + .. warning:: + + When this buffer is read-only and `bypass_readonly` was not passed, + the `EditReadOnlyBuffer` exception will be caught by the + `KeyProcessor` and is silently suppressed. This is important to + keep in mind when writing key bindings, because it won't do what + you expect, and there won't be a stack trace. Use try/finally + around this function if you need some cleanup code. + """ + # Don't allow editing of read-only buffers. + if not bypass_readonly and self.read_only(): + raise EditReadOnlyBuffer() + + # Set text and cursor position first. + text_changed = self._set_text(value.text) + cursor_position_changed = self._set_cursor_position(value.cursor_position) + + # Now handle change events. (We do this when text/cursor position is + # both set and consistent.) + if text_changed: + self._text_changed() + self.history_search_text = None + + if cursor_position_changed: + self._cursor_position_changed() + + @property + def is_returnable(self) -> bool: + """ + True when there is something handling accept. + """ + return bool(self.accept_handler) + + # End of + + def save_to_undo_stack(self, clear_redo_stack: bool = True) -> None: + """ + Safe current state (input text and cursor position), so that we can + restore it by calling undo. + """ + # Safe if the text is different from the text at the top of the stack + # is different. If the text is the same, just update the cursor position. + if self._undo_stack and self._undo_stack[-1][0] == self.text: + self._undo_stack[-1] = (self._undo_stack[-1][0], self.cursor_position) + else: + self._undo_stack.append((self.text, self.cursor_position)) + + # Saving anything to the undo stack, clears the redo stack. + if clear_redo_stack: + self._redo_stack = [] + + def transform_lines( + self, + line_index_iterator: Iterable[int], + transform_callback: Callable[[str], str], + ) -> str: + """ + Transforms the text on a range of lines. + When the iterator yield an index not in the range of lines that the + document contains, it skips them silently. + + To uppercase some lines:: + + new_text = transform_lines(range(5,10), lambda text: text.upper()) + + :param line_index_iterator: Iterator of line numbers (int) + :param transform_callback: callable that takes the original text of a + line, and return the new text for this line. + + :returns: The new text. + """ + # Split lines + lines = self.text.split("\n") + + # Apply transformation + for index in line_index_iterator: + try: + lines[index] = transform_callback(lines[index]) + except IndexError: + pass + + return "\n".join(lines) + + def transform_current_line(self, transform_callback: Callable[[str], str]) -> None: + """ + Apply the given transformation function to the current line. + + :param transform_callback: callable that takes a string and return a new string. + """ + document = self.document + a = document.cursor_position + document.get_start_of_line_position() + b = document.cursor_position + document.get_end_of_line_position() + self.text = ( + document.text[:a] + + transform_callback(document.text[a:b]) + + document.text[b:] + ) + + def transform_region( + self, from_: int, to: int, transform_callback: Callable[[str], str] + ) -> None: + """ + Transform a part of the input string. + + :param from_: (int) start position. + :param to: (int) end position. + :param transform_callback: Callable which accepts a string and returns + the transformed string. + """ + assert from_ < to + + self.text = "".join( + [ + self.text[:from_] + + transform_callback(self.text[from_:to]) + + self.text[to:] + ] + ) + + def cursor_left(self, count: int = 1) -> None: + self.cursor_position += self.document.get_cursor_left_position(count=count) + + def cursor_right(self, count: int = 1) -> None: + self.cursor_position += self.document.get_cursor_right_position(count=count) + + def cursor_up(self, count: int = 1) -> None: + """(for multiline edit). Move cursor to the previous line.""" + original_column = self.preferred_column or self.document.cursor_position_col + self.cursor_position += self.document.get_cursor_up_position( + count=count, preferred_column=original_column + ) + + # Remember the original column for the next up/down movement. + self.preferred_column = original_column + + def cursor_down(self, count: int = 1) -> None: + """(for multiline edit). Move cursor to the next line.""" + original_column = self.preferred_column or self.document.cursor_position_col + self.cursor_position += self.document.get_cursor_down_position( + count=count, preferred_column=original_column + ) + + # Remember the original column for the next up/down movement. + self.preferred_column = original_column + + def auto_up( + self, count: int = 1, go_to_start_of_line_if_history_changes: bool = False + ) -> None: + """ + If we're not on the first line (of a multiline input) go a line up, + otherwise go back in history. (If nothing is selected.) + """ + if self.complete_state: + self.complete_previous(count=count) + elif self.document.cursor_position_row > 0: + self.cursor_up(count=count) + elif not self.selection_state: + self.history_backward(count=count) + + # Go to the start of the line? + if go_to_start_of_line_if_history_changes: + self.cursor_position += self.document.get_start_of_line_position() + + def auto_down( + self, count: int = 1, go_to_start_of_line_if_history_changes: bool = False + ) -> None: + """ + If we're not on the last line (of a multiline input) go a line down, + otherwise go forward in history. (If nothing is selected.) + """ + if self.complete_state: + self.complete_next(count=count) + elif self.document.cursor_position_row < self.document.line_count - 1: + self.cursor_down(count=count) + elif not self.selection_state: + self.history_forward(count=count) + + # Go to the start of the line? + if go_to_start_of_line_if_history_changes: + self.cursor_position += self.document.get_start_of_line_position() + + def delete_before_cursor(self, count: int = 1) -> str: + """ + Delete specified number of characters before cursor and return the + deleted text. + """ + assert count >= 0 + deleted = "" + + if self.cursor_position > 0: + deleted = self.text[self.cursor_position - count : self.cursor_position] + + new_text = ( + self.text[: self.cursor_position - count] + + self.text[self.cursor_position :] + ) + new_cursor_position = self.cursor_position - len(deleted) + + # Set new Document atomically. + self.document = Document(new_text, new_cursor_position) + + return deleted + + def delete(self, count: int = 1) -> str: + """ + Delete specified number of characters and Return the deleted text. + """ + if self.cursor_position < len(self.text): + deleted = self.document.text_after_cursor[:count] + self.text = ( + self.text[: self.cursor_position] + + self.text[self.cursor_position + len(deleted) :] + ) + return deleted + else: + return "" + + def join_next_line(self, separator: str = " ") -> None: + """ + Join the next line to the current one by deleting the line ending after + the current line. + """ + if not self.document.on_last_line: + self.cursor_position += self.document.get_end_of_line_position() + self.delete() + + # Remove spaces. + self.text = ( + self.document.text_before_cursor + + separator + + self.document.text_after_cursor.lstrip(" ") + ) + + def join_selected_lines(self, separator: str = " ") -> None: + """ + Join the selected lines. + """ + assert self.selection_state + + # Get lines. + from_, to = sorted( + [self.cursor_position, self.selection_state.original_cursor_position] + ) + + before = self.text[:from_] + lines = self.text[from_:to].splitlines() + after = self.text[to:] + + # Replace leading spaces with just one space. + lines = [l.lstrip(" ") + separator for l in lines] + + # Set new document. + self.document = Document( + text=before + "".join(lines) + after, + cursor_position=len(before + "".join(lines[:-1])) - 1, + ) + + def swap_characters_before_cursor(self) -> None: + """ + Swap the last two characters before the cursor. + """ + pos = self.cursor_position + + if pos >= 2: + a = self.text[pos - 2] + b = self.text[pos - 1] + + self.text = self.text[: pos - 2] + b + a + self.text[pos:] + + def go_to_history(self, index: int) -> None: + """ + Go to this item in the history. + """ + if index < len(self._working_lines): + self.working_index = index + self.cursor_position = len(self.text) + + def complete_next(self, count: int = 1, disable_wrap_around: bool = False) -> None: + """ + Browse to the next completions. + (Does nothing if there are no completion.) + """ + index: int | None + + if self.complete_state: + completions_count = len(self.complete_state.completions) + + if self.complete_state.complete_index is None: + index = 0 + elif self.complete_state.complete_index == completions_count - 1: + index = None + + if disable_wrap_around: + return + else: + index = min( + completions_count - 1, self.complete_state.complete_index + count + ) + self.go_to_completion(index) + + def complete_previous( + self, count: int = 1, disable_wrap_around: bool = False + ) -> None: + """ + Browse to the previous completions. + (Does nothing if there are no completion.) + """ + index: int | None + + if self.complete_state: + if self.complete_state.complete_index == 0: + index = None + + if disable_wrap_around: + return + elif self.complete_state.complete_index is None: + index = len(self.complete_state.completions) - 1 + else: + index = max(0, self.complete_state.complete_index - count) + + self.go_to_completion(index) + + def cancel_completion(self) -> None: + """ + Cancel completion, go back to the original text. + """ + if self.complete_state: + self.go_to_completion(None) + self.complete_state = None + + def _set_completions(self, completions: list[Completion]) -> CompletionState: + """ + Start completions. (Generate list of completions and initialize.) + + By default, no completion will be selected. + """ + self.complete_state = CompletionState( + original_document=self.document, completions=completions + ) + + # Trigger event. This should eventually invalidate the layout. + self.on_completions_changed.fire() + + return self.complete_state + + def start_history_lines_completion(self) -> None: + """ + Start a completion based on all the other lines in the document and the + history. + """ + found_completions: set[str] = set() + completions = [] + + # For every line of the whole history, find matches with the current line. + current_line = self.document.current_line_before_cursor.lstrip() + + for i, string in enumerate(self._working_lines): + for j, l in enumerate(string.split("\n")): + l = l.strip() + if l and l.startswith(current_line): + # When a new line has been found. + if l not in found_completions: + found_completions.add(l) + + # Create completion. + if i == self.working_index: + display_meta = "Current, line %s" % (j + 1) + else: + display_meta = f"History {i + 1}, line {j + 1}" + + completions.append( + Completion( + text=l, + start_position=-len(current_line), + display_meta=display_meta, + ) + ) + + self._set_completions(completions=completions[::-1]) + self.go_to_completion(0) + + def go_to_completion(self, index: int | None) -> None: + """ + Select a completion from the list of current completions. + """ + assert self.complete_state + + # Set new completion + state = self.complete_state + state.go_to_index(index) + + # Set text/cursor position + new_text, new_cursor_position = state.new_text_and_position() + self.document = Document(new_text, new_cursor_position) + + # (changing text/cursor position will unset complete_state.) + self.complete_state = state + + def apply_completion(self, completion: Completion) -> None: + """ + Insert a given completion. + """ + # If there was already a completion active, cancel that one. + if self.complete_state: + self.go_to_completion(None) + self.complete_state = None + + # Insert text from the given completion. + self.delete_before_cursor(-completion.start_position) + self.insert_text(completion.text) + + def _set_history_search(self) -> None: + """ + Set `history_search_text`. + (The text before the cursor will be used for filtering the history.) + """ + if self.enable_history_search(): + if self.history_search_text is None: + self.history_search_text = self.document.text_before_cursor + else: + self.history_search_text = None + + def _history_matches(self, i: int) -> bool: + """ + True when the current entry matches the history search. + (when we don't have history search, it's also True.) + """ + return self.history_search_text is None or self._working_lines[i].startswith( + self.history_search_text + ) + + def history_forward(self, count: int = 1) -> None: + """ + Move forwards through the history. + + :param count: Amount of items to move forward. + """ + self._set_history_search() + + # Go forward in history. + found_something = False + + for i in range(self.working_index + 1, len(self._working_lines)): + if self._history_matches(i): + self.working_index = i + count -= 1 + found_something = True + if count == 0: + break + + # If we found an entry, move cursor to the end of the first line. + if found_something: + self.cursor_position = 0 + self.cursor_position += self.document.get_end_of_line_position() + + def history_backward(self, count: int = 1) -> None: + """ + Move backwards through history. + """ + self._set_history_search() + + # Go back in history. + found_something = False + + for i in range(self.working_index - 1, -1, -1): + if self._history_matches(i): + self.working_index = i + count -= 1 + found_something = True + if count == 0: + break + + # If we move to another entry, move cursor to the end of the line. + if found_something: + self.cursor_position = len(self.text) + + def yank_nth_arg(self, n: int | None = None, _yank_last_arg: bool = False) -> None: + """ + Pick nth word from previous history entry (depending on current + `yank_nth_arg_state`) and insert it at current position. Rotate through + history if called repeatedly. If no `n` has been given, take the first + argument. (The second word.) + + :param n: (None or int), The index of the word from the previous line + to take. + """ + assert n is None or isinstance(n, int) + history_strings = self.history.get_strings() + + if not len(history_strings): + return + + # Make sure we have a `YankNthArgState`. + if self.yank_nth_arg_state is None: + state = YankNthArgState(n=-1 if _yank_last_arg else 1) + else: + state = self.yank_nth_arg_state + + if n is not None: + state.n = n + + # Get new history position. + new_pos = state.history_position - 1 + if -new_pos > len(history_strings): + new_pos = -1 + + # Take argument from line. + line = history_strings[new_pos] + + words = [w.strip() for w in _QUOTED_WORDS_RE.split(line)] + words = [w for w in words if w] + try: + word = words[state.n] + except IndexError: + word = "" + + # Insert new argument. + if state.previous_inserted_word: + self.delete_before_cursor(len(state.previous_inserted_word)) + self.insert_text(word) + + # Save state again for next completion. (Note that the 'insert' + # operation from above clears `self.yank_nth_arg_state`.) + state.previous_inserted_word = word + state.history_position = new_pos + self.yank_nth_arg_state = state + + def yank_last_arg(self, n: int | None = None) -> None: + """ + Like `yank_nth_arg`, but if no argument has been given, yank the last + word by default. + """ + self.yank_nth_arg(n=n, _yank_last_arg=True) + + def start_selection( + self, selection_type: SelectionType = SelectionType.CHARACTERS + ) -> None: + """ + Take the current cursor position as the start of this selection. + """ + self.selection_state = SelectionState(self.cursor_position, selection_type) + + def copy_selection(self, _cut: bool = False) -> ClipboardData: + """ + Copy selected text and return :class:`.ClipboardData` instance. + + Notice that this doesn't store the copied data on the clipboard yet. + You can store it like this: + + .. code:: python + + data = buffer.copy_selection() + get_app().clipboard.set_data(data) + """ + new_document, clipboard_data = self.document.cut_selection() + if _cut: + self.document = new_document + + self.selection_state = None + return clipboard_data + + def cut_selection(self) -> ClipboardData: + """ + Delete selected text and return :class:`.ClipboardData` instance. + """ + return self.copy_selection(_cut=True) + + def paste_clipboard_data( + self, + data: ClipboardData, + paste_mode: PasteMode = PasteMode.EMACS, + count: int = 1, + ) -> None: + """ + Insert the data from the clipboard. + """ + assert isinstance(data, ClipboardData) + assert paste_mode in (PasteMode.VI_BEFORE, PasteMode.VI_AFTER, PasteMode.EMACS) + + original_document = self.document + self.document = self.document.paste_clipboard_data( + data, paste_mode=paste_mode, count=count + ) + + # Remember original document. This assignment should come at the end, + # because assigning to 'document' will erase it. + self.document_before_paste = original_document + + def newline(self, copy_margin: bool = True) -> None: + """ + Insert a line ending at the current position. + """ + if copy_margin: + self.insert_text("\n" + self.document.leading_whitespace_in_current_line) + else: + self.insert_text("\n") + + def insert_line_above(self, copy_margin: bool = True) -> None: + """ + Insert a new line above the current one. + """ + if copy_margin: + insert = self.document.leading_whitespace_in_current_line + "\n" + else: + insert = "\n" + + self.cursor_position += self.document.get_start_of_line_position() + self.insert_text(insert) + self.cursor_position -= 1 + + def insert_line_below(self, copy_margin: bool = True) -> None: + """ + Insert a new line below the current one. + """ + if copy_margin: + insert = "\n" + self.document.leading_whitespace_in_current_line + else: + insert = "\n" + + self.cursor_position += self.document.get_end_of_line_position() + self.insert_text(insert) + + def insert_text( + self, + data: str, + overwrite: bool = False, + move_cursor: bool = True, + fire_event: bool = True, + ) -> None: + """ + Insert characters at cursor position. + + :param fire_event: Fire `on_text_insert` event. This is mainly used to + trigger autocompletion while typing. + """ + # Original text & cursor position. + otext = self.text + ocpos = self.cursor_position + + # In insert/text mode. + if overwrite: + # Don't overwrite the newline itself. Just before the line ending, + # it should act like insert mode. + overwritten_text = otext[ocpos : ocpos + len(data)] + if "\n" in overwritten_text: + overwritten_text = overwritten_text[: overwritten_text.find("\n")] + + text = otext[:ocpos] + data + otext[ocpos + len(overwritten_text) :] + else: + text = otext[:ocpos] + data + otext[ocpos:] + + if move_cursor: + cpos = self.cursor_position + len(data) + else: + cpos = self.cursor_position + + # Set new document. + # (Set text and cursor position at the same time. Otherwise, setting + # the text will fire a change event before the cursor position has been + # set. It works better to have this atomic.) + self.document = Document(text, cpos) + + # Fire 'on_text_insert' event. + if fire_event: # XXX: rename to `start_complete`. + self.on_text_insert.fire() + + # Only complete when "complete_while_typing" is enabled. + if self.completer and self.complete_while_typing(): + get_app().create_background_task(self._async_completer()) + + # Call auto_suggest. + if self.auto_suggest: + get_app().create_background_task(self._async_suggester()) + + def undo(self) -> None: + # Pop from the undo-stack until we find a text that if different from + # the current text. (The current logic of `save_to_undo_stack` will + # cause that the top of the undo stack is usually the same as the + # current text, so in that case we have to pop twice.) + while self._undo_stack: + text, pos = self._undo_stack.pop() + + if text != self.text: + # Push current text to redo stack. + self._redo_stack.append((self.text, self.cursor_position)) + + # Set new text/cursor_position. + self.document = Document(text, cursor_position=pos) + break + + def redo(self) -> None: + if self._redo_stack: + # Copy current state on undo stack. + self.save_to_undo_stack(clear_redo_stack=False) + + # Pop state from redo stack. + text, pos = self._redo_stack.pop() + self.document = Document(text, cursor_position=pos) + + def validate(self, set_cursor: bool = False) -> bool: + """ + Returns `True` if valid. + + :param set_cursor: Set the cursor position, if an error was found. + """ + # Don't call the validator again, if it was already called for the + # current input. + if self.validation_state != ValidationState.UNKNOWN: + return self.validation_state == ValidationState.VALID + + # Call validator. + if self.validator: + try: + self.validator.validate(self.document) + except ValidationError as e: + # Set cursor position (don't allow invalid values.) + if set_cursor: + self.cursor_position = min( + max(0, e.cursor_position), len(self.text) + ) + + self.validation_state = ValidationState.INVALID + self.validation_error = e + return False + + # Handle validation result. + self.validation_state = ValidationState.VALID + self.validation_error = None + return True + + async def _validate_async(self) -> None: + """ + Asynchronous version of `validate()`. + This one doesn't set the cursor position. + + We have both variants, because a synchronous version is required. + Handling the ENTER key needs to be completely synchronous, otherwise + stuff like type-ahead is going to give very weird results. (People + could type input while the ENTER key is still processed.) + + An asynchronous version is required if we have `validate_while_typing` + enabled. + """ + while True: + # Don't call the validator again, if it was already called for the + # current input. + if self.validation_state != ValidationState.UNKNOWN: + return + + # Call validator. + error = None + document = self.document + + if self.validator: + try: + await self.validator.validate_async(self.document) + except ValidationError as e: + error = e + + # If the document changed during the validation, try again. + if self.document != document: + continue + + # Handle validation result. + if error: + self.validation_state = ValidationState.INVALID + else: + self.validation_state = ValidationState.VALID + + self.validation_error = error + get_app().invalidate() # Trigger redraw (display error). + + def append_to_history(self) -> None: + """ + Append the current input to the history. + """ + # Save at the tail of the history. (But don't if the last entry the + # history is already the same.) + if self.text: + history_strings = self.history.get_strings() + if not len(history_strings) or history_strings[-1] != self.text: + self.history.append_string(self.text) + + def _search( + self, + search_state: SearchState, + include_current_position: bool = False, + count: int = 1, + ) -> tuple[int, int] | None: + """ + Execute search. Return (working_index, cursor_position) tuple when this + search is applied. Returns `None` when this text cannot be found. + """ + assert count > 0 + + text = search_state.text + direction = search_state.direction + ignore_case = search_state.ignore_case() + + def search_once( + working_index: int, document: Document + ) -> tuple[int, Document] | None: + """ + Do search one time. + Return (working_index, document) or `None` + """ + if direction == SearchDirection.FORWARD: + # Try find at the current input. + new_index = document.find( + text, + include_current_position=include_current_position, + ignore_case=ignore_case, + ) + + if new_index is not None: + return ( + working_index, + Document(document.text, document.cursor_position + new_index), + ) + else: + # No match, go forward in the history. (Include len+1 to wrap around.) + # (Here we should always include all cursor positions, because + # it's a different line.) + for i in range(working_index + 1, len(self._working_lines) + 1): + i %= len(self._working_lines) + + document = Document(self._working_lines[i], 0) + new_index = document.find( + text, include_current_position=True, ignore_case=ignore_case + ) + if new_index is not None: + return (i, Document(document.text, new_index)) + else: + # Try find at the current input. + new_index = document.find_backwards(text, ignore_case=ignore_case) + + if new_index is not None: + return ( + working_index, + Document(document.text, document.cursor_position + new_index), + ) + else: + # No match, go back in the history. (Include -1 to wrap around.) + for i in range(working_index - 1, -2, -1): + i %= len(self._working_lines) + + document = Document( + self._working_lines[i], len(self._working_lines[i]) + ) + new_index = document.find_backwards( + text, ignore_case=ignore_case + ) + if new_index is not None: + return ( + i, + Document(document.text, len(document.text) + new_index), + ) + return None + + # Do 'count' search iterations. + working_index = self.working_index + document = self.document + for _ in range(count): + result = search_once(working_index, document) + if result is None: + return None # Nothing found. + else: + working_index, document = result + + return (working_index, document.cursor_position) + + def document_for_search(self, search_state: SearchState) -> Document: + """ + Return a :class:`~prompt_toolkit.document.Document` instance that has + the text/cursor position for this search, if we would apply it. This + will be used in the + :class:`~prompt_toolkit.layout.BufferControl` to display feedback while + searching. + """ + search_result = self._search(search_state, include_current_position=True) + + if search_result is None: + return self.document + else: + working_index, cursor_position = search_result + + # Keep selection, when `working_index` was not changed. + if working_index == self.working_index: + selection = self.selection_state + else: + selection = None + + return Document( + self._working_lines[working_index], cursor_position, selection=selection + ) + + def get_search_position( + self, + search_state: SearchState, + include_current_position: bool = True, + count: int = 1, + ) -> int: + """ + Get the cursor position for this search. + (This operation won't change the `working_index`. It's won't go through + the history. Vi text objects can't span multiple items.) + """ + search_result = self._search( + search_state, include_current_position=include_current_position, count=count + ) + + if search_result is None: + return self.cursor_position + else: + working_index, cursor_position = search_result + return cursor_position + + def apply_search( + self, + search_state: SearchState, + include_current_position: bool = True, + count: int = 1, + ) -> None: + """ + Apply search. If something is found, set `working_index` and + `cursor_position`. + """ + search_result = self._search( + search_state, include_current_position=include_current_position, count=count + ) + + if search_result is not None: + working_index, cursor_position = search_result + self.working_index = working_index + self.cursor_position = cursor_position + + def exit_selection(self) -> None: + self.selection_state = None + + def _editor_simple_tempfile(self) -> tuple[str, Callable[[], None]]: + """ + Simple (file) tempfile implementation. + Return (tempfile, cleanup_func). + """ + suffix = to_str(self.tempfile_suffix) + descriptor, filename = tempfile.mkstemp(suffix) + + os.write(descriptor, self.text.encode("utf-8")) + os.close(descriptor) + + def cleanup() -> None: + os.unlink(filename) + + return filename, cleanup + + def _editor_complex_tempfile(self) -> tuple[str, Callable[[], None]]: + # Complex (directory) tempfile implementation. + headtail = to_str(self.tempfile) + if not headtail: + # Revert to simple case. + return self._editor_simple_tempfile() + headtail = str(headtail) + + # Try to make according to tempfile logic. + head, tail = os.path.split(headtail) + if os.path.isabs(head): + head = head[1:] + + dirpath = tempfile.mkdtemp() + if head: + dirpath = os.path.join(dirpath, head) + # Assume there is no issue creating dirs in this temp dir. + os.makedirs(dirpath) + + # Open the filename and write current text. + filename = os.path.join(dirpath, tail) + with open(filename, "w", encoding="utf-8") as fh: + fh.write(self.text) + + def cleanup() -> None: + shutil.rmtree(dirpath) + + return filename, cleanup + + def open_in_editor(self, validate_and_handle: bool = False) -> asyncio.Task[None]: + """ + Open code in editor. + + This returns a future, and runs in a thread executor. + """ + if self.read_only(): + raise EditReadOnlyBuffer() + + # Write current text to temporary file + if self.tempfile: + filename, cleanup_func = self._editor_complex_tempfile() + else: + filename, cleanup_func = self._editor_simple_tempfile() + + async def run() -> None: + try: + # Open in editor + # (We need to use `run_in_terminal`, because not all editors go to + # the alternate screen buffer, and some could influence the cursor + # position.) + success = await run_in_terminal( + lambda: self._open_file_in_editor(filename), in_executor=True + ) + + # Read content again. + if success: + with open(filename, "rb") as f: + text = f.read().decode("utf-8") + + # Drop trailing newline. (Editors are supposed to add it at the + # end, but we don't need it.) + if text.endswith("\n"): + text = text[:-1] + + self.document = Document(text=text, cursor_position=len(text)) + + # Accept the input. + if validate_and_handle: + self.validate_and_handle() + + finally: + # Clean up temp dir/file. + cleanup_func() + + return get_app().create_background_task(run()) + + def _open_file_in_editor(self, filename: str) -> bool: + """ + Call editor executable. + + Return True when we received a zero return code. + """ + # If the 'VISUAL' or 'EDITOR' environment variable has been set, use that. + # Otherwise, fall back to the first available editor that we can find. + visual = os.environ.get("VISUAL") + editor = os.environ.get("EDITOR") + + editors = [ + visual, + editor, + # Order of preference. + "/usr/bin/editor", + "/usr/bin/nano", + "/usr/bin/pico", + "/usr/bin/vi", + "/usr/bin/emacs", + ] + + for e in editors: + if e: + try: + # Use 'shlex.split()', because $VISUAL can contain spaces + # and quotes. + returncode = subprocess.call(shlex.split(e) + [filename]) + return returncode == 0 + + except OSError: + # Executable does not exist, try the next one. + pass + + return False + + def start_completion( + self, + select_first: bool = False, + select_last: bool = False, + insert_common_part: bool = False, + complete_event: CompleteEvent | None = None, + ) -> None: + """ + Start asynchronous autocompletion of this buffer. + (This will do nothing if a previous completion was still in progress.) + """ + # Only one of these options can be selected. + assert select_first + select_last + insert_common_part <= 1 + + get_app().create_background_task( + self._async_completer( + select_first=select_first, + select_last=select_last, + insert_common_part=insert_common_part, + complete_event=complete_event + or CompleteEvent(completion_requested=True), + ) + ) + + def _create_completer_coroutine(self) -> Callable[..., Coroutine[Any, Any, None]]: + """ + Create function for asynchronous autocompletion. + + (This consumes the asynchronous completer generator, which possibly + runs the completion algorithm in another thread.) + """ + + def completion_does_nothing(document: Document, completion: Completion) -> bool: + """ + Return `True` if applying this completion doesn't have any effect. + (When it doesn't insert any new text. + """ + text_before_cursor = document.text_before_cursor + replaced_text = text_before_cursor[ + len(text_before_cursor) + completion.start_position : + ] + return replaced_text == completion.text + + @_only_one_at_a_time + async def async_completer( + select_first: bool = False, + select_last: bool = False, + insert_common_part: bool = False, + complete_event: CompleteEvent | None = None, + ) -> None: + document = self.document + complete_event = complete_event or CompleteEvent(text_inserted=True) + + # Don't complete when we already have completions. + if self.complete_state or not self.completer: + return + + # Create an empty CompletionState. + complete_state = CompletionState(original_document=self.document) + self.complete_state = complete_state + + def proceed() -> bool: + """Keep retrieving completions. Input text has not yet changed + while generating completions.""" + return self.complete_state == complete_state + + refresh_needed = asyncio.Event() + + async def refresh_while_loading() -> None: + """Background loop to refresh the UI at most 3 times a second + while the completion are loading. Calling + `on_completions_changed.fire()` for every completion that we + receive is too expensive when there are many completions. (We + could tune `Application.max_render_postpone_time` and + `Application.min_redraw_interval`, but having this here is a + better approach.) + """ + while True: + self.on_completions_changed.fire() + refresh_needed.clear() + await asyncio.sleep(0.3) + await refresh_needed.wait() + + refresh_task = asyncio.ensure_future(refresh_while_loading()) + try: + # Load. + async with aclosing( + self.completer.get_completions_async(document, complete_event) + ) as async_generator: + async for completion in async_generator: + complete_state.completions.append(completion) + refresh_needed.set() + + # If the input text changes, abort. + if not proceed(): + break + + # Always stop at 10k completions. + if ( + len(complete_state.completions) + >= self.max_number_of_completions + ): + break + finally: + refresh_task.cancel() + + # Refresh one final time after we got everything. + self.on_completions_changed.fire() + + completions = complete_state.completions + + # When there is only one completion, which has nothing to add, ignore it. + if len(completions) == 1 and completion_does_nothing( + document, completions[0] + ): + del completions[:] + + # Set completions if the text was not yet changed. + if proceed(): + # When no completions were found, or when the user selected + # already a completion by using the arrow keys, don't do anything. + if ( + not self.complete_state + or self.complete_state.complete_index is not None + ): + return + + # When there are no completions, reset completion state anyway. + if not completions: + self.complete_state = None + # Render the ui if the completion menu was shown + # it is needed especially if there is one completion and it was deleted. + self.on_completions_changed.fire() + return + + # Select first/last or insert common part, depending on the key + # binding. (For this we have to wait until all completions are + # loaded.) + + if select_first: + self.go_to_completion(0) + + elif select_last: + self.go_to_completion(len(completions) - 1) + + elif insert_common_part: + common_part = get_common_complete_suffix(document, completions) + if common_part: + # Insert the common part, update completions. + self.insert_text(common_part) + if len(completions) > 1: + # (Don't call `async_completer` again, but + # recalculate completions. See: + # https://github.com/ipython/ipython/issues/9658) + completions[:] = [ + c.new_completion_from_position(len(common_part)) + for c in completions + ] + + self._set_completions(completions=completions) + else: + self.complete_state = None + else: + # When we were asked to insert the "common" + # prefix, but there was no common suffix but + # still exactly one match, then select the + # first. (It could be that we have a completion + # which does * expansion, like '*.py', with + # exactly one match.) + if len(completions) == 1: + self.go_to_completion(0) + + else: + # If the last operation was an insert, (not a delete), restart + # the completion coroutine. + + if self.document.text_before_cursor == document.text_before_cursor: + return # Nothing changed. + + if self.document.text_before_cursor.startswith( + document.text_before_cursor + ): + raise _Retry + + return async_completer + + def _create_auto_suggest_coroutine(self) -> Callable[[], Coroutine[Any, Any, None]]: + """ + Create function for asynchronous auto suggestion. + (This can be in another thread.) + """ + + @_only_one_at_a_time + async def async_suggestor() -> None: + document = self.document + + # Don't suggest when we already have a suggestion. + if self.suggestion or not self.auto_suggest: + return + + suggestion = await self.auto_suggest.get_suggestion_async(self, document) + + # Set suggestion only if the text was not yet changed. + if self.document == document: + # Set suggestion and redraw interface. + self.suggestion = suggestion + self.on_suggestion_set.fire() + else: + # Otherwise, restart thread. + raise _Retry + + return async_suggestor + + def _create_auto_validate_coroutine( + self, + ) -> Callable[[], Coroutine[Any, Any, None]]: + """ + Create a function for asynchronous validation while typing. + (This can be in another thread.) + """ + + @_only_one_at_a_time + async def async_validator() -> None: + await self._validate_async() + + return async_validator + + def validate_and_handle(self) -> None: + """ + Validate buffer and handle the accept action. + """ + valid = self.validate(set_cursor=True) + + # When the validation succeeded, accept the input. + if valid: + if self.accept_handler: + keep_text = self.accept_handler(self) + else: + keep_text = False + + self.append_to_history() + + if not keep_text: + self.reset() + + +_T = TypeVar("_T", bound=Callable[..., Coroutine[Any, Any, None]]) + + +def _only_one_at_a_time(coroutine: _T) -> _T: + """ + Decorator that only starts the coroutine only if the previous call has + finished. (Used to make sure that we have only one autocompleter, auto + suggestor and validator running at a time.) + + When the coroutine raises `_Retry`, it is restarted. + """ + running = False + + @wraps(coroutine) + async def new_coroutine(*a: Any, **kw: Any) -> Any: + nonlocal running + + # Don't start a new function, if the previous is still in progress. + if running: + return + + running = True + + try: + while True: + try: + await coroutine(*a, **kw) + except _Retry: + continue + else: + return None + finally: + running = False + + return cast(_T, new_coroutine) + + +class _Retry(Exception): + "Retry in `_only_one_at_a_time`." + + +def indent(buffer: Buffer, from_row: int, to_row: int, count: int = 1) -> None: + """ + Indent text of a :class:`.Buffer` object. + """ + current_row = buffer.document.cursor_position_row + current_col = buffer.document.cursor_position_col + line_range = range(from_row, to_row) + + # Apply transformation. + indent_content = " " * count + new_text = buffer.transform_lines(line_range, lambda l: indent_content + l) + buffer.document = Document( + new_text, Document(new_text).translate_row_col_to_index(current_row, 0) + ) + + # Place cursor in the same position in text after indenting + buffer.cursor_position += current_col + len(indent_content) + + +def unindent(buffer: Buffer, from_row: int, to_row: int, count: int = 1) -> None: + """ + Unindent text of a :class:`.Buffer` object. + """ + current_row = buffer.document.cursor_position_row + current_col = buffer.document.cursor_position_col + line_range = range(from_row, to_row) + + indent_content = " " * count + + def transform(text: str) -> str: + remove = indent_content + if text.startswith(remove): + return text[len(remove) :] + else: + return text.lstrip() + + # Apply transformation. + new_text = buffer.transform_lines(line_range, transform) + buffer.document = Document( + new_text, Document(new_text).translate_row_col_to_index(current_row, 0) + ) + + # Place cursor in the same position in text after dedent + buffer.cursor_position += current_col - len(indent_content) + + +def reshape_text(buffer: Buffer, from_row: int, to_row: int) -> None: + """ + Reformat text, taking the width into account. + `to_row` is included. + (Vi 'gq' operator.) + """ + lines = buffer.text.splitlines(True) + lines_before = lines[:from_row] + lines_after = lines[to_row + 1 :] + lines_to_reformat = lines[from_row : to_row + 1] + + if lines_to_reformat: + # Take indentation from the first line. + match = re.search(r"^\s*", lines_to_reformat[0]) + length = match.end() if match else 0 # `match` can't be None, actually. + + indent = lines_to_reformat[0][:length].replace("\n", "") + + # Now, take all the 'words' from the lines to be reshaped. + words = "".join(lines_to_reformat).split() + + # And reshape. + width = (buffer.text_width or 80) - len(indent) + reshaped_text = [indent] + current_width = 0 + for w in words: + if current_width: + if len(w) + current_width + 1 > width: + reshaped_text.append("\n") + reshaped_text.append(indent) + current_width = 0 + else: + reshaped_text.append(" ") + current_width += 1 + + reshaped_text.append(w) + current_width += len(w) + + if reshaped_text[-1] != "\n": + reshaped_text.append("\n") + + # Apply result. + buffer.document = Document( + text="".join(lines_before + reshaped_text + lines_after), + cursor_position=len("".join(lines_before + reshaped_text)), + ) diff --git a/lib/prompt_toolkit/cache.py b/lib/prompt_toolkit/cache.py new file mode 100644 index 0000000..01dd1f7 --- /dev/null +++ b/lib/prompt_toolkit/cache.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from collections import deque +from functools import wraps +from typing import Any, Callable, Dict, Generic, Hashable, Tuple, TypeVar, cast + +__all__ = [ + "SimpleCache", + "FastDictCache", + "memoized", +] + +_T = TypeVar("_T", bound=Hashable) +_U = TypeVar("_U") + + +class SimpleCache(Generic[_T, _U]): + """ + Very simple cache that discards the oldest item when the cache size is + exceeded. + + :param maxsize: Maximum size of the cache. (Don't make it too big.) + """ + + def __init__(self, maxsize: int = 8) -> None: + assert maxsize > 0 + + self._data: dict[_T, _U] = {} + self._keys: deque[_T] = deque() + self.maxsize: int = maxsize + + def get(self, key: _T, getter_func: Callable[[], _U]) -> _U: + """ + Get object from the cache. + If not found, call `getter_func` to resolve it, and put that on the top + of the cache instead. + """ + # Look in cache first. + try: + return self._data[key] + except KeyError: + # Not found? Get it. + value = getter_func() + self._data[key] = value + self._keys.append(key) + + # Remove the oldest key when the size is exceeded. + if len(self._data) > self.maxsize: + key_to_remove = self._keys.popleft() + if key_to_remove in self._data: + del self._data[key_to_remove] + + return value + + def clear(self) -> None: + "Clear cache." + self._data = {} + self._keys = deque() + + +_K = TypeVar("_K", bound=Tuple[Hashable, ...]) +_V = TypeVar("_V") + + +class FastDictCache(Dict[_K, _V]): + """ + Fast, lightweight cache which keeps at most `size` items. + It will discard the oldest items in the cache first. + + The cache is a dictionary, which doesn't keep track of access counts. + It is perfect to cache little immutable objects which are not expensive to + create, but where a dictionary lookup is still much faster than an object + instantiation. + + :param get_value: Callable that's called in case of a missing key. + """ + + # NOTE: This cache is used to cache `prompt_toolkit.layout.screen.Char` and + # `prompt_toolkit.Document`. Make sure to keep this really lightweight. + # Accessing the cache should stay faster than instantiating new + # objects. + # (Dictionary lookups are really fast.) + # SimpleCache is still required for cases where the cache key is not + # the same as the arguments given to the function that creates the + # value.) + def __init__(self, get_value: Callable[..., _V], size: int = 1000000) -> None: + assert size > 0 + + self._keys: deque[_K] = deque() + self.get_value = get_value + self.size = size + + def __missing__(self, key: _K) -> _V: + # Remove the oldest key when the size is exceeded. + if len(self) > self.size: + key_to_remove = self._keys.popleft() + if key_to_remove in self: + del self[key_to_remove] + + result = self.get_value(*key) + self[key] = result + self._keys.append(key) + return result + + +_F = TypeVar("_F", bound=Callable[..., object]) + + +def memoized(maxsize: int = 1024) -> Callable[[_F], _F]: + """ + Memoization decorator for immutable classes and pure functions. + """ + + def decorator(obj: _F) -> _F: + cache: SimpleCache[Hashable, Any] = SimpleCache(maxsize=maxsize) + + @wraps(obj) + def new_callable(*a: Any, **kw: Any) -> Any: + def create_new() -> Any: + return obj(*a, **kw) + + key = (a, tuple(sorted(kw.items()))) + return cache.get(key, create_new) + + return cast(_F, new_callable) + + return decorator diff --git a/lib/prompt_toolkit/clipboard/__init__.py b/lib/prompt_toolkit/clipboard/__init__.py new file mode 100644 index 0000000..e72f30e --- /dev/null +++ b/lib/prompt_toolkit/clipboard/__init__.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from .base import Clipboard, ClipboardData, DummyClipboard, DynamicClipboard +from .in_memory import InMemoryClipboard + +# We are not importing `PyperclipClipboard` here, because it would require the +# `pyperclip` module to be present. + +# from .pyperclip import PyperclipClipboard + +__all__ = [ + "Clipboard", + "ClipboardData", + "DummyClipboard", + "DynamicClipboard", + "InMemoryClipboard", +] diff --git a/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..f71f900 Binary files /dev/null and b/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..0dfe3a1 Binary files /dev/null and b/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc b/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc new file mode 100644 index 0000000..30b7922 Binary files /dev/null and b/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc b/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc new file mode 100644 index 0000000..1b2400c Binary files /dev/null and b/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/clipboard/base.py b/lib/prompt_toolkit/clipboard/base.py new file mode 100644 index 0000000..28cfdcd --- /dev/null +++ b/lib/prompt_toolkit/clipboard/base.py @@ -0,0 +1,109 @@ +""" +Clipboard for command line interface. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import Callable + +from prompt_toolkit.selection import SelectionType + +__all__ = [ + "Clipboard", + "ClipboardData", + "DummyClipboard", + "DynamicClipboard", +] + + +class ClipboardData: + """ + Text on the clipboard. + + :param text: string + :param type: :class:`~prompt_toolkit.selection.SelectionType` + """ + + def __init__( + self, text: str = "", type: SelectionType = SelectionType.CHARACTERS + ) -> None: + self.text = text + self.type = type + + +class Clipboard(metaclass=ABCMeta): + """ + Abstract baseclass for clipboards. + (An implementation can be in memory, it can share the X11 or Windows + keyboard, or can be persistent.) + """ + + @abstractmethod + def set_data(self, data: ClipboardData) -> None: + """ + Set data to the clipboard. + + :param data: :class:`~.ClipboardData` instance. + """ + + def set_text(self, text: str) -> None: # Not abstract. + """ + Shortcut for setting plain text on clipboard. + """ + self.set_data(ClipboardData(text)) + + def rotate(self) -> None: + """ + For Emacs mode, rotate the kill ring. + """ + + @abstractmethod + def get_data(self) -> ClipboardData: + """ + Return clipboard data. + """ + + +class DummyClipboard(Clipboard): + """ + Clipboard implementation that doesn't remember anything. + """ + + def set_data(self, data: ClipboardData) -> None: + pass + + def set_text(self, text: str) -> None: + pass + + def rotate(self) -> None: + pass + + def get_data(self) -> ClipboardData: + return ClipboardData() + + +class DynamicClipboard(Clipboard): + """ + Clipboard class that can dynamically returns any Clipboard. + + :param get_clipboard: Callable that returns a :class:`.Clipboard` instance. + """ + + def __init__(self, get_clipboard: Callable[[], Clipboard | None]) -> None: + self.get_clipboard = get_clipboard + + def _clipboard(self) -> Clipboard: + return self.get_clipboard() or DummyClipboard() + + def set_data(self, data: ClipboardData) -> None: + self._clipboard().set_data(data) + + def set_text(self, text: str) -> None: + self._clipboard().set_text(text) + + def rotate(self) -> None: + self._clipboard().rotate() + + def get_data(self) -> ClipboardData: + return self._clipboard().get_data() diff --git a/lib/prompt_toolkit/clipboard/in_memory.py b/lib/prompt_toolkit/clipboard/in_memory.py new file mode 100644 index 0000000..d9ae081 --- /dev/null +++ b/lib/prompt_toolkit/clipboard/in_memory.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections import deque + +from .base import Clipboard, ClipboardData + +__all__ = [ + "InMemoryClipboard", +] + + +class InMemoryClipboard(Clipboard): + """ + Default clipboard implementation. + Just keep the data in memory. + + This implements a kill-ring, for Emacs mode. + """ + + def __init__(self, data: ClipboardData | None = None, max_size: int = 60) -> None: + assert max_size >= 1 + + self.max_size = max_size + self._ring: deque[ClipboardData] = deque() + + if data is not None: + self.set_data(data) + + def set_data(self, data: ClipboardData) -> None: + self._ring.appendleft(data) + + while len(self._ring) > self.max_size: + self._ring.pop() + + def get_data(self) -> ClipboardData: + if self._ring: + return self._ring[0] + else: + return ClipboardData() + + def rotate(self) -> None: + if self._ring: + # Add the very first item at the end. + self._ring.append(self._ring.popleft()) diff --git a/lib/prompt_toolkit/clipboard/pyperclip.py b/lib/prompt_toolkit/clipboard/pyperclip.py new file mode 100644 index 0000000..66eb711 --- /dev/null +++ b/lib/prompt_toolkit/clipboard/pyperclip.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import pyperclip + +from prompt_toolkit.selection import SelectionType + +from .base import Clipboard, ClipboardData + +__all__ = [ + "PyperclipClipboard", +] + + +class PyperclipClipboard(Clipboard): + """ + Clipboard that synchronizes with the Windows/Mac/Linux system clipboard, + using the pyperclip module. + """ + + def __init__(self) -> None: + self._data: ClipboardData | None = None + + def set_data(self, data: ClipboardData) -> None: + self._data = data + pyperclip.copy(data.text) + + def get_data(self) -> ClipboardData: + text = pyperclip.paste() + + # When the clipboard data is equal to what we copied last time, reuse + # the `ClipboardData` instance. That way we're sure to keep the same + # `SelectionType`. + if self._data and self._data.text == text: + return self._data + + # Pyperclip returned something else. Create a new `ClipboardData` + # instance. + else: + return ClipboardData( + text=text, + type=SelectionType.LINES if "\n" in text else SelectionType.CHARACTERS, + ) diff --git a/lib/prompt_toolkit/completion/__init__.py b/lib/prompt_toolkit/completion/__init__.py new file mode 100644 index 0000000..f65a94e --- /dev/null +++ b/lib/prompt_toolkit/completion/__init__.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from .base import ( + CompleteEvent, + Completer, + Completion, + ConditionalCompleter, + DummyCompleter, + DynamicCompleter, + ThreadedCompleter, + get_common_complete_suffix, + merge_completers, +) +from .deduplicate import DeduplicateCompleter +from .filesystem import ExecutableCompleter, PathCompleter +from .fuzzy_completer import FuzzyCompleter, FuzzyWordCompleter +from .nested import NestedCompleter +from .word_completer import WordCompleter + +__all__ = [ + # Base. + "Completion", + "Completer", + "ThreadedCompleter", + "DummyCompleter", + "DynamicCompleter", + "CompleteEvent", + "ConditionalCompleter", + "merge_completers", + "get_common_complete_suffix", + # Filesystem. + "PathCompleter", + "ExecutableCompleter", + # Fuzzy + "FuzzyCompleter", + "FuzzyWordCompleter", + # Nested. + "NestedCompleter", + # Word completer. + "WordCompleter", + # Deduplicate + "DeduplicateCompleter", +] diff --git a/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..2b933a8 Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..9324327 Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc new file mode 100644 index 0000000..e1fb59a Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc new file mode 100644 index 0000000..2dc0a52 Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc new file mode 100644 index 0000000..befcfdc Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc new file mode 100644 index 0000000..be12938 Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc b/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc new file mode 100644 index 0000000..f3d7618 Binary files /dev/null and b/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/completion/base.py b/lib/prompt_toolkit/completion/base.py new file mode 100644 index 0000000..3846ef7 --- /dev/null +++ b/lib/prompt_toolkit/completion/base.py @@ -0,0 +1,438 @@ +""" """ + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import AsyncGenerator, Callable, Iterable, Sequence + +from prompt_toolkit.document import Document +from prompt_toolkit.eventloop import aclosing, generator_to_async_generator +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.formatted_text import AnyFormattedText, StyleAndTextTuples + +__all__ = [ + "Completion", + "Completer", + "ThreadedCompleter", + "DummyCompleter", + "DynamicCompleter", + "CompleteEvent", + "ConditionalCompleter", + "merge_completers", + "get_common_complete_suffix", +] + + +class Completion: + """ + :param text: The new string that will be inserted into the document. + :param start_position: Position relative to the cursor_position where the + new text will start. The text will be inserted between the + start_position and the original cursor position. + :param display: (optional string or formatted text) If the completion has + to be displayed differently in the completion menu. + :param display_meta: (Optional string or formatted text) Meta information + about the completion, e.g. the path or source where it's coming from. + This can also be a callable that returns a string. + :param style: Style string. + :param selected_style: Style string, used for a selected completion. + This can override the `style` parameter. + """ + + def __init__( + self, + text: str, + start_position: int = 0, + display: AnyFormattedText | None = None, + display_meta: AnyFormattedText | None = None, + style: str = "", + selected_style: str = "", + ) -> None: + from prompt_toolkit.formatted_text import to_formatted_text + + self.text = text + self.start_position = start_position + self._display_meta = display_meta + + if display is None: + display = text + + self.display = to_formatted_text(display) + + self.style = style + self.selected_style = selected_style + + assert self.start_position <= 0 + + def __repr__(self) -> str: + if isinstance(self.display, str) and self.display == self.text: + return f"{self.__class__.__name__}(text={self.text!r}, start_position={self.start_position!r})" + else: + return f"{self.__class__.__name__}(text={self.text!r}, start_position={self.start_position!r}, display={self.display!r})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Completion): + return False + return ( + self.text == other.text + and self.start_position == other.start_position + and self.display == other.display + and self._display_meta == other._display_meta + ) + + def __hash__(self) -> int: + return hash((self.text, self.start_position, self.display, self._display_meta)) + + @property + def display_text(self) -> str: + "The 'display' field as plain text." + from prompt_toolkit.formatted_text import fragment_list_to_text + + return fragment_list_to_text(self.display) + + @property + def display_meta(self) -> StyleAndTextTuples: + "Return meta-text. (This is lazy when using a callable)." + from prompt_toolkit.formatted_text import to_formatted_text + + return to_formatted_text(self._display_meta or "") + + @property + def display_meta_text(self) -> str: + "The 'meta' field as plain text." + from prompt_toolkit.formatted_text import fragment_list_to_text + + return fragment_list_to_text(self.display_meta) + + def new_completion_from_position(self, position: int) -> Completion: + """ + (Only for internal use!) + Get a new completion by splitting this one. Used by `Application` when + it needs to have a list of new completions after inserting the common + prefix. + """ + assert position - self.start_position >= 0 + + return Completion( + text=self.text[position - self.start_position :], + display=self.display, + display_meta=self._display_meta, + ) + + +class CompleteEvent: + """ + Event that called the completer. + + :param text_inserted: When True, it means that completions are requested + because of a text insert. (`Buffer.complete_while_typing`.) + :param completion_requested: When True, it means that the user explicitly + pressed the `Tab` key in order to view the completions. + + These two flags can be used for instance to implement a completer that + shows some completions when ``Tab`` has been pressed, but not + automatically when the user presses a space. (Because of + `complete_while_typing`.) + """ + + def __init__( + self, text_inserted: bool = False, completion_requested: bool = False + ) -> None: + assert not (text_inserted and completion_requested) + + #: Automatic completion while typing. + self.text_inserted = text_inserted + + #: Used explicitly requested completion by pressing 'tab'. + self.completion_requested = completion_requested + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(text_inserted={self.text_inserted!r}, completion_requested={self.completion_requested!r})" + + +class Completer(metaclass=ABCMeta): + """ + Base class for completer implementations. + """ + + @abstractmethod + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + """ + This should be a generator that yields :class:`.Completion` instances. + + If the generation of completions is something expensive (that takes a + lot of time), consider wrapping this `Completer` class in a + `ThreadedCompleter`. In that case, the completer algorithm runs in a + background thread and completions will be displayed as soon as they + arrive. + + :param document: :class:`~prompt_toolkit.document.Document` instance. + :param complete_event: :class:`.CompleteEvent` instance. + """ + while False: + yield + + async def get_completions_async( + self, document: Document, complete_event: CompleteEvent + ) -> AsyncGenerator[Completion, None]: + """ + Asynchronous generator for completions. (Probably, you won't have to + override this.) + + Asynchronous generator of :class:`.Completion` objects. + """ + for item in self.get_completions(document, complete_event): + yield item + + +class ThreadedCompleter(Completer): + """ + Wrapper that runs the `get_completions` generator in a thread. + + (Use this to prevent the user interface from becoming unresponsive if the + generation of completions takes too much time.) + + The completions will be displayed as soon as they are produced. The user + can already select a completion, even if not all completions are displayed. + """ + + def __init__(self, completer: Completer) -> None: + self.completer = completer + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + return self.completer.get_completions(document, complete_event) + + async def get_completions_async( + self, document: Document, complete_event: CompleteEvent + ) -> AsyncGenerator[Completion, None]: + """ + Asynchronous generator of completions. + """ + # NOTE: Right now, we are consuming the `get_completions` generator in + # a synchronous background thread, then passing the results one + # at a time over a queue, and consuming this queue in the main + # thread (that's what `generator_to_async_generator` does). That + # means that if the completer is *very* slow, we'll be showing + # completions in the UI once they are computed. + + # It's very tempting to replace this implementation with the + # commented code below for several reasons: + + # - `generator_to_async_generator` is not perfect and hard to get + # right. It's a lot of complexity for little gain. The + # implementation needs a huge buffer for it to be efficient + # when there are many completions (like 50k+). + # - Normally, a completer is supposed to be fast, users can have + # "complete while typing" enabled, and want to see the + # completions within a second. Handling one completion at a + # time, and rendering once we get it here doesn't make any + # sense if this is quick anyway. + # - Completers like `FuzzyCompleter` prepare all completions + # anyway so that they can be sorted by accuracy before they are + # yielded. At the point that we start yielding completions + # here, we already have all completions. + # - The `Buffer` class has complex logic to invalidate the UI + # while it is consuming the completions. We don't want to + # invalidate the UI for every completion (if there are many), + # but we want to do it often enough so that completions are + # being displayed while they are produced. + + # We keep the current behavior mainly for backward-compatibility. + # Similarly, it would be better for this function to not return + # an async generator, but simply be a coroutine that returns a + # list of `Completion` objects, containing all completions at + # once. + + # Note that this argument doesn't mean we shouldn't use + # `ThreadedCompleter`. It still makes sense to produce + # completions in a background thread, because we don't want to + # freeze the UI while the user is typing. But sending the + # completions one at a time to the UI maybe isn't worth it. + + # def get_all_in_thread() -> List[Completion]: + # return list(self.get_completions(document, complete_event)) + + # completions = await get_running_loop().run_in_executor(None, get_all_in_thread) + # for completion in completions: + # yield completion + + async with aclosing( + generator_to_async_generator( + lambda: self.completer.get_completions(document, complete_event) + ) + ) as async_generator: + async for completion in async_generator: + yield completion + + def __repr__(self) -> str: + return f"ThreadedCompleter({self.completer!r})" + + +class DummyCompleter(Completer): + """ + A completer that doesn't return any completion. + """ + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + return [] + + def __repr__(self) -> str: + return "DummyCompleter()" + + +class DynamicCompleter(Completer): + """ + Completer class that can dynamically returns any Completer. + + :param get_completer: Callable that returns a :class:`.Completer` instance. + """ + + def __init__(self, get_completer: Callable[[], Completer | None]) -> None: + self.get_completer = get_completer + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + completer = self.get_completer() or DummyCompleter() + return completer.get_completions(document, complete_event) + + async def get_completions_async( + self, document: Document, complete_event: CompleteEvent + ) -> AsyncGenerator[Completion, None]: + completer = self.get_completer() or DummyCompleter() + + async for completion in completer.get_completions_async( + document, complete_event + ): + yield completion + + def __repr__(self) -> str: + return f"DynamicCompleter({self.get_completer!r} -> {self.get_completer()!r})" + + +class ConditionalCompleter(Completer): + """ + Wrapper around any other completer that will enable/disable the completions + depending on whether the received condition is satisfied. + + :param completer: :class:`.Completer` instance. + :param filter: :class:`.Filter` instance. + """ + + def __init__(self, completer: Completer, filter: FilterOrBool) -> None: + self.completer = completer + self.filter = to_filter(filter) + + def __repr__(self) -> str: + return f"ConditionalCompleter({self.completer!r}, filter={self.filter!r})" + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + # Get all completions in a blocking way. + if self.filter(): + yield from self.completer.get_completions(document, complete_event) + + async def get_completions_async( + self, document: Document, complete_event: CompleteEvent + ) -> AsyncGenerator[Completion, None]: + # Get all completions in a non-blocking way. + if self.filter(): + async with aclosing( + self.completer.get_completions_async(document, complete_event) + ) as async_generator: + async for item in async_generator: + yield item + + +class _MergedCompleter(Completer): + """ + Combine several completers into one. + """ + + def __init__(self, completers: Sequence[Completer]) -> None: + self.completers = completers + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + # Get all completions from the other completers in a blocking way. + for completer in self.completers: + yield from completer.get_completions(document, complete_event) + + async def get_completions_async( + self, document: Document, complete_event: CompleteEvent + ) -> AsyncGenerator[Completion, None]: + # Get all completions from the other completers in a non-blocking way. + for completer in self.completers: + async with aclosing( + completer.get_completions_async(document, complete_event) + ) as async_generator: + async for item in async_generator: + yield item + + +def merge_completers( + completers: Sequence[Completer], deduplicate: bool = False +) -> Completer: + """ + Combine several completers into one. + + :param deduplicate: If `True`, wrap the result in a `DeduplicateCompleter` + so that completions that would result in the same text will be + deduplicated. + """ + if deduplicate: + from .deduplicate import DeduplicateCompleter + + return DeduplicateCompleter(_MergedCompleter(completers)) + + return _MergedCompleter(completers) + + +def get_common_complete_suffix( + document: Document, completions: Sequence[Completion] +) -> str: + """ + Return the common prefix for all completions. + """ + + # Take only completions that don't change the text before the cursor. + def doesnt_change_before_cursor(completion: Completion) -> bool: + end = completion.text[: -completion.start_position] + return document.text_before_cursor.endswith(end) + + completions2 = [c for c in completions if doesnt_change_before_cursor(c)] + + # When there is at least one completion that changes the text before the + # cursor, don't return any common part. + if len(completions2) != len(completions): + return "" + + # Return the common prefix. + def get_suffix(completion: Completion) -> str: + return completion.text[-completion.start_position :] + + return _commonprefix([get_suffix(c) for c in completions2]) + + +def _commonprefix(strings: Iterable[str]) -> str: + # Similar to os.path.commonprefix + if not strings: + return "" + + else: + s1 = min(strings) + s2 = max(strings) + + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + + return s1 diff --git a/lib/prompt_toolkit/completion/deduplicate.py b/lib/prompt_toolkit/completion/deduplicate.py new file mode 100644 index 0000000..c3d5256 --- /dev/null +++ b/lib/prompt_toolkit/completion/deduplicate.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import Iterable + +from prompt_toolkit.document import Document + +from .base import CompleteEvent, Completer, Completion + +__all__ = ["DeduplicateCompleter"] + + +class DeduplicateCompleter(Completer): + """ + Wrapper around a completer that removes duplicates. Only the first unique + completions are kept. + + Completions are considered to be a duplicate if they result in the same + document text when they would be applied. + """ + + def __init__(self, completer: Completer) -> None: + self.completer = completer + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + # Keep track of the document strings we'd get after applying any completion. + found_so_far: set[str] = set() + + for completion in self.completer.get_completions(document, complete_event): + text_if_applied = ( + document.text[: document.cursor_position + completion.start_position] + + completion.text + + document.text[document.cursor_position :] + ) + + if text_if_applied == document.text: + # Don't include completions that don't have any effect at all. + continue + + if text_if_applied in found_so_far: + continue + + found_so_far.add(text_if_applied) + yield completion diff --git a/lib/prompt_toolkit/completion/filesystem.py b/lib/prompt_toolkit/completion/filesystem.py new file mode 100644 index 0000000..8e7f87e --- /dev/null +++ b/lib/prompt_toolkit/completion/filesystem.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import os +from typing import Callable, Iterable + +from prompt_toolkit.completion import CompleteEvent, Completer, Completion +from prompt_toolkit.document import Document + +__all__ = [ + "PathCompleter", + "ExecutableCompleter", +] + + +class PathCompleter(Completer): + """ + Complete for Path variables. + + :param get_paths: Callable which returns a list of directories to look into + when the user enters a relative path. + :param file_filter: Callable which takes a filename and returns whether + this file should show up in the completion. ``None`` + when no filtering has to be done. + :param min_input_len: Don't do autocompletion when the input string is shorter. + """ + + def __init__( + self, + only_directories: bool = False, + get_paths: Callable[[], list[str]] | None = None, + file_filter: Callable[[str], bool] | None = None, + min_input_len: int = 0, + expanduser: bool = False, + ) -> None: + self.only_directories = only_directories + self.get_paths = get_paths or (lambda: ["."]) + self.file_filter = file_filter or (lambda _: True) + self.min_input_len = min_input_len + self.expanduser = expanduser + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + text = document.text_before_cursor + + # Complete only when we have at least the minimal input length, + # otherwise, we can too many results and autocompletion will become too + # heavy. + if len(text) < self.min_input_len: + return + + try: + # Do tilde expansion. + if self.expanduser: + text = os.path.expanduser(text) + + # Directories where to look. + dirname = os.path.dirname(text) + if dirname: + directories = [ + os.path.dirname(os.path.join(p, text)) for p in self.get_paths() + ] + else: + directories = self.get_paths() + + # Start of current file. + prefix = os.path.basename(text) + + # Get all filenames. + filenames = [] + for directory in directories: + # Look for matches in this directory. + if os.path.isdir(directory): + for filename in os.listdir(directory): + if filename.startswith(prefix): + filenames.append((directory, filename)) + + # Sort + filenames = sorted(filenames, key=lambda k: k[1]) + + # Yield them. + for directory, filename in filenames: + completion = filename[len(prefix) :] + full_name = os.path.join(directory, filename) + + if os.path.isdir(full_name): + # For directories, add a slash to the filename. + # (We don't add them to the `completion`. Users can type it + # to trigger the autocompletion themselves.) + filename += "/" + elif self.only_directories: + continue + + if not self.file_filter(full_name): + continue + + yield Completion( + text=completion, + start_position=0, + display=filename, + ) + except OSError: + pass + + +class ExecutableCompleter(PathCompleter): + """ + Complete only executable files in the current path. + """ + + def __init__(self) -> None: + super().__init__( + only_directories=False, + min_input_len=1, + get_paths=lambda: os.environ.get("PATH", "").split(os.pathsep), + file_filter=lambda name: os.access(name, os.X_OK), + expanduser=True, + ) diff --git a/lib/prompt_toolkit/completion/fuzzy_completer.py b/lib/prompt_toolkit/completion/fuzzy_completer.py new file mode 100644 index 0000000..82625ab --- /dev/null +++ b/lib/prompt_toolkit/completion/fuzzy_completer.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import re +from typing import Callable, Iterable, NamedTuple, Sequence + +from prompt_toolkit.document import Document +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.formatted_text import AnyFormattedText, StyleAndTextTuples + +from .base import CompleteEvent, Completer, Completion +from .word_completer import WordCompleter + +__all__ = [ + "FuzzyCompleter", + "FuzzyWordCompleter", +] + + +class FuzzyCompleter(Completer): + """ + Fuzzy completion. + This wraps any other completer and turns it into a fuzzy completer. + + If the list of words is: ["leopard" , "gorilla", "dinosaur", "cat", "bee"] + Then trying to complete "oar" would yield "leopard" and "dinosaur", but not + the others, because they match the regular expression 'o.*a.*r'. + Similar, in another application "djm" could expand to "django_migrations". + + The results are sorted by relevance, which is defined as the start position + and the length of the match. + + Notice that this is not really a tool to work around spelling mistakes, + like what would be possible with difflib. The purpose is rather to have a + quicker or more intuitive way to filter the given completions, especially + when many completions have a common prefix. + + Fuzzy algorithm is based on this post: + https://blog.amjith.com/fuzzyfinder-in-10-lines-of-python + + :param completer: A :class:`~.Completer` instance. + :param WORD: When True, use WORD characters. + :param pattern: Regex pattern which selects the characters before the + cursor that are considered for the fuzzy matching. + :param enable_fuzzy: (bool or `Filter`) Enabled the fuzzy behavior. For + easily turning fuzzyness on or off according to a certain condition. + """ + + def __init__( + self, + completer: Completer, + WORD: bool = False, + pattern: str | None = None, + enable_fuzzy: FilterOrBool = True, + ) -> None: + assert pattern is None or pattern.startswith("^") + + self.completer = completer + self.pattern = pattern + self.WORD = WORD + self.pattern = pattern + self.enable_fuzzy = to_filter(enable_fuzzy) + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + if self.enable_fuzzy(): + return self._get_fuzzy_completions(document, complete_event) + else: + return self.completer.get_completions(document, complete_event) + + def _get_pattern(self) -> str: + if self.pattern: + return self.pattern + if self.WORD: + return r"[^\s]+" + return "^[a-zA-Z0-9_]*" + + def _get_fuzzy_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + word_before_cursor = document.get_word_before_cursor( + pattern=re.compile(self._get_pattern()) + ) + + # Get completions + document2 = Document( + text=document.text[: document.cursor_position - len(word_before_cursor)], + cursor_position=document.cursor_position - len(word_before_cursor), + ) + + inner_completions = list( + self.completer.get_completions(document2, complete_event) + ) + + fuzzy_matches: list[_FuzzyMatch] = [] + + if word_before_cursor == "": + # If word before the cursor is an empty string, consider all + # completions, without filtering everything with an empty regex + # pattern. + fuzzy_matches = [_FuzzyMatch(0, 0, compl) for compl in inner_completions] + else: + pat = ".*?".join(map(re.escape, word_before_cursor)) + pat = f"(?=({pat}))" # lookahead regex to manage overlapping matches + regex = re.compile(pat, re.IGNORECASE) + for compl in inner_completions: + matches = list(regex.finditer(compl.text)) + if matches: + # Prefer the match, closest to the left, then shortest. + best = min(matches, key=lambda m: (m.start(), len(m.group(1)))) + fuzzy_matches.append( + _FuzzyMatch(len(best.group(1)), best.start(), compl) + ) + + def sort_key(fuzzy_match: _FuzzyMatch) -> tuple[int, int]: + "Sort by start position, then by the length of the match." + return fuzzy_match.start_pos, fuzzy_match.match_length + + fuzzy_matches = sorted(fuzzy_matches, key=sort_key) + + for match in fuzzy_matches: + # Include these completions, but set the correct `display` + # attribute and `start_position`. + yield Completion( + text=match.completion.text, + start_position=match.completion.start_position + - len(word_before_cursor), + # We access to private `_display_meta` attribute, because that one is lazy. + display_meta=match.completion._display_meta, + display=self._get_display(match, word_before_cursor), + style=match.completion.style, + ) + + def _get_display( + self, fuzzy_match: _FuzzyMatch, word_before_cursor: str + ) -> AnyFormattedText: + """ + Generate formatted text for the display label. + """ + + def get_display() -> AnyFormattedText: + m = fuzzy_match + word = m.completion.text + + if m.match_length == 0: + # No highlighting when we have zero length matches (no input text). + # In this case, use the original display text (which can include + # additional styling or characters). + return m.completion.display + + result: StyleAndTextTuples = [] + + # Text before match. + result.append(("class:fuzzymatch.outside", word[: m.start_pos])) + + # The match itself. + characters = list(word_before_cursor) + + for c in word[m.start_pos : m.start_pos + m.match_length]: + classname = "class:fuzzymatch.inside" + if characters and c.lower() == characters[0].lower(): + classname += ".character" + del characters[0] + + result.append((classname, c)) + + # Text after match. + result.append( + ("class:fuzzymatch.outside", word[m.start_pos + m.match_length :]) + ) + + return result + + return get_display() + + +class FuzzyWordCompleter(Completer): + """ + Fuzzy completion on a list of words. + + (This is basically a `WordCompleter` wrapped in a `FuzzyCompleter`.) + + :param words: List of words or callable that returns a list of words. + :param meta_dict: Optional dict mapping words to their meta-information. + :param WORD: When True, use WORD characters. + """ + + def __init__( + self, + words: Sequence[str] | Callable[[], Sequence[str]], + meta_dict: dict[str, str] | None = None, + WORD: bool = False, + ) -> None: + self.words = words + self.meta_dict = meta_dict or {} + self.WORD = WORD + + self.word_completer = WordCompleter( + words=self.words, WORD=self.WORD, meta_dict=self.meta_dict + ) + + self.fuzzy_completer = FuzzyCompleter(self.word_completer, WORD=self.WORD) + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + return self.fuzzy_completer.get_completions(document, complete_event) + + +class _FuzzyMatch(NamedTuple): + match_length: int + start_pos: int + completion: Completion diff --git a/lib/prompt_toolkit/completion/nested.py b/lib/prompt_toolkit/completion/nested.py new file mode 100644 index 0000000..b72b69e --- /dev/null +++ b/lib/prompt_toolkit/completion/nested.py @@ -0,0 +1,109 @@ +""" +Nestedcompleter for completion of hierarchical data structures. +""" + +from __future__ import annotations + +from typing import Any, Iterable, Mapping, Set, Union + +from prompt_toolkit.completion import CompleteEvent, Completer, Completion +from prompt_toolkit.completion.word_completer import WordCompleter +from prompt_toolkit.document import Document + +__all__ = ["NestedCompleter"] + +# NestedDict = Mapping[str, Union['NestedDict', Set[str], None, Completer]] +NestedDict = Mapping[str, Union[Any, Set[str], None, Completer]] + + +class NestedCompleter(Completer): + """ + Completer which wraps around several other completers, and calls any the + one that corresponds with the first word of the input. + + By combining multiple `NestedCompleter` instances, we can achieve multiple + hierarchical levels of autocompletion. This is useful when `WordCompleter` + is not sufficient. + + If you need multiple levels, check out the `from_nested_dict` classmethod. + """ + + def __init__( + self, options: dict[str, Completer | None], ignore_case: bool = True + ) -> None: + self.options = options + self.ignore_case = ignore_case + + def __repr__(self) -> str: + return f"NestedCompleter({self.options!r}, ignore_case={self.ignore_case!r})" + + @classmethod + def from_nested_dict(cls, data: NestedDict) -> NestedCompleter: + """ + Create a `NestedCompleter`, starting from a nested dictionary data + structure, like this: + + .. code:: + + data = { + 'show': { + 'version': None, + 'interfaces': None, + 'clock': None, + 'ip': {'interface': {'brief'}} + }, + 'exit': None + 'enable': None + } + + The value should be `None` if there is no further completion at some + point. If all values in the dictionary are None, it is also possible to + use a set instead. + + Values in this data structure can be a completers as well. + """ + options: dict[str, Completer | None] = {} + for key, value in data.items(): + if isinstance(value, Completer): + options[key] = value + elif isinstance(value, dict): + options[key] = cls.from_nested_dict(value) + elif isinstance(value, set): + options[key] = cls.from_nested_dict(dict.fromkeys(value)) + else: + assert value is None + options[key] = None + + return cls(options) + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + # Split document. + text = document.text_before_cursor.lstrip() + stripped_len = len(document.text_before_cursor) - len(text) + + # If there is a space, check for the first term, and use a + # subcompleter. + if " " in text: + first_term = text.split()[0] + completer = self.options.get(first_term) + + # If we have a sub completer, use this for the completions. + if completer is not None: + remaining_text = text[len(first_term) :].lstrip() + move_cursor = len(text) - len(remaining_text) + stripped_len + + new_document = Document( + remaining_text, + cursor_position=document.cursor_position - move_cursor, + ) + + yield from completer.get_completions(new_document, complete_event) + + # No space in the input: behave exactly like `WordCompleter`. + else: + completer = WordCompleter( + list(self.options.keys()), ignore_case=self.ignore_case + ) + yield from completer.get_completions(document, complete_event) diff --git a/lib/prompt_toolkit/completion/word_completer.py b/lib/prompt_toolkit/completion/word_completer.py new file mode 100644 index 0000000..2e12405 --- /dev/null +++ b/lib/prompt_toolkit/completion/word_completer.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Callable, Iterable, Mapping, Pattern, Sequence + +from prompt_toolkit.completion import CompleteEvent, Completer, Completion +from prompt_toolkit.document import Document +from prompt_toolkit.formatted_text import AnyFormattedText + +__all__ = [ + "WordCompleter", +] + + +class WordCompleter(Completer): + """ + Simple autocompletion on a list of words. + + :param words: List of words or callable that returns a list of words. + :param ignore_case: If True, case-insensitive completion. + :param meta_dict: Optional dict mapping words to their meta-text. (This + should map strings to strings or formatted text.) + :param WORD: When True, use WORD characters. + :param sentence: When True, don't complete by comparing the word before the + cursor, but by comparing all the text before the cursor. In this case, + the list of words is just a list of strings, where each string can + contain spaces. (Can not be used together with the WORD option.) + :param match_middle: When True, match not only the start, but also in the + middle of the word. + :param pattern: Optional compiled regex for finding the word before + the cursor to complete. When given, use this regex pattern instead of + default one (see document._FIND_WORD_RE) + """ + + def __init__( + self, + words: Sequence[str] | Callable[[], Sequence[str]], + ignore_case: bool = False, + display_dict: Mapping[str, AnyFormattedText] | None = None, + meta_dict: Mapping[str, AnyFormattedText] | None = None, + WORD: bool = False, + sentence: bool = False, + match_middle: bool = False, + pattern: Pattern[str] | None = None, + ) -> None: + assert not (WORD and sentence) + + self.words = words + self.ignore_case = ignore_case + self.display_dict = display_dict or {} + self.meta_dict = meta_dict or {} + self.WORD = WORD + self.sentence = sentence + self.match_middle = match_middle + self.pattern = pattern + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + # Get list of words. + words = self.words + if callable(words): + words = words() + + # Get word/text before cursor. + if self.sentence: + word_before_cursor = document.text_before_cursor + else: + word_before_cursor = document.get_word_before_cursor( + WORD=self.WORD, pattern=self.pattern + ) + + if self.ignore_case: + word_before_cursor = word_before_cursor.lower() + + def word_matches(word: str) -> bool: + """True when the word before the cursor matches.""" + if self.ignore_case: + word = word.lower() + + if self.match_middle: + return word_before_cursor in word + else: + return word.startswith(word_before_cursor) + + for a in words: + if word_matches(a): + display = self.display_dict.get(a, a) + display_meta = self.meta_dict.get(a, "") + yield Completion( + text=a, + start_position=-len(word_before_cursor), + display=display, + display_meta=display_meta, + ) diff --git a/lib/prompt_toolkit/contrib/__init__.py b/lib/prompt_toolkit/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..e6660e8 Binary files /dev/null and b/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/completers/__init__.py b/lib/prompt_toolkit/contrib/completers/__init__.py new file mode 100644 index 0000000..172fe6f --- /dev/null +++ b/lib/prompt_toolkit/contrib/completers/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from .system import SystemCompleter + +__all__ = ["SystemCompleter"] diff --git a/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..7f68406 Binary files /dev/null and b/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc b/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc new file mode 100644 index 0000000..5225462 Binary files /dev/null and b/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/completers/system.py b/lib/prompt_toolkit/contrib/completers/system.py new file mode 100644 index 0000000..5d990e5 --- /dev/null +++ b/lib/prompt_toolkit/contrib/completers/system.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from prompt_toolkit.completion.filesystem import ExecutableCompleter, PathCompleter +from prompt_toolkit.contrib.regular_languages.compiler import compile +from prompt_toolkit.contrib.regular_languages.completion import GrammarCompleter + +__all__ = [ + "SystemCompleter", +] + + +class SystemCompleter(GrammarCompleter): + """ + Completer for system commands. + """ + + def __init__(self) -> None: + # Compile grammar. + g = compile( + r""" + # First we have an executable. + (?P[^\s]+) + + # Ignore literals in between. + ( + \s+ + ("[^"]*" | '[^']*' | [^'"]+ ) + )* + + \s+ + + # Filename as parameters. + ( + (?P[^\s]+) | + "(?P[^\s]+)" | + '(?P[^\s]+)' + ) + """, + escape_funcs={ + "double_quoted_filename": (lambda string: string.replace('"', '\\"')), + "single_quoted_filename": (lambda string: string.replace("'", "\\'")), + }, + unescape_funcs={ + "double_quoted_filename": ( + lambda string: string.replace('\\"', '"') + ), # XXX: not entirely correct. + "single_quoted_filename": (lambda string: string.replace("\\'", "'")), + }, + ) + + # Create GrammarCompleter + super().__init__( + g, + { + "executable": ExecutableCompleter(), + "filename": PathCompleter(only_directories=False, expanduser=True), + "double_quoted_filename": PathCompleter( + only_directories=False, expanduser=True + ), + "single_quoted_filename": PathCompleter( + only_directories=False, expanduser=True + ), + }, + ) diff --git a/lib/prompt_toolkit/contrib/regular_languages/__init__.py b/lib/prompt_toolkit/contrib/regular_languages/__init__.py new file mode 100644 index 0000000..38b027c --- /dev/null +++ b/lib/prompt_toolkit/contrib/regular_languages/__init__.py @@ -0,0 +1,80 @@ +r""" +Tool for expressing the grammar of an input as a regular language. +================================================================== + +The grammar for the input of many simple command line interfaces can be +expressed by a regular language. Examples are PDB (the Python debugger); a +simple (bash-like) shell with "pwd", "cd", "cat" and "ls" commands; arguments +that you can pass to an executable; etc. It is possible to use regular +expressions for validation and parsing of such a grammar. (More about regular +languages: http://en.wikipedia.org/wiki/Regular_language) + +Example +------- + +Let's take the pwd/cd/cat/ls example. We want to have a shell that accepts +these three commands. "cd" is followed by a quoted directory name and "cat" is +followed by a quoted file name. (We allow quotes inside the filename when +they're escaped with a backslash.) We could define the grammar using the +following regular expression:: + + grammar = \s* ( + pwd | + ls | + (cd \s+ " ([^"]|\.)+ ") | + (cat \s+ " ([^"]|\.)+ ") + ) \s* + + +What can we do with this grammar? +--------------------------------- + +- Syntax highlighting: We could use this for instance to give file names + different color. +- Parse the result: .. We can extract the file names and commands by using a + regular expression with named groups. +- Input validation: .. Don't accept anything that does not match this grammar. + When combined with a parser, we can also recursively do + filename validation (and accept only existing files.) +- Autocompletion: .... Each part of the grammar can have its own autocompleter. + "cat" has to be completed using file names, while "cd" + has to be completed using directory names. + +How does it work? +----------------- + +As a user of this library, you have to define the grammar of the input as a +regular expression. The parts of this grammar where autocompletion, validation +or any other processing is required need to be marked using a regex named +group. Like ``(?P...)`` for instance. + +When the input is processed for validation (for instance), the regex will +execute, the named group is captured, and the validator associated with this +named group will test the captured string. + +There is one tricky bit: + + Often we operate on incomplete input (this is by definition the case for + autocompletion) and we have to decide for the cursor position in which + possible state the grammar it could be and in which way variables could be + matched up to that point. + +To solve this problem, the compiler takes the original regular expression and +translates it into a set of other regular expressions which each match certain +prefixes of the original regular expression. We generate one prefix regular +expression for every named variable (with this variable being the end of that +expression). + + +TODO: some examples of: + - How to create a highlighter from this grammar. + - How to create a validator from this grammar. + - How to create an autocompleter from this grammar. + - How to create a parser from this grammar. +""" + +from __future__ import annotations + +from .compiler import compile + +__all__ = ["compile"] diff --git a/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..8432b29 Binary files /dev/null and b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc new file mode 100644 index 0000000..d40b544 Binary files /dev/null and b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc new file mode 100644 index 0000000..3b737ce Binary files /dev/null and b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc new file mode 100644 index 0000000..7556fd5 Binary files /dev/null and b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc new file mode 100644 index 0000000..cf88d4f Binary files /dev/null and b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc new file mode 100644 index 0000000..4f183d9 Binary files /dev/null and b/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/regular_languages/compiler.py b/lib/prompt_toolkit/contrib/regular_languages/compiler.py new file mode 100644 index 0000000..4009d54 --- /dev/null +++ b/lib/prompt_toolkit/contrib/regular_languages/compiler.py @@ -0,0 +1,579 @@ +r""" +Compiler for a regular grammar. + +Example usage:: + + # Create and compile grammar. + p = compile('add \s+ (?P[^\s]+) \s+ (?P[^\s]+)') + + # Match input string. + m = p.match('add 23 432') + + # Get variables. + m.variables().get('var1') # Returns "23" + m.variables().get('var2') # Returns "432" + + +Partial matches are possible:: + + # Create and compile grammar. + p = compile(''' + # Operators with two arguments. + ((?P[^\s]+) \s+ (?P[^\s]+) \s+ (?P[^\s]+)) | + + # Operators with only one arguments. + ((?P[^\s]+) \s+ (?P[^\s]+)) + ''') + + # Match partial input string. + m = p.match_prefix('add 23') + + # Get variables. (Notice that both operator1 and operator2 contain the + # value "add".) This is because our input is incomplete, and we don't know + # yet in which rule of the regex we we'll end up. It could also be that + # `operator1` and `operator2` have a different autocompleter and we want to + # call all possible autocompleters that would result in valid input.) + m.variables().get('var1') # Returns "23" + m.variables().get('operator1') # Returns "add" + m.variables().get('operator2') # Returns "add" + +""" + +from __future__ import annotations + +import re +from typing import Callable, Dict, Iterable, Iterator, Pattern, TypeVar, overload +from typing import Match as RegexMatch + +from .regex_parser import ( + AnyNode, + Lookahead, + Node, + NodeSequence, + Regex, + Repeat, + Variable, + parse_regex, + tokenize_regex, +) + +__all__ = ["compile", "Match", "Variables"] + + +# Name of the named group in the regex, matching trailing input. +# (Trailing input is when the input contains characters after the end of the +# expression has been matched.) +_INVALID_TRAILING_INPUT = "invalid_trailing" + +EscapeFuncDict = Dict[str, Callable[[str], str]] + + +class _CompiledGrammar: + """ + Compiles a grammar. This will take the parse tree of a regular expression + and compile the grammar. + + :param root_node: :class~`.regex_parser.Node` instance. + :param escape_funcs: `dict` mapping variable names to escape callables. + :param unescape_funcs: `dict` mapping variable names to unescape callables. + """ + + def __init__( + self, + root_node: Node, + escape_funcs: EscapeFuncDict | None = None, + unescape_funcs: EscapeFuncDict | None = None, + ) -> None: + self.root_node = root_node + self.escape_funcs = escape_funcs or {} + self.unescape_funcs = unescape_funcs or {} + + #: Dictionary that will map the regex names to Node instances. + self._group_names_to_nodes: dict[ + str, str + ] = {} # Maps regex group names to varnames. + counter = [0] + + def create_group_func(node: Variable) -> str: + name = f"n{counter[0]}" + self._group_names_to_nodes[name] = node.varname + counter[0] += 1 + return name + + # Compile regex strings. + self._re_pattern = f"^{self._transform(root_node, create_group_func)}$" + self._re_prefix_patterns = list( + self._transform_prefix(root_node, create_group_func) + ) + + # Compile the regex itself. + flags = re.DOTALL # Note that we don't need re.MULTILINE! (^ and $ + # still represent the start and end of input text.) + self._re = re.compile(self._re_pattern, flags) + self._re_prefix = [re.compile(t, flags) for t in self._re_prefix_patterns] + + # We compile one more set of regexes, similar to `_re_prefix`, but accept any trailing + # input. This will ensure that we can still highlight the input correctly, even when the + # input contains some additional characters at the end that don't match the grammar.) + self._re_prefix_with_trailing_input = [ + re.compile( + r"(?:{})(?P<{}>.*?)$".format(t.rstrip("$"), _INVALID_TRAILING_INPUT), + flags, + ) + for t in self._re_prefix_patterns + ] + + def escape(self, varname: str, value: str) -> str: + """ + Escape `value` to fit in the place of this variable into the grammar. + """ + f = self.escape_funcs.get(varname) + return f(value) if f else value + + def unescape(self, varname: str, value: str) -> str: + """ + Unescape `value`. + """ + f = self.unescape_funcs.get(varname) + return f(value) if f else value + + @classmethod + def _transform( + cls, root_node: Node, create_group_func: Callable[[Variable], str] + ) -> str: + """ + Turn a :class:`Node` object into a regular expression. + + :param root_node: The :class:`Node` instance for which we generate the grammar. + :param create_group_func: A callable which takes a `Node` and returns the next + free name for this node. + """ + + def transform(node: Node) -> str: + # Turn `AnyNode` into an OR. + if isinstance(node, AnyNode): + return "(?:{})".format("|".join(transform(c) for c in node.children)) + + # Concatenate a `NodeSequence` + elif isinstance(node, NodeSequence): + return "".join(transform(c) for c in node.children) + + # For Regex and Lookahead nodes, just insert them literally. + elif isinstance(node, Regex): + return node.regex + + elif isinstance(node, Lookahead): + before = "(?!" if node.negative else "(=" + return before + transform(node.childnode) + ")" + + # A `Variable` wraps the children into a named group. + elif isinstance(node, Variable): + return f"(?P<{create_group_func(node)}>{transform(node.childnode)})" + + # `Repeat`. + elif isinstance(node, Repeat): + if node.max_repeat is None: + if node.min_repeat == 0: + repeat_sign = "*" + elif node.min_repeat == 1: + repeat_sign = "+" + else: + repeat_sign = "{%i,%s}" % ( + node.min_repeat, + ("" if node.max_repeat is None else str(node.max_repeat)), + ) + + return "(?:{}){}{}".format( + transform(node.childnode), + repeat_sign, + ("" if node.greedy else "?"), + ) + else: + raise TypeError(f"Got {node!r}") + + return transform(root_node) + + @classmethod + def _transform_prefix( + cls, root_node: Node, create_group_func: Callable[[Variable], str] + ) -> Iterable[str]: + """ + Yield all the regular expressions matching a prefix of the grammar + defined by the `Node` instance. + + For each `Variable`, one regex pattern will be generated, with this + named group at the end. This is required because a regex engine will + terminate once a match is found. For autocompletion however, we need + the matches for all possible paths, so that we can provide completions + for each `Variable`. + + - So, in the case of an `Any` (`A|B|C)', we generate a pattern for each + clause. This is one for `A`, one for `B` and one for `C`. Unless some + groups don't contain a `Variable`, then these can be merged together. + - In the case of a `NodeSequence` (`ABC`), we generate a pattern for + each prefix that ends with a variable, and one pattern for the whole + sequence. So, that's one for `A`, one for `AB` and one for `ABC`. + + :param root_node: The :class:`Node` instance for which we generate the grammar. + :param create_group_func: A callable which takes a `Node` and returns the next + free name for this node. + """ + + def contains_variable(node: Node) -> bool: + if isinstance(node, Regex): + return False + elif isinstance(node, Variable): + return True + elif isinstance(node, (Lookahead, Repeat)): + return contains_variable(node.childnode) + elif isinstance(node, (NodeSequence, AnyNode)): + return any(contains_variable(child) for child in node.children) + + return False + + def transform(node: Node) -> Iterable[str]: + # Generate separate pattern for all terms that contain variables + # within this OR. Terms that don't contain a variable can be merged + # together in one pattern. + if isinstance(node, AnyNode): + # If we have a definition like: + # (?P .*) | (?P .*) + # Then we want to be able to generate completions for both the + # name as well as the city. We do this by yielding two + # different regular expressions, because the engine won't + # follow multiple paths, if multiple are possible. + children_with_variable = [] + children_without_variable = [] + for c in node.children: + if contains_variable(c): + children_with_variable.append(c) + else: + children_without_variable.append(c) + + for c in children_with_variable: + yield from transform(c) + + # Merge options without variable together. + if children_without_variable: + yield "|".join( + r for c in children_without_variable for r in transform(c) + ) + + # For a sequence, generate a pattern for each prefix that ends with + # a variable + one pattern of the complete sequence. + # (This is because, for autocompletion, we match the text before + # the cursor, and completions are given for the variable that we + # match right before the cursor.) + elif isinstance(node, NodeSequence): + # For all components in the sequence, compute prefix patterns, + # as well as full patterns. + complete = [cls._transform(c, create_group_func) for c in node.children] + prefixes = [list(transform(c)) for c in node.children] + variable_nodes = [contains_variable(c) for c in node.children] + + # If any child is contains a variable, we should yield a + # pattern up to that point, so that we are sure this will be + # matched. + for i in range(len(node.children)): + if variable_nodes[i]: + for c_str in prefixes[i]: + yield "".join(complete[:i]) + c_str + + # If there are non-variable nodes, merge all the prefixes into + # one pattern. If the input is: "[part1] [part2] [part3]", then + # this gets compiled into: + # (complete1 + (complete2 + (complete3 | partial3) | partial2) | partial1 ) + # For nodes that contain a variable, we skip the "|partial" + # part here, because thees are matched with the previous + # patterns. + if not all(variable_nodes): + result = [] + + # Start with complete patterns. + for i in range(len(node.children)): + result.append("(?:") + result.append(complete[i]) + + # Add prefix patterns. + for i in range(len(node.children) - 1, -1, -1): + if variable_nodes[i]: + # No need to yield a prefix for this one, we did + # the variable prefixes earlier. + result.append(")") + else: + result.append("|(?:") + # If this yields multiple, we should yield all combinations. + assert len(prefixes[i]) == 1 + result.append(prefixes[i][0]) + result.append("))") + + yield "".join(result) + + elif isinstance(node, Regex): + yield f"(?:{node.regex})?" + + elif isinstance(node, Lookahead): + if node.negative: + yield f"(?!{cls._transform(node.childnode, create_group_func)})" + else: + # Not sure what the correct semantics are in this case. + # (Probably it's not worth implementing this.) + raise Exception("Positive lookahead not yet supported.") + + elif isinstance(node, Variable): + # (Note that we should not append a '?' here. the 'transform' + # method will already recursively do that.) + for c_str in transform(node.childnode): + yield f"(?P<{create_group_func(node)}>{c_str})" + + elif isinstance(node, Repeat): + # If we have a repetition of 8 times. That would mean that the + # current input could have for instance 7 times a complete + # match, followed by a partial match. + prefix = cls._transform(node.childnode, create_group_func) + + if node.max_repeat == 1: + yield from transform(node.childnode) + else: + for c_str in transform(node.childnode): + if node.max_repeat: + repeat_sign = "{,%i}" % (node.max_repeat - 1) + else: + repeat_sign = "*" + yield "(?:{}){}{}{}".format( + prefix, + repeat_sign, + ("" if node.greedy else "?"), + c_str, + ) + + else: + raise TypeError(f"Got {node!r}") + + for r in transform(root_node): + yield f"^(?:{r})$" + + def match(self, string: str) -> Match | None: + """ + Match the string with the grammar. + Returns a :class:`Match` instance or `None` when the input doesn't match the grammar. + + :param string: The input string. + """ + m = self._re.match(string) + + if m: + return Match( + string, [(self._re, m)], self._group_names_to_nodes, self.unescape_funcs + ) + return None + + def match_prefix(self, string: str) -> Match | None: + """ + Do a partial match of the string with the grammar. The returned + :class:`Match` instance can contain multiple representations of the + match. This will never return `None`. If it doesn't match at all, the "trailing input" + part will capture all of the input. + + :param string: The input string. + """ + # First try to match using `_re_prefix`. If nothing is found, use the patterns that + # also accept trailing characters. + for patterns in [self._re_prefix, self._re_prefix_with_trailing_input]: + matches = [(r, r.match(string)) for r in patterns] + matches2 = [(r, m) for r, m in matches if m] + + if matches2 != []: + return Match( + string, matches2, self._group_names_to_nodes, self.unescape_funcs + ) + + return None + + +class Match: + """ + :param string: The input string. + :param re_matches: List of (compiled_re_pattern, re_match) tuples. + :param group_names_to_nodes: Dictionary mapping all the re group names to the matching Node instances. + """ + + def __init__( + self, + string: str, + re_matches: list[tuple[Pattern[str], RegexMatch[str]]], + group_names_to_nodes: dict[str, str], + unescape_funcs: dict[str, Callable[[str], str]], + ): + self.string = string + self._re_matches = re_matches + self._group_names_to_nodes = group_names_to_nodes + self._unescape_funcs = unescape_funcs + + def _nodes_to_regs(self) -> list[tuple[str, tuple[int, int]]]: + """ + Return a list of (varname, reg) tuples. + """ + + def get_tuples() -> Iterable[tuple[str, tuple[int, int]]]: + for r, re_match in self._re_matches: + for group_name, group_index in r.groupindex.items(): + if group_name != _INVALID_TRAILING_INPUT: + regs = re_match.regs + reg = regs[group_index] + node = self._group_names_to_nodes[group_name] + yield (node, reg) + + return list(get_tuples()) + + def _nodes_to_values(self) -> list[tuple[str, str, tuple[int, int]]]: + """ + Returns list of (Node, string_value) tuples. + """ + + def is_none(sl: tuple[int, int]) -> bool: + return sl[0] == -1 and sl[1] == -1 + + def get(sl: tuple[int, int]) -> str: + return self.string[sl[0] : sl[1]] + + return [ + (varname, get(slice), slice) + for varname, slice in self._nodes_to_regs() + if not is_none(slice) + ] + + def _unescape(self, varname: str, value: str) -> str: + unwrapper = self._unescape_funcs.get(varname) + return unwrapper(value) if unwrapper else value + + def variables(self) -> Variables: + """ + Returns :class:`Variables` instance. + """ + return Variables( + [(k, self._unescape(k, v), sl) for k, v, sl in self._nodes_to_values()] + ) + + def trailing_input(self) -> MatchVariable | None: + """ + Get the `MatchVariable` instance, representing trailing input, if there is any. + "Trailing input" is input at the end that does not match the grammar anymore, but + when this is removed from the end of the input, the input would be a valid string. + """ + slices: list[tuple[int, int]] = [] + + # Find all regex group for the name _INVALID_TRAILING_INPUT. + for r, re_match in self._re_matches: + for group_name, group_index in r.groupindex.items(): + if group_name == _INVALID_TRAILING_INPUT: + slices.append(re_match.regs[group_index]) + + # Take the smallest part. (Smaller trailing text means that a larger input has + # been matched, so that is better.) + if slices: + slice = (max(i[0] for i in slices), max(i[1] for i in slices)) + value = self.string[slice[0] : slice[1]] + return MatchVariable("", value, slice) + return None + + def end_nodes(self) -> Iterable[MatchVariable]: + """ + Yields `MatchVariable` instances for all the nodes having their end + position at the end of the input string. + """ + for varname, reg in self._nodes_to_regs(): + # If this part goes until the end of the input string. + if reg[1] == len(self.string): + value = self._unescape(varname, self.string[reg[0] : reg[1]]) + yield MatchVariable(varname, value, (reg[0], reg[1])) + + +_T = TypeVar("_T") + + +class Variables: + def __init__(self, tuples: list[tuple[str, str, tuple[int, int]]]) -> None: + #: List of (varname, value, slice) tuples. + self._tuples = tuples + + def __repr__(self) -> str: + return "{}({})".format( + self.__class__.__name__, + ", ".join(f"{k}={v!r}" for k, v, _ in self._tuples), + ) + + @overload + def get(self, key: str) -> str | None: ... + + @overload + def get(self, key: str, default: str | _T) -> str | _T: ... + + def get(self, key: str, default: str | _T | None = None) -> str | _T | None: + items = self.getall(key) + return items[0] if items else default + + def getall(self, key: str) -> list[str]: + return [v for k, v, _ in self._tuples if k == key] + + def __getitem__(self, key: str) -> str | None: + return self.get(key) + + def __iter__(self) -> Iterator[MatchVariable]: + """ + Yield `MatchVariable` instances. + """ + for varname, value, slice in self._tuples: + yield MatchVariable(varname, value, slice) + + +class MatchVariable: + """ + Represents a match of a variable in the grammar. + + :param varname: (string) Name of the variable. + :param value: (string) Value of this variable. + :param slice: (start, stop) tuple, indicating the position of this variable + in the input string. + """ + + def __init__(self, varname: str, value: str, slice: tuple[int, int]) -> None: + self.varname = varname + self.value = value + self.slice = slice + + self.start = self.slice[0] + self.stop = self.slice[1] + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.varname!r}, {self.value!r})" + + +def compile( + expression: str, + escape_funcs: EscapeFuncDict | None = None, + unescape_funcs: EscapeFuncDict | None = None, +) -> _CompiledGrammar: + """ + Compile grammar (given as regex string), returning a `CompiledGrammar` + instance. + """ + return _compile_from_parse_tree( + parse_regex(tokenize_regex(expression)), + escape_funcs=escape_funcs, + unescape_funcs=unescape_funcs, + ) + + +def _compile_from_parse_tree( + root_node: Node, + escape_funcs: EscapeFuncDict | None = None, + unescape_funcs: EscapeFuncDict | None = None, +) -> _CompiledGrammar: + """ + Compile grammar (given as parse tree), returning a `CompiledGrammar` + instance. + """ + return _CompiledGrammar( + root_node, escape_funcs=escape_funcs, unescape_funcs=unescape_funcs + ) diff --git a/lib/prompt_toolkit/contrib/regular_languages/completion.py b/lib/prompt_toolkit/contrib/regular_languages/completion.py new file mode 100644 index 0000000..19ebaad --- /dev/null +++ b/lib/prompt_toolkit/contrib/regular_languages/completion.py @@ -0,0 +1,100 @@ +""" +Completer for a regular grammar. +""" + +from __future__ import annotations + +from typing import Iterable + +from prompt_toolkit.completion import CompleteEvent, Completer, Completion +from prompt_toolkit.document import Document + +from .compiler import Match, _CompiledGrammar + +__all__ = [ + "GrammarCompleter", +] + + +class GrammarCompleter(Completer): + """ + Completer which can be used for autocompletion according to variables in + the grammar. Each variable can have a different autocompleter. + + :param compiled_grammar: `GrammarCompleter` instance. + :param completers: `dict` mapping variable names of the grammar to the + `Completer` instances to be used for each variable. + """ + + def __init__( + self, compiled_grammar: _CompiledGrammar, completers: dict[str, Completer] + ) -> None: + self.compiled_grammar = compiled_grammar + self.completers = completers + + def get_completions( + self, document: Document, complete_event: CompleteEvent + ) -> Iterable[Completion]: + m = self.compiled_grammar.match_prefix(document.text_before_cursor) + + if m: + yield from self._remove_duplicates( + self._get_completions_for_match(m, complete_event) + ) + + def _get_completions_for_match( + self, match: Match, complete_event: CompleteEvent + ) -> Iterable[Completion]: + """ + Yield all the possible completions for this input string. + (The completer assumes that the cursor position was at the end of the + input string.) + """ + for match_variable in match.end_nodes(): + varname = match_variable.varname + start = match_variable.start + + completer = self.completers.get(varname) + + if completer: + text = match_variable.value + + # Unwrap text. + unwrapped_text = self.compiled_grammar.unescape(varname, text) + + # Create a document, for the completions API (text/cursor_position) + document = Document(unwrapped_text, len(unwrapped_text)) + + # Call completer + for completion in completer.get_completions(document, complete_event): + new_text = ( + unwrapped_text[: len(text) + completion.start_position] + + completion.text + ) + + # Wrap again. + yield Completion( + text=self.compiled_grammar.escape(varname, new_text), + start_position=start - len(match.string), + display=completion.display, + display_meta=completion.display_meta, + ) + + def _remove_duplicates(self, items: Iterable[Completion]) -> Iterable[Completion]: + """ + Remove duplicates, while keeping the order. + (Sometimes we have duplicates, because the there several matches of the + same grammar, each yielding similar completions.) + """ + + def hash_completion(completion: Completion) -> tuple[str, int]: + return completion.text, completion.start_position + + yielded_so_far: set[tuple[str, int]] = set() + + for completion in items: + hash_value = hash_completion(completion) + + if hash_value not in yielded_so_far: + yielded_so_far.add(hash_value) + yield completion diff --git a/lib/prompt_toolkit/contrib/regular_languages/lexer.py b/lib/prompt_toolkit/contrib/regular_languages/lexer.py new file mode 100644 index 0000000..c5434cf --- /dev/null +++ b/lib/prompt_toolkit/contrib/regular_languages/lexer.py @@ -0,0 +1,94 @@ +""" +`GrammarLexer` is compatible with other lexers and can be used to highlight +the input using a regular grammar with annotations. +""" + +from __future__ import annotations + +from typing import Callable + +from prompt_toolkit.document import Document +from prompt_toolkit.formatted_text.base import StyleAndTextTuples +from prompt_toolkit.formatted_text.utils import split_lines +from prompt_toolkit.lexers import Lexer + +from .compiler import _CompiledGrammar + +__all__ = [ + "GrammarLexer", +] + + +class GrammarLexer(Lexer): + """ + Lexer which can be used for highlighting of fragments according to variables in the grammar. + + (It does not actual lexing of the string, but it exposes an API, compatible + with the Pygments lexer class.) + + :param compiled_grammar: Grammar as returned by the `compile()` function. + :param lexers: Dictionary mapping variable names of the regular grammar to + the lexers that should be used for this part. (This can + call other lexers recursively.) If you wish a part of the + grammar to just get one fragment, use a + `prompt_toolkit.lexers.SimpleLexer`. + """ + + def __init__( + self, + compiled_grammar: _CompiledGrammar, + default_style: str = "", + lexers: dict[str, Lexer] | None = None, + ) -> None: + self.compiled_grammar = compiled_grammar + self.default_style = default_style + self.lexers = lexers or {} + + def _get_text_fragments(self, text: str) -> StyleAndTextTuples: + m = self.compiled_grammar.match_prefix(text) + + if m: + characters: StyleAndTextTuples = [(self.default_style, c) for c in text] + + for v in m.variables(): + # If we have a `Lexer` instance for this part of the input. + # Tokenize recursively and apply tokens. + lexer = self.lexers.get(v.varname) + + if lexer: + document = Document(text[v.start : v.stop]) + lexer_tokens_for_line = lexer.lex_document(document) + text_fragments: StyleAndTextTuples = [] + for i in range(len(document.lines)): + text_fragments.extend(lexer_tokens_for_line(i)) + text_fragments.append(("", "\n")) + if text_fragments: + text_fragments.pop() + + i = v.start + for t, s, *_ in text_fragments: + for c in s: + if characters[i][0] == self.default_style: + characters[i] = (t, characters[i][1]) + i += 1 + + # Highlight trailing input. + trailing_input = m.trailing_input() + if trailing_input: + for i in range(trailing_input.start, trailing_input.stop): + characters[i] = ("class:trailing-input", characters[i][1]) + + return characters + else: + return [("", text)] + + def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: + lines = list(split_lines(self._get_text_fragments(document.text))) + + def get_line(lineno: int) -> StyleAndTextTuples: + try: + return lines[lineno] + except IndexError: + return [] + + return get_line diff --git a/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py b/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py new file mode 100644 index 0000000..353e54f --- /dev/null +++ b/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py @@ -0,0 +1,279 @@ +""" +Parser for parsing a regular expression. +Take a string representing a regular expression and return the root node of its +parse tree. + +usage:: + + root_node = parse_regex('(hello|world)') + +Remarks: +- The regex parser processes multiline, it ignores all whitespace and supports + multiple named groups with the same name and #-style comments. + +Limitations: +- Lookahead is not supported. +""" + +from __future__ import annotations + +import re + +__all__ = [ + "Repeat", + "Variable", + "Regex", + "Lookahead", + "tokenize_regex", + "parse_regex", +] + + +class Node: + """ + Base class for all the grammar nodes. + (You don't initialize this one.) + """ + + def __add__(self, other_node: Node) -> NodeSequence: + return NodeSequence([self, other_node]) + + def __or__(self, other_node: Node) -> AnyNode: + return AnyNode([self, other_node]) + + +class AnyNode(Node): + """ + Union operation (OR operation) between several grammars. You don't + initialize this yourself, but it's a result of a "Grammar1 | Grammar2" + operation. + """ + + def __init__(self, children: list[Node]) -> None: + self.children = children + + def __or__(self, other_node: Node) -> AnyNode: + return AnyNode(self.children + [other_node]) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.children!r})" + + +class NodeSequence(Node): + """ + Concatenation operation of several grammars. You don't initialize this + yourself, but it's a result of a "Grammar1 + Grammar2" operation. + """ + + def __init__(self, children: list[Node]) -> None: + self.children = children + + def __add__(self, other_node: Node) -> NodeSequence: + return NodeSequence(self.children + [other_node]) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.children!r})" + + +class Regex(Node): + """ + Regular expression. + """ + + def __init__(self, regex: str) -> None: + re.compile(regex) # Validate + + self.regex = regex + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(/{self.regex}/)" + + +class Lookahead(Node): + """ + Lookahead expression. + """ + + def __init__(self, childnode: Node, negative: bool = False) -> None: + self.childnode = childnode + self.negative = negative + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.childnode!r})" + + +class Variable(Node): + """ + Mark a variable in the regular grammar. This will be translated into a + named group. Each variable can have his own completer, validator, etc.. + + :param childnode: The grammar which is wrapped inside this variable. + :param varname: String. + """ + + def __init__(self, childnode: Node, varname: str = "") -> None: + self.childnode = childnode + self.varname = varname + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(childnode={self.childnode!r}, varname={self.varname!r})" + + +class Repeat(Node): + def __init__( + self, + childnode: Node, + min_repeat: int = 0, + max_repeat: int | None = None, + greedy: bool = True, + ) -> None: + self.childnode = childnode + self.min_repeat = min_repeat + self.max_repeat = max_repeat + self.greedy = greedy + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(childnode={self.childnode!r})" + + +def tokenize_regex(input: str) -> list[str]: + """ + Takes a string, representing a regular expression as input, and tokenizes + it. + + :param input: string, representing a regular expression. + :returns: List of tokens. + """ + # Regular expression for tokenizing other regular expressions. + p = re.compile( + r"""^( + \(\?P\<[a-zA-Z0-9_-]+\> | # Start of named group. + \(\?#[^)]*\) | # Comment + \(\?= | # Start of lookahead assertion + \(\?! | # Start of negative lookahead assertion + \(\?<= | # If preceded by. + \(\?< | # If not preceded by. + \(?: | # Start of group. (non capturing.) + \( | # Start of group. + \(?[iLmsux] | # Flags. + \(?P=[a-zA-Z]+\) | # Back reference to named group + \) | # End of group. + \{[^{}]*\} | # Repetition + \*\? | \+\? | \?\?\ | # Non greedy repetition. + \* | \+ | \? | # Repetition + \#.*\n | # Comment + \\. | + + # Character group. + \[ + ( [^\]\\] | \\.)* + \] | + + [^(){}] | + . + )""", + re.VERBOSE, + ) + + tokens = [] + + while input: + m = p.match(input) + if m: + token, input = input[: m.end()], input[m.end() :] + if not token.isspace(): + tokens.append(token) + else: + raise Exception("Could not tokenize input regex.") + + return tokens + + +def parse_regex(regex_tokens: list[str]) -> Node: + """ + Takes a list of tokens from the tokenizer, and returns a parse tree. + """ + # We add a closing brace because that represents the final pop of the stack. + tokens: list[str] = [")"] + regex_tokens[::-1] + + def wrap(lst: list[Node]) -> Node: + """Turn list into sequence when it contains several items.""" + if len(lst) == 1: + return lst[0] + else: + return NodeSequence(lst) + + def _parse() -> Node: + or_list: list[list[Node]] = [] + result: list[Node] = [] + + def wrapped_result() -> Node: + if or_list == []: + return wrap(result) + else: + or_list.append(result) + return AnyNode([wrap(i) for i in or_list]) + + while tokens: + t = tokens.pop() + + if t.startswith("(?P<"): + variable = Variable(_parse(), varname=t[4:-1]) + result.append(variable) + + elif t in ("*", "*?"): + greedy = t == "*" + result[-1] = Repeat(result[-1], greedy=greedy) + + elif t in ("+", "+?"): + greedy = t == "+" + result[-1] = Repeat(result[-1], min_repeat=1, greedy=greedy) + + elif t in ("?", "??"): + if result == []: + raise Exception("Nothing to repeat." + repr(tokens)) + else: + greedy = t == "?" + result[-1] = Repeat( + result[-1], min_repeat=0, max_repeat=1, greedy=greedy + ) + + elif t == "|": + or_list.append(result) + result = [] + + elif t in ("(", "(?:"): + result.append(_parse()) + + elif t == "(?!": + result.append(Lookahead(_parse(), negative=True)) + + elif t == "(?=": + result.append(Lookahead(_parse(), negative=False)) + + elif t == ")": + return wrapped_result() + + elif t.startswith("#"): + pass + + elif t.startswith("{"): + # TODO: implement! + raise Exception(f"{t}-style repetition not yet supported") + + elif t.startswith("(?"): + raise Exception(f"{t!r} not supported") + + elif t.isspace(): + pass + else: + result.append(Regex(t)) + + raise Exception("Expecting ')' token") + + result = _parse() + + if len(tokens) != 0: + raise Exception("Unmatched parentheses.") + else: + return result diff --git a/lib/prompt_toolkit/contrib/regular_languages/validation.py b/lib/prompt_toolkit/contrib/regular_languages/validation.py new file mode 100644 index 0000000..e6cfd74 --- /dev/null +++ b/lib/prompt_toolkit/contrib/regular_languages/validation.py @@ -0,0 +1,60 @@ +""" +Validator for a regular language. +""" + +from __future__ import annotations + +from prompt_toolkit.document import Document +from prompt_toolkit.validation import ValidationError, Validator + +from .compiler import _CompiledGrammar + +__all__ = [ + "GrammarValidator", +] + + +class GrammarValidator(Validator): + """ + Validator which can be used for validation according to variables in + the grammar. Each variable can have its own validator. + + :param compiled_grammar: `GrammarCompleter` instance. + :param validators: `dict` mapping variable names of the grammar to the + `Validator` instances to be used for each variable. + """ + + def __init__( + self, compiled_grammar: _CompiledGrammar, validators: dict[str, Validator] + ) -> None: + self.compiled_grammar = compiled_grammar + self.validators = validators + + def validate(self, document: Document) -> None: + # Parse input document. + # We use `match`, not `match_prefix`, because for validation, we want + # the actual, unambiguous interpretation of the input. + m = self.compiled_grammar.match(document.text) + + if m: + for v in m.variables(): + validator = self.validators.get(v.varname) + + if validator: + # Unescape text. + unwrapped_text = self.compiled_grammar.unescape(v.varname, v.value) + + # Create a document, for the completions API (text/cursor_position) + inner_document = Document(unwrapped_text, len(unwrapped_text)) + + try: + validator.validate(inner_document) + except ValidationError as e: + raise ValidationError( + cursor_position=v.start + e.cursor_position, + message=e.message, + ) from e + else: + raise ValidationError( + cursor_position=len(document.text), message="Invalid command" + ) diff --git a/lib/prompt_toolkit/contrib/ssh/__init__.py b/lib/prompt_toolkit/contrib/ssh/__init__.py new file mode 100644 index 0000000..bbc1c21 --- /dev/null +++ b/lib/prompt_toolkit/contrib/ssh/__init__.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from .server import PromptToolkitSSHServer, PromptToolkitSSHSession + +__all__ = [ + "PromptToolkitSSHSession", + "PromptToolkitSSHServer", +] diff --git a/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..2872e1c Binary files /dev/null and b/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc b/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc new file mode 100644 index 0000000..cfd9b86 Binary files /dev/null and b/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/ssh/server.py b/lib/prompt_toolkit/contrib/ssh/server.py new file mode 100644 index 0000000..4badc6c --- /dev/null +++ b/lib/prompt_toolkit/contrib/ssh/server.py @@ -0,0 +1,178 @@ +""" +Utility for running a prompt_toolkit application in an asyncssh server. +""" + +from __future__ import annotations + +import asyncio +import traceback +from asyncio import get_running_loop +from typing import Any, Callable, Coroutine, TextIO, cast + +import asyncssh + +from prompt_toolkit.application.current import AppSession, create_app_session +from prompt_toolkit.data_structures import Size +from prompt_toolkit.input import PipeInput, create_pipe_input +from prompt_toolkit.output.vt100 import Vt100_Output + +__all__ = ["PromptToolkitSSHSession", "PromptToolkitSSHServer"] + + +class PromptToolkitSSHSession(asyncssh.SSHServerSession): # type: ignore + def __init__( + self, + interact: Callable[[PromptToolkitSSHSession], Coroutine[Any, Any, None]], + *, + enable_cpr: bool, + ) -> None: + self.interact = interact + self.enable_cpr = enable_cpr + self.interact_task: asyncio.Task[None] | None = None + self._chan: Any | None = None + self.app_session: AppSession | None = None + + # PipInput object, for sending input in the CLI. + # (This is something that we can use in the prompt_toolkit event loop, + # but still write date in manually.) + self._input: PipeInput | None = None + self._output: Vt100_Output | None = None + + # Output object. Don't render to the real stdout, but write everything + # in the SSH channel. + class Stdout: + def write(s, data: str) -> None: + try: + if self._chan is not None: + self._chan.write(data.replace("\n", "\r\n")) + except BrokenPipeError: + pass # Channel not open for sending. + + def isatty(s) -> bool: + return True + + def flush(s) -> None: + pass + + @property + def encoding(s) -> str: + assert self._chan is not None + return str(self._chan._orig_chan.get_encoding()[0]) + + self.stdout = cast(TextIO, Stdout()) + + def _get_size(self) -> Size: + """ + Callable that returns the current `Size`, required by Vt100_Output. + """ + if self._chan is None: + return Size(rows=20, columns=79) + else: + width, height, pixwidth, pixheight = self._chan.get_terminal_size() + return Size(rows=height, columns=width) + + def connection_made(self, chan: Any) -> None: + self._chan = chan + + def shell_requested(self) -> bool: + return True + + def session_started(self) -> None: + self.interact_task = get_running_loop().create_task(self._interact()) + + async def _interact(self) -> None: + if self._chan is None: + # Should not happen. + raise Exception("`_interact` called before `connection_made`.") + + if hasattr(self._chan, "set_line_mode") and self._chan._editor is not None: + # Disable the line editing provided by asyncssh. Prompt_toolkit + # provides the line editing. + self._chan.set_line_mode(False) + + term = self._chan.get_terminal_type() + + self._output = Vt100_Output( + self.stdout, self._get_size, term=term, enable_cpr=self.enable_cpr + ) + + with create_pipe_input() as self._input: + with create_app_session(input=self._input, output=self._output) as session: + self.app_session = session + try: + await self.interact(self) + except BaseException: + traceback.print_exc() + finally: + # Close the connection. + self._chan.close() + self._input.close() + + def terminal_size_changed( + self, width: int, height: int, pixwidth: object, pixheight: object + ) -> None: + # Send resize event to the current application. + if self.app_session and self.app_session.app: + self.app_session.app._on_resize() + + def data_received(self, data: str, datatype: object) -> None: + if self._input is None: + # Should not happen. + return + + self._input.send_text(data) + + +class PromptToolkitSSHServer(asyncssh.SSHServer): + """ + Run a prompt_toolkit application over an asyncssh server. + + This takes one argument, an `interact` function, which is called for each + connection. This should be an asynchronous function that runs the + prompt_toolkit applications. This function runs in an `AppSession`, which + means that we can have multiple UI interactions concurrently. + + Example usage: + + .. code:: python + + async def interact(ssh_session: PromptToolkitSSHSession) -> None: + await yes_no_dialog("my title", "my text").run_async() + + prompt_session = PromptSession() + text = await prompt_session.prompt_async("Type something: ") + print_formatted_text('You said: ', text) + + server = PromptToolkitSSHServer(interact=interact) + loop = get_running_loop() + loop.run_until_complete( + asyncssh.create_server( + lambda: MySSHServer(interact), + "", + port, + server_host_keys=["/etc/ssh/..."], + ) + ) + loop.run_forever() + + :param enable_cpr: When `True`, the default, try to detect whether the SSH + client runs in a terminal that responds to "cursor position requests". + That way, we can properly determine how much space there is available + for the UI (especially for drop down menus) to render. + """ + + def __init__( + self, + interact: Callable[[PromptToolkitSSHSession], Coroutine[Any, Any, None]], + *, + enable_cpr: bool = True, + ) -> None: + self.interact = interact + self.enable_cpr = enable_cpr + + def begin_auth(self, username: str) -> bool: + # No authentication. + return False + + def session_requested(self) -> PromptToolkitSSHSession: + return PromptToolkitSSHSession(self.interact, enable_cpr=self.enable_cpr) diff --git a/lib/prompt_toolkit/contrib/telnet/__init__.py b/lib/prompt_toolkit/contrib/telnet/__init__.py new file mode 100644 index 0000000..de902b4 --- /dev/null +++ b/lib/prompt_toolkit/contrib/telnet/__init__.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from .server import TelnetServer + +__all__ = [ + "TelnetServer", +] diff --git a/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..925f381 Binary files /dev/null and b/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc b/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc new file mode 100644 index 0000000..d2286ce Binary files /dev/null and b/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc b/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc new file mode 100644 index 0000000..0bffbd5 Binary files /dev/null and b/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc b/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc new file mode 100644 index 0000000..ff29595 Binary files /dev/null and b/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/contrib/telnet/log.py b/lib/prompt_toolkit/contrib/telnet/log.py new file mode 100644 index 0000000..476dffc --- /dev/null +++ b/lib/prompt_toolkit/contrib/telnet/log.py @@ -0,0 +1,13 @@ +""" +Python logger for the telnet server. +""" + +from __future__ import annotations + +import logging + +logger = logging.getLogger(__package__) + +__all__ = [ + "logger", +] diff --git a/lib/prompt_toolkit/contrib/telnet/protocol.py b/lib/prompt_toolkit/contrib/telnet/protocol.py new file mode 100644 index 0000000..58286e2 --- /dev/null +++ b/lib/prompt_toolkit/contrib/telnet/protocol.py @@ -0,0 +1,209 @@ +""" +Parser for the Telnet protocol. (Not a complete implementation of the telnet +specification, but sufficient for a command line interface.) + +Inspired by `Twisted.conch.telnet`. +""" + +from __future__ import annotations + +import struct +from typing import Callable, Generator + +from .log import logger + +__all__ = [ + "TelnetProtocolParser", +] + + +def int2byte(number: int) -> bytes: + return bytes((number,)) + + +# Telnet constants. +NOP = int2byte(0) +SGA = int2byte(3) + +IAC = int2byte(255) +DO = int2byte(253) +DONT = int2byte(254) +LINEMODE = int2byte(34) +SB = int2byte(250) +WILL = int2byte(251) +WONT = int2byte(252) +MODE = int2byte(1) +SE = int2byte(240) +ECHO = int2byte(1) +NAWS = int2byte(31) +LINEMODE = int2byte(34) +SUPPRESS_GO_AHEAD = int2byte(3) + +TTYPE = int2byte(24) +SEND = int2byte(1) +IS = int2byte(0) + +DM = int2byte(242) +BRK = int2byte(243) +IP = int2byte(244) +AO = int2byte(245) +AYT = int2byte(246) +EC = int2byte(247) +EL = int2byte(248) +GA = int2byte(249) + + +class TelnetProtocolParser: + """ + Parser for the Telnet protocol. + Usage:: + + def data_received(data): + print(data) + + def size_received(rows, columns): + print(rows, columns) + + p = TelnetProtocolParser(data_received, size_received) + p.feed(binary_data) + """ + + def __init__( + self, + data_received_callback: Callable[[bytes], None], + size_received_callback: Callable[[int, int], None], + ttype_received_callback: Callable[[str], None], + ) -> None: + self.data_received_callback = data_received_callback + self.size_received_callback = size_received_callback + self.ttype_received_callback = ttype_received_callback + + self._parser = self._parse_coroutine() + self._parser.send(None) # type: ignore + + def received_data(self, data: bytes) -> None: + self.data_received_callback(data) + + def do_received(self, data: bytes) -> None: + """Received telnet DO command.""" + logger.info("DO %r", data) + + def dont_received(self, data: bytes) -> None: + """Received telnet DONT command.""" + logger.info("DONT %r", data) + + def will_received(self, data: bytes) -> None: + """Received telnet WILL command.""" + logger.info("WILL %r", data) + + def wont_received(self, data: bytes) -> None: + """Received telnet WONT command.""" + logger.info("WONT %r", data) + + def command_received(self, command: bytes, data: bytes) -> None: + if command == DO: + self.do_received(data) + + elif command == DONT: + self.dont_received(data) + + elif command == WILL: + self.will_received(data) + + elif command == WONT: + self.wont_received(data) + + else: + logger.info("command received %r %r", command, data) + + def naws(self, data: bytes) -> None: + """ + Received NAWS. (Window dimensions.) + """ + if len(data) == 4: + # NOTE: the first parameter of struct.unpack should be + # a 'str' object. Both on Py2/py3. This crashes on OSX + # otherwise. + columns, rows = struct.unpack("!HH", data) + self.size_received_callback(rows, columns) + else: + logger.warning("Wrong number of NAWS bytes") + + def ttype(self, data: bytes) -> None: + """ + Received terminal type. + """ + subcmd, data = data[0:1], data[1:] + if subcmd == IS: + ttype = data.decode("ascii") + self.ttype_received_callback(ttype) + else: + logger.warning("Received a non-IS terminal type Subnegotiation") + + def negotiate(self, data: bytes) -> None: + """ + Got negotiate data. + """ + command, payload = data[0:1], data[1:] + + if command == NAWS: + self.naws(payload) + elif command == TTYPE: + self.ttype(payload) + else: + logger.info("Negotiate (%r got bytes)", len(data)) + + def _parse_coroutine(self) -> Generator[None, bytes, None]: + """ + Parser state machine. + Every 'yield' expression returns the next byte. + """ + while True: + d = yield + + if d == int2byte(0): + pass # NOP + + # Go to state escaped. + elif d == IAC: + d2 = yield + + if d2 == IAC: + self.received_data(d2) + + # Handle simple commands. + elif d2 in (NOP, DM, BRK, IP, AO, AYT, EC, EL, GA): + self.command_received(d2, b"") + + # Handle IAC-[DO/DONT/WILL/WONT] commands. + elif d2 in (DO, DONT, WILL, WONT): + d3 = yield + self.command_received(d2, d3) + + # Subnegotiation + elif d2 == SB: + # Consume everything until next IAC-SE + data = [] + + while True: + d3 = yield + + if d3 == IAC: + d4 = yield + if d4 == SE: + break + else: + data.append(d4) + else: + data.append(d3) + + self.negotiate(b"".join(data)) + else: + self.received_data(d) + + def feed(self, data: bytes) -> None: + """ + Feed data to the parser. + """ + for b in data: + self._parser.send(int2byte(b)) diff --git a/lib/prompt_toolkit/contrib/telnet/server.py b/lib/prompt_toolkit/contrib/telnet/server.py new file mode 100644 index 0000000..bc2ad53 --- /dev/null +++ b/lib/prompt_toolkit/contrib/telnet/server.py @@ -0,0 +1,428 @@ +""" +Telnet server. +""" + +from __future__ import annotations + +import asyncio +import contextvars +import socket +from asyncio import get_running_loop +from typing import Any, Callable, Coroutine, TextIO, cast + +from prompt_toolkit.application.current import create_app_session, get_app +from prompt_toolkit.application.run_in_terminal import run_in_terminal +from prompt_toolkit.data_structures import Size +from prompt_toolkit.formatted_text import AnyFormattedText, to_formatted_text +from prompt_toolkit.input import PipeInput, create_pipe_input +from prompt_toolkit.output.vt100 import Vt100_Output +from prompt_toolkit.renderer import print_formatted_text as print_formatted_text +from prompt_toolkit.styles import BaseStyle, DummyStyle + +from .log import logger +from .protocol import ( + DO, + ECHO, + IAC, + LINEMODE, + MODE, + NAWS, + SB, + SE, + SEND, + SUPPRESS_GO_AHEAD, + TTYPE, + WILL, + TelnetProtocolParser, +) + +__all__ = [ + "TelnetServer", +] + + +def int2byte(number: int) -> bytes: + return bytes((number,)) + + +def _initialize_telnet(connection: socket.socket) -> None: + logger.info("Initializing telnet connection") + + # Iac Do Linemode + connection.send(IAC + DO + LINEMODE) + + # Suppress Go Ahead. (This seems important for Putty to do correct echoing.) + # This will allow bi-directional operation. + connection.send(IAC + WILL + SUPPRESS_GO_AHEAD) + + # Iac sb + connection.send(IAC + SB + LINEMODE + MODE + int2byte(0) + IAC + SE) + + # IAC Will Echo + connection.send(IAC + WILL + ECHO) + + # Negotiate window size + connection.send(IAC + DO + NAWS) + + # Negotiate terminal type + # Assume the client will accept the negotiation with `IAC + WILL + TTYPE` + connection.send(IAC + DO + TTYPE) + + # We can then select the first terminal type supported by the client, + # which is generally the best type the client supports + # The client should reply with a `IAC + SB + TTYPE + IS + ttype + IAC + SE` + connection.send(IAC + SB + TTYPE + SEND + IAC + SE) + + +class _ConnectionStdout: + """ + Wrapper around socket which provides `write` and `flush` methods for the + Vt100_Output output. + """ + + def __init__(self, connection: socket.socket, encoding: str) -> None: + self._encoding = encoding + self._connection = connection + self._errors = "strict" + self._buffer: list[bytes] = [] + self._closed = False + + def write(self, data: str) -> None: + data = data.replace("\n", "\r\n") + self._buffer.append(data.encode(self._encoding, errors=self._errors)) + self.flush() + + def isatty(self) -> bool: + return True + + def flush(self) -> None: + try: + if not self._closed: + self._connection.send(b"".join(self._buffer)) + except OSError as e: + logger.warning(f"Couldn't send data over socket: {e}") + + self._buffer = [] + + def close(self) -> None: + self._closed = True + + @property + def encoding(self) -> str: + return self._encoding + + @property + def errors(self) -> str: + return self._errors + + +class TelnetConnection: + """ + Class that represents one Telnet connection. + """ + + def __init__( + self, + conn: socket.socket, + addr: tuple[str, int], + interact: Callable[[TelnetConnection], Coroutine[Any, Any, None]], + server: TelnetServer, + encoding: str, + style: BaseStyle | None, + vt100_input: PipeInput, + enable_cpr: bool = True, + ) -> None: + self.conn = conn + self.addr = addr + self.interact = interact + self.server = server + self.encoding = encoding + self.style = style + self._closed = False + self._ready = asyncio.Event() + self.vt100_input = vt100_input + self.enable_cpr = enable_cpr + self.vt100_output: Vt100_Output | None = None + + # Create "Output" object. + self.size = Size(rows=40, columns=79) + + # Initialize. + _initialize_telnet(conn) + + # Create output. + def get_size() -> Size: + return self.size + + self.stdout = cast(TextIO, _ConnectionStdout(conn, encoding=encoding)) + + def data_received(data: bytes) -> None: + """TelnetProtocolParser 'data_received' callback""" + self.vt100_input.send_bytes(data) + + def size_received(rows: int, columns: int) -> None: + """TelnetProtocolParser 'size_received' callback""" + self.size = Size(rows=rows, columns=columns) + if self.vt100_output is not None and self.context: + self.context.run(lambda: get_app()._on_resize()) + + def ttype_received(ttype: str) -> None: + """TelnetProtocolParser 'ttype_received' callback""" + self.vt100_output = Vt100_Output( + self.stdout, get_size, term=ttype, enable_cpr=enable_cpr + ) + self._ready.set() + + self.parser = TelnetProtocolParser(data_received, size_received, ttype_received) + self.context: contextvars.Context | None = None + + async def run_application(self) -> None: + """ + Run application. + """ + + def handle_incoming_data() -> None: + data = self.conn.recv(1024) + if data: + self.feed(data) + else: + # Connection closed by client. + logger.info("Connection closed by client. {!r} {!r}".format(*self.addr)) + self.close() + + # Add reader. + loop = get_running_loop() + loop.add_reader(self.conn, handle_incoming_data) + + try: + # Wait for v100_output to be properly instantiated + await self._ready.wait() + with create_app_session(input=self.vt100_input, output=self.vt100_output): + self.context = contextvars.copy_context() + await self.interact(self) + finally: + self.close() + + def feed(self, data: bytes) -> None: + """ + Handler for incoming data. (Called by TelnetServer.) + """ + self.parser.feed(data) + + def close(self) -> None: + """ + Closed by client. + """ + if not self._closed: + self._closed = True + + self.vt100_input.close() + get_running_loop().remove_reader(self.conn) + self.conn.close() + self.stdout.close() + + def send(self, formatted_text: AnyFormattedText) -> None: + """ + Send text to the client. + """ + if self.vt100_output is None: + return + formatted_text = to_formatted_text(formatted_text) + print_formatted_text( + self.vt100_output, formatted_text, self.style or DummyStyle() + ) + + def send_above_prompt(self, formatted_text: AnyFormattedText) -> None: + """ + Send text to the client. + This is asynchronous, returns a `Future`. + """ + formatted_text = to_formatted_text(formatted_text) + return self._run_in_terminal(lambda: self.send(formatted_text)) + + def _run_in_terminal(self, func: Callable[[], None]) -> None: + # Make sure that when an application was active for this connection, + # that we print the text above the application. + if self.context: + self.context.run(run_in_terminal, func) + else: + raise RuntimeError("Called _run_in_terminal outside `run_application`.") + + def erase_screen(self) -> None: + """ + Erase the screen and move the cursor to the top. + """ + if self.vt100_output is None: + return + self.vt100_output.erase_screen() + self.vt100_output.cursor_goto(0, 0) + self.vt100_output.flush() + + +async def _dummy_interact(connection: TelnetConnection) -> None: + pass + + +class TelnetServer: + """ + Telnet server implementation. + + Example:: + + async def interact(connection): + connection.send("Welcome") + session = PromptSession() + result = await session.prompt_async(message="Say something: ") + connection.send(f"You said: {result}\n") + + async def main(): + server = TelnetServer(interact=interact, port=2323) + await server.run() + """ + + def __init__( + self, + host: str = "127.0.0.1", + port: int = 23, + interact: Callable[ + [TelnetConnection], Coroutine[Any, Any, None] + ] = _dummy_interact, + encoding: str = "utf-8", + style: BaseStyle | None = None, + enable_cpr: bool = True, + ) -> None: + self.host = host + self.port = port + self.interact = interact + self.encoding = encoding + self.style = style + self.enable_cpr = enable_cpr + + self._run_task: asyncio.Task[None] | None = None + self._application_tasks: list[asyncio.Task[None]] = [] + + self.connections: set[TelnetConnection] = set() + + @classmethod + def _create_socket(cls, host: str, port: int) -> socket.socket: + # Create and bind socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + + s.listen(4) + return s + + async def run(self, ready_cb: Callable[[], None] | None = None) -> None: + """ + Run the telnet server, until this gets cancelled. + + :param ready_cb: Callback that will be called at the point that we're + actually listening. + """ + socket = self._create_socket(self.host, self.port) + logger.info( + "Listening for telnet connections on %s port %r", self.host, self.port + ) + + get_running_loop().add_reader(socket, lambda: self._accept(socket)) + + if ready_cb: + ready_cb() + + try: + # Run forever, until cancelled. + await asyncio.Future() + finally: + get_running_loop().remove_reader(socket) + socket.close() + + # Wait for all applications to finish. + for t in self._application_tasks: + t.cancel() + + # (This is similar to + # `Application.cancel_and_wait_for_background_tasks`. We wait for the + # background tasks to complete, but don't propagate exceptions, because + # we can't use `ExceptionGroup` yet.) + if len(self._application_tasks) > 0: + await asyncio.wait( + self._application_tasks, + timeout=None, + return_when=asyncio.ALL_COMPLETED, + ) + + def start(self) -> None: + """ + Deprecated: Use `.run()` instead. + + Start the telnet server (stop by calling and awaiting `stop()`). + """ + if self._run_task is not None: + # Already running. + return + + self._run_task = get_running_loop().create_task(self.run()) + + async def stop(self) -> None: + """ + Deprecated: Use `.run()` instead. + + Stop a telnet server that was started using `.start()` and wait for the + cancellation to complete. + """ + if self._run_task is not None: + self._run_task.cancel() + try: + await self._run_task + except asyncio.CancelledError: + pass + + def _accept(self, listen_socket: socket.socket) -> None: + """ + Accept new incoming connection. + """ + conn, addr = listen_socket.accept() + logger.info("New connection %r %r", *addr) + + # Run application for this connection. + async def run() -> None: + try: + with create_pipe_input() as vt100_input: + connection = TelnetConnection( + conn, + addr, + self.interact, + self, + encoding=self.encoding, + style=self.style, + vt100_input=vt100_input, + enable_cpr=self.enable_cpr, + ) + self.connections.add(connection) + + logger.info("Starting interaction %r %r", *addr) + try: + await connection.run_application() + finally: + self.connections.remove(connection) + logger.info("Stopping interaction %r %r", *addr) + except EOFError: + # Happens either when the connection is closed by the client + # (e.g., when the user types 'control-]', then 'quit' in the + # telnet client) or when the user types control-d in a prompt + # and this is not handled by the interact function. + logger.info("Unhandled EOFError in telnet application.") + except KeyboardInterrupt: + # Unhandled control-c propagated by a prompt. + logger.info("Unhandled KeyboardInterrupt in telnet application.") + except BaseException as e: + print(f"Got {type(e).__name__}", e) + import traceback + + traceback.print_exc() + finally: + self._application_tasks.remove(task) + + task = get_running_loop().create_task(run()) + self._application_tasks.append(task) diff --git a/lib/prompt_toolkit/cursor_shapes.py b/lib/prompt_toolkit/cursor_shapes.py new file mode 100644 index 0000000..01d1092 --- /dev/null +++ b/lib/prompt_toolkit/cursor_shapes.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from enum import Enum +from typing import TYPE_CHECKING, Any, Callable, Union + +from prompt_toolkit.enums import EditingMode +from prompt_toolkit.key_binding.vi_state import InputMode + +if TYPE_CHECKING: + from .application import Application + +__all__ = [ + "CursorShape", + "CursorShapeConfig", + "SimpleCursorShapeConfig", + "ModalCursorShapeConfig", + "DynamicCursorShapeConfig", + "to_cursor_shape_config", +] + + +class CursorShape(Enum): + # Default value that should tell the output implementation to never send + # cursor shape escape sequences. This is the default right now, because + # before this `CursorShape` functionality was introduced into + # prompt_toolkit itself, people had workarounds to send cursor shapes + # escapes into the terminal, by monkey patching some of prompt_toolkit's + # internals. We don't want the default prompt_toolkit implementation to + # interfere with that. E.g., IPython patches the `ViState.input_mode` + # property. See: https://github.com/ipython/ipython/pull/13501/files + _NEVER_CHANGE = "_NEVER_CHANGE" + + BLOCK = "BLOCK" + BEAM = "BEAM" + UNDERLINE = "UNDERLINE" + BLINKING_BLOCK = "BLINKING_BLOCK" + BLINKING_BEAM = "BLINKING_BEAM" + BLINKING_UNDERLINE = "BLINKING_UNDERLINE" + + +class CursorShapeConfig(ABC): + @abstractmethod + def get_cursor_shape(self, application: Application[Any]) -> CursorShape: + """ + Return the cursor shape to be used in the current state. + """ + + +AnyCursorShapeConfig = Union[CursorShape, CursorShapeConfig, None] + + +class SimpleCursorShapeConfig(CursorShapeConfig): + """ + Always show the given cursor shape. + """ + + def __init__(self, cursor_shape: CursorShape = CursorShape._NEVER_CHANGE) -> None: + self.cursor_shape = cursor_shape + + def get_cursor_shape(self, application: Application[Any]) -> CursorShape: + return self.cursor_shape + + +class ModalCursorShapeConfig(CursorShapeConfig): + """ + Show cursor shape according to the current input mode. + """ + + def get_cursor_shape(self, application: Application[Any]) -> CursorShape: + if application.editing_mode == EditingMode.VI: + if application.vi_state.input_mode in { + InputMode.NAVIGATION, + }: + return CursorShape.BLOCK + if application.vi_state.input_mode in { + InputMode.INSERT, + InputMode.INSERT_MULTIPLE, + }: + return CursorShape.BEAM + if application.vi_state.input_mode in { + InputMode.REPLACE, + InputMode.REPLACE_SINGLE, + }: + return CursorShape.UNDERLINE + elif application.editing_mode == EditingMode.EMACS: + # like vi's INSERT + return CursorShape.BEAM + + # Default + return CursorShape.BLOCK + + +class DynamicCursorShapeConfig(CursorShapeConfig): + def __init__( + self, get_cursor_shape_config: Callable[[], AnyCursorShapeConfig] + ) -> None: + self.get_cursor_shape_config = get_cursor_shape_config + + def get_cursor_shape(self, application: Application[Any]) -> CursorShape: + return to_cursor_shape_config(self.get_cursor_shape_config()).get_cursor_shape( + application + ) + + +def to_cursor_shape_config(value: AnyCursorShapeConfig) -> CursorShapeConfig: + """ + Take a `CursorShape` instance or `CursorShapeConfig` and turn it into a + `CursorShapeConfig`. + """ + if value is None: + return SimpleCursorShapeConfig() + + if isinstance(value, CursorShape): + return SimpleCursorShapeConfig(value) + + return value diff --git a/lib/prompt_toolkit/data_structures.py b/lib/prompt_toolkit/data_structures.py new file mode 100644 index 0000000..27dd458 --- /dev/null +++ b/lib/prompt_toolkit/data_structures.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from typing import NamedTuple + +__all__ = [ + "Point", + "Size", +] + + +class Point(NamedTuple): + x: int + y: int + + +class Size(NamedTuple): + rows: int + columns: int diff --git a/lib/prompt_toolkit/document.py b/lib/prompt_toolkit/document.py new file mode 100644 index 0000000..c9fbd9a --- /dev/null +++ b/lib/prompt_toolkit/document.py @@ -0,0 +1,1182 @@ +""" +The `Document` that implements all the text operations/querying. +""" + +from __future__ import annotations + +import bisect +import re +import string +import weakref +from typing import Callable, Dict, Iterable, List, NoReturn, Pattern, cast + +from .clipboard import ClipboardData +from .filters import vi_mode +from .selection import PasteMode, SelectionState, SelectionType + +__all__ = [ + "Document", +] + + +# Regex for finding "words" in documents. (We consider a group of alnum +# characters a word, but also a group of special characters a word, as long as +# it doesn't contain a space.) +# (This is a 'word' in Vi.) +_FIND_WORD_RE = re.compile(r"([a-zA-Z0-9_]+|[^a-zA-Z0-9_\s]+)") +_FIND_CURRENT_WORD_RE = re.compile(r"^([a-zA-Z0-9_]+|[^a-zA-Z0-9_\s]+)") +_FIND_CURRENT_WORD_INCLUDE_TRAILING_WHITESPACE_RE = re.compile( + r"^(([a-zA-Z0-9_]+|[^a-zA-Z0-9_\s]+)\s*)" +) + +# Regex for finding "WORDS" in documents. +# (This is a 'WORD in Vi.) +_FIND_BIG_WORD_RE = re.compile(r"([^\s]+)") +_FIND_CURRENT_BIG_WORD_RE = re.compile(r"^([^\s]+)") +_FIND_CURRENT_BIG_WORD_INCLUDE_TRAILING_WHITESPACE_RE = re.compile(r"^([^\s]+\s*)") + +# Share the Document._cache between all Document instances. +# (Document instances are considered immutable. That means that if another +# `Document` is constructed with the same text, it should have the same +# `_DocumentCache`.) +_text_to_document_cache: dict[str, _DocumentCache] = cast( + Dict[str, "_DocumentCache"], + weakref.WeakValueDictionary(), # Maps document.text to DocumentCache instance. +) + + +class _ImmutableLineList(List[str]): + """ + Some protection for our 'lines' list, which is assumed to be immutable in the cache. + (Useful for detecting obvious bugs.) + """ + + def _error(self, *a: object, **kw: object) -> NoReturn: + raise NotImplementedError("Attempt to modify an immutable list.") + + __setitem__ = _error + append = _error + clear = _error + extend = _error + insert = _error + pop = _error + remove = _error + reverse = _error + sort = _error + + +class _DocumentCache: + def __init__(self) -> None: + #: List of lines for the Document text. + self.lines: _ImmutableLineList | None = None + + #: List of index positions, pointing to the start of all the lines. + self.line_indexes: list[int] | None = None + + +class Document: + """ + This is a immutable class around the text and cursor position, and contains + methods for querying this data, e.g. to give the text before the cursor. + + This class is usually instantiated by a :class:`~prompt_toolkit.buffer.Buffer` + object, and accessed as the `document` property of that class. + + :param text: string + :param cursor_position: int + :param selection: :class:`.SelectionState` + """ + + __slots__ = ("_text", "_cursor_position", "_selection", "_cache") + + def __init__( + self, + text: str = "", + cursor_position: int | None = None, + selection: SelectionState | None = None, + ) -> None: + # Check cursor position. It can also be right after the end. (Where we + # insert text.) + assert cursor_position is None or cursor_position <= len(text), AssertionError( + f"cursor_position={cursor_position!r}, len_text={len(text)!r}" + ) + + # By default, if no cursor position was given, make sure to put the + # cursor position is at the end of the document. This is what makes + # sense in most places. + if cursor_position is None: + cursor_position = len(text) + + # Keep these attributes private. A `Document` really has to be + # considered to be immutable, because otherwise the caching will break + # things. Because of that, we wrap these into read-only properties. + self._text = text + self._cursor_position = cursor_position + self._selection = selection + + # Cache for lines/indexes. (Shared with other Document instances that + # contain the same text. + try: + self._cache = _text_to_document_cache[self.text] + except KeyError: + self._cache = _DocumentCache() + _text_to_document_cache[self.text] = self._cache + + # XX: For some reason, above, we can't use 'WeakValueDictionary.setdefault'. + # This fails in Pypy3. `self._cache` becomes None, because that's what + # 'setdefault' returns. + # self._cache = _text_to_document_cache.setdefault(self.text, _DocumentCache()) + # assert self._cache + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.text!r}, {self.cursor_position!r})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Document): + return False + + return ( + self.text == other.text + and self.cursor_position == other.cursor_position + and self.selection == other.selection + ) + + @property + def text(self) -> str: + "The document text." + return self._text + + @property + def cursor_position(self) -> int: + "The document cursor position." + return self._cursor_position + + @property + def selection(self) -> SelectionState | None: + ":class:`.SelectionState` object." + return self._selection + + @property + def current_char(self) -> str: + """Return character under cursor or an empty string.""" + return self._get_char_relative_to_cursor(0) or "" + + @property + def char_before_cursor(self) -> str: + """Return character before the cursor or an empty string.""" + return self._get_char_relative_to_cursor(-1) or "" + + @property + def text_before_cursor(self) -> str: + return self.text[: self.cursor_position :] + + @property + def text_after_cursor(self) -> str: + return self.text[self.cursor_position :] + + @property + def current_line_before_cursor(self) -> str: + """Text from the start of the line until the cursor.""" + _, _, text = self.text_before_cursor.rpartition("\n") + return text + + @property + def current_line_after_cursor(self) -> str: + """Text from the cursor until the end of the line.""" + text, _, _ = self.text_after_cursor.partition("\n") + return text + + @property + def lines(self) -> list[str]: + """ + Array of all the lines. + """ + # Cache, because this one is reused very often. + if self._cache.lines is None: + self._cache.lines = _ImmutableLineList(self.text.split("\n")) + + return self._cache.lines + + @property + def _line_start_indexes(self) -> list[int]: + """ + Array pointing to the start indexes of all the lines. + """ + # Cache, because this is often reused. (If it is used, it's often used + # many times. And this has to be fast for editing big documents!) + if self._cache.line_indexes is None: + # Create list of line lengths. + line_lengths = map(len, self.lines) + + # Calculate cumulative sums. + indexes = [0] + append = indexes.append + pos = 0 + + for line_length in line_lengths: + pos += line_length + 1 + append(pos) + + # Remove the last item. (This is not a new line.) + if len(indexes) > 1: + indexes.pop() + + self._cache.line_indexes = indexes + + return self._cache.line_indexes + + @property + def lines_from_current(self) -> list[str]: + """ + Array of the lines starting from the current line, until the last line. + """ + return self.lines[self.cursor_position_row :] + + @property + def line_count(self) -> int: + r"""Return the number of lines in this document. If the document ends + with a trailing \n, that counts as the beginning of a new line.""" + return len(self.lines) + + @property + def current_line(self) -> str: + """Return the text on the line where the cursor is. (when the input + consists of just one line, it equals `text`.""" + return self.current_line_before_cursor + self.current_line_after_cursor + + @property + def leading_whitespace_in_current_line(self) -> str: + """The leading whitespace in the left margin of the current line.""" + current_line = self.current_line + length = len(current_line) - len(current_line.lstrip()) + return current_line[:length] + + def _get_char_relative_to_cursor(self, offset: int = 0) -> str: + """ + Return character relative to cursor position, or empty string + """ + try: + return self.text[self.cursor_position + offset] + except IndexError: + return "" + + @property + def on_first_line(self) -> bool: + """ + True when we are at the first line. + """ + return self.cursor_position_row == 0 + + @property + def on_last_line(self) -> bool: + """ + True when we are at the last line. + """ + return self.cursor_position_row == self.line_count - 1 + + @property + def cursor_position_row(self) -> int: + """ + Current row. (0-based.) + """ + row, _ = self._find_line_start_index(self.cursor_position) + return row + + @property + def cursor_position_col(self) -> int: + """ + Current column. (0-based.) + """ + # (Don't use self.text_before_cursor to calculate this. Creating + # substrings and doing rsplit is too expensive for getting the cursor + # position.) + _, line_start_index = self._find_line_start_index(self.cursor_position) + return self.cursor_position - line_start_index + + def _find_line_start_index(self, index: int) -> tuple[int, int]: + """ + For the index of a character at a certain line, calculate the index of + the first character on that line. + + Return (row, index) tuple. + """ + indexes = self._line_start_indexes + + pos = bisect.bisect_right(indexes, index) - 1 + return pos, indexes[pos] + + def translate_index_to_position(self, index: int) -> tuple[int, int]: + """ + Given an index for the text, return the corresponding (row, col) tuple. + (0-based. Returns (0, 0) for index=0.) + """ + # Find start of this line. + row, row_index = self._find_line_start_index(index) + col = index - row_index + + return row, col + + def translate_row_col_to_index(self, row: int, col: int) -> int: + """ + Given a (row, col) tuple, return the corresponding index. + (Row and col params are 0-based.) + + Negative row/col values are turned into zero. + """ + try: + result = self._line_start_indexes[row] + line = self.lines[row] + except IndexError: + if row < 0: + result = self._line_start_indexes[0] + line = self.lines[0] + else: + result = self._line_start_indexes[-1] + line = self.lines[-1] + + result += max(0, min(col, len(line))) + + # Keep in range. (len(self.text) is included, because the cursor can be + # right after the end of the text as well.) + result = max(0, min(result, len(self.text))) + return result + + @property + def is_cursor_at_the_end(self) -> bool: + """True when the cursor is at the end of the text.""" + return self.cursor_position == len(self.text) + + @property + def is_cursor_at_the_end_of_line(self) -> bool: + """True when the cursor is at the end of this line.""" + return self.current_char in ("\n", "") + + def has_match_at_current_position(self, sub: str) -> bool: + """ + `True` when this substring is found at the cursor position. + """ + return self.text.find(sub, self.cursor_position) == self.cursor_position + + def find( + self, + sub: str, + in_current_line: bool = False, + include_current_position: bool = False, + ignore_case: bool = False, + count: int = 1, + ) -> int | None: + """ + Find `text` after the cursor, return position relative to the cursor + position. Return `None` if nothing was found. + + :param count: Find the n-th occurrence. + """ + assert isinstance(ignore_case, bool) + + if in_current_line: + text = self.current_line_after_cursor + else: + text = self.text_after_cursor + + if not include_current_position: + if len(text) == 0: + return None # (Otherwise, we always get a match for the empty string.) + else: + text = text[1:] + + flags = re.IGNORECASE if ignore_case else 0 + iterator = re.finditer(re.escape(sub), text, flags) + + try: + for i, match in enumerate(iterator): + if i + 1 == count: + if include_current_position: + return match.start(0) + else: + return match.start(0) + 1 + except StopIteration: + pass + return None + + def find_all(self, sub: str, ignore_case: bool = False) -> list[int]: + """ + Find all occurrences of the substring. Return a list of absolute + positions in the document. + """ + flags = re.IGNORECASE if ignore_case else 0 + return [a.start() for a in re.finditer(re.escape(sub), self.text, flags)] + + def find_backwards( + self, + sub: str, + in_current_line: bool = False, + ignore_case: bool = False, + count: int = 1, + ) -> int | None: + """ + Find `text` before the cursor, return position relative to the cursor + position. Return `None` if nothing was found. + + :param count: Find the n-th occurrence. + """ + if in_current_line: + before_cursor = self.current_line_before_cursor[::-1] + else: + before_cursor = self.text_before_cursor[::-1] + + flags = re.IGNORECASE if ignore_case else 0 + iterator = re.finditer(re.escape(sub[::-1]), before_cursor, flags) + + try: + for i, match in enumerate(iterator): + if i + 1 == count: + return -match.start(0) - len(sub) + except StopIteration: + pass + return None + + def get_word_before_cursor( + self, WORD: bool = False, pattern: Pattern[str] | None = None + ) -> str: + """ + Give the word before the cursor. + If we have whitespace before the cursor this returns an empty string. + + :param pattern: (None or compiled regex). When given, use this regex + pattern. + """ + if self._is_word_before_cursor_complete(WORD=WORD, pattern=pattern): + # Space before the cursor or no text before cursor. + return "" + + text_before_cursor = self.text_before_cursor + start = self.find_start_of_previous_word(WORD=WORD, pattern=pattern) or 0 + + return text_before_cursor[len(text_before_cursor) + start :] + + def _is_word_before_cursor_complete( + self, WORD: bool = False, pattern: Pattern[str] | None = None + ) -> bool: + if pattern: + return self.find_start_of_previous_word(WORD=WORD, pattern=pattern) is None + else: + return ( + self.text_before_cursor == "" or self.text_before_cursor[-1:].isspace() + ) + + def find_start_of_previous_word( + self, count: int = 1, WORD: bool = False, pattern: Pattern[str] | None = None + ) -> int | None: + """ + Return an index relative to the cursor position pointing to the start + of the previous word. Return `None` if nothing was found. + + :param pattern: (None or compiled regex). When given, use this regex + pattern. + """ + assert not (WORD and pattern) + + # Reverse the text before the cursor, in order to do an efficient + # backwards search. + text_before_cursor = self.text_before_cursor[::-1] + + if pattern: + regex = pattern + elif WORD: + regex = _FIND_BIG_WORD_RE + else: + regex = _FIND_WORD_RE + + iterator = regex.finditer(text_before_cursor) + + try: + for i, match in enumerate(iterator): + if i + 1 == count: + return -match.end(0) + except StopIteration: + pass + return None + + def find_boundaries_of_current_word( + self, + WORD: bool = False, + include_leading_whitespace: bool = False, + include_trailing_whitespace: bool = False, + ) -> tuple[int, int]: + """ + Return the relative boundaries (startpos, endpos) of the current word under the + cursor. (This is at the current line, because line boundaries obviously + don't belong to any word.) + If not on a word, this returns (0,0) + """ + text_before_cursor = self.current_line_before_cursor[::-1] + text_after_cursor = self.current_line_after_cursor + + def get_regex(include_whitespace: bool) -> Pattern[str]: + return { + (False, False): _FIND_CURRENT_WORD_RE, + (False, True): _FIND_CURRENT_WORD_INCLUDE_TRAILING_WHITESPACE_RE, + (True, False): _FIND_CURRENT_BIG_WORD_RE, + (True, True): _FIND_CURRENT_BIG_WORD_INCLUDE_TRAILING_WHITESPACE_RE, + }[(WORD, include_whitespace)] + + match_before = get_regex(include_leading_whitespace).search(text_before_cursor) + match_after = get_regex(include_trailing_whitespace).search(text_after_cursor) + + # When there is a match before and after, and we're not looking for + # WORDs, make sure that both the part before and after the cursor are + # either in the [a-zA-Z_] alphabet or not. Otherwise, drop the part + # before the cursor. + if not WORD and match_before and match_after: + c1 = self.text[self.cursor_position - 1] + c2 = self.text[self.cursor_position] + alphabet = string.ascii_letters + "0123456789_" + + if (c1 in alphabet) != (c2 in alphabet): + match_before = None + + return ( + -match_before.end(1) if match_before else 0, + match_after.end(1) if match_after else 0, + ) + + def get_word_under_cursor(self, WORD: bool = False) -> str: + """ + Return the word, currently below the cursor. + This returns an empty string when the cursor is on a whitespace region. + """ + start, end = self.find_boundaries_of_current_word(WORD=WORD) + return self.text[self.cursor_position + start : self.cursor_position + end] + + def find_next_word_beginning( + self, count: int = 1, WORD: bool = False + ) -> int | None: + """ + Return an index relative to the cursor position pointing to the start + of the next word. Return `None` if nothing was found. + """ + if count < 0: + return self.find_previous_word_beginning(count=-count, WORD=WORD) + + regex = _FIND_BIG_WORD_RE if WORD else _FIND_WORD_RE + iterator = regex.finditer(self.text_after_cursor) + + try: + for i, match in enumerate(iterator): + # Take first match, unless it's the word on which we're right now. + if i == 0 and match.start(1) == 0: + count += 1 + + if i + 1 == count: + return match.start(1) + except StopIteration: + pass + return None + + def find_next_word_ending( + self, include_current_position: bool = False, count: int = 1, WORD: bool = False + ) -> int | None: + """ + Return an index relative to the cursor position pointing to the end + of the next word. Return `None` if nothing was found. + """ + if count < 0: + return self.find_previous_word_ending(count=-count, WORD=WORD) + + if include_current_position: + text = self.text_after_cursor + else: + text = self.text_after_cursor[1:] + + regex = _FIND_BIG_WORD_RE if WORD else _FIND_WORD_RE + iterable = regex.finditer(text) + + try: + for i, match in enumerate(iterable): + if i + 1 == count: + value = match.end(1) + + if include_current_position: + return value + else: + return value + 1 + + except StopIteration: + pass + return None + + def find_previous_word_beginning( + self, count: int = 1, WORD: bool = False + ) -> int | None: + """ + Return an index relative to the cursor position pointing to the start + of the previous word. Return `None` if nothing was found. + """ + if count < 0: + return self.find_next_word_beginning(count=-count, WORD=WORD) + + regex = _FIND_BIG_WORD_RE if WORD else _FIND_WORD_RE + iterator = regex.finditer(self.text_before_cursor[::-1]) + + try: + for i, match in enumerate(iterator): + if i + 1 == count: + return -match.end(1) + except StopIteration: + pass + return None + + def find_previous_word_ending( + self, count: int = 1, WORD: bool = False + ) -> int | None: + """ + Return an index relative to the cursor position pointing to the end + of the previous word. Return `None` if nothing was found. + """ + if count < 0: + return self.find_next_word_ending(count=-count, WORD=WORD) + + text_before_cursor = self.text_after_cursor[:1] + self.text_before_cursor[::-1] + + regex = _FIND_BIG_WORD_RE if WORD else _FIND_WORD_RE + iterator = regex.finditer(text_before_cursor) + + try: + for i, match in enumerate(iterator): + # Take first match, unless it's the word on which we're right now. + if i == 0 and match.start(1) == 0: + count += 1 + + if i + 1 == count: + return -match.start(1) + 1 + except StopIteration: + pass + return None + + def find_next_matching_line( + self, match_func: Callable[[str], bool], count: int = 1 + ) -> int | None: + """ + Look downwards for empty lines. + Return the line index, relative to the current line. + """ + result = None + + for index, line in enumerate(self.lines[self.cursor_position_row + 1 :]): + if match_func(line): + result = 1 + index + count -= 1 + + if count == 0: + break + + return result + + def find_previous_matching_line( + self, match_func: Callable[[str], bool], count: int = 1 + ) -> int | None: + """ + Look upwards for empty lines. + Return the line index, relative to the current line. + """ + result = None + + for index, line in enumerate(self.lines[: self.cursor_position_row][::-1]): + if match_func(line): + result = -1 - index + count -= 1 + + if count == 0: + break + + return result + + def get_cursor_left_position(self, count: int = 1) -> int: + """ + Relative position for cursor left. + """ + if count < 0: + return self.get_cursor_right_position(-count) + + return -min(self.cursor_position_col, count) + + def get_cursor_right_position(self, count: int = 1) -> int: + """ + Relative position for cursor_right. + """ + if count < 0: + return self.get_cursor_left_position(-count) + + return min(count, len(self.current_line_after_cursor)) + + def get_cursor_up_position( + self, count: int = 1, preferred_column: int | None = None + ) -> int: + """ + Return the relative cursor position (character index) where we would be if the + user pressed the arrow-up button. + + :param preferred_column: When given, go to this column instead of + staying at the current column. + """ + assert count >= 1 + column = ( + self.cursor_position_col if preferred_column is None else preferred_column + ) + + return ( + self.translate_row_col_to_index( + max(0, self.cursor_position_row - count), column + ) + - self.cursor_position + ) + + def get_cursor_down_position( + self, count: int = 1, preferred_column: int | None = None + ) -> int: + """ + Return the relative cursor position (character index) where we would be if the + user pressed the arrow-down button. + + :param preferred_column: When given, go to this column instead of + staying at the current column. + """ + assert count >= 1 + column = ( + self.cursor_position_col if preferred_column is None else preferred_column + ) + + return ( + self.translate_row_col_to_index(self.cursor_position_row + count, column) + - self.cursor_position + ) + + def find_enclosing_bracket_right( + self, left_ch: str, right_ch: str, end_pos: int | None = None + ) -> int | None: + """ + Find the right bracket enclosing current position. Return the relative + position to the cursor position. + + When `end_pos` is given, don't look past the position. + """ + if self.current_char == right_ch: + return 0 + + if end_pos is None: + end_pos = len(self.text) + else: + end_pos = min(len(self.text), end_pos) + + stack = 1 + + # Look forward. + for i in range(self.cursor_position + 1, end_pos): + c = self.text[i] + + if c == left_ch: + stack += 1 + elif c == right_ch: + stack -= 1 + + if stack == 0: + return i - self.cursor_position + + return None + + def find_enclosing_bracket_left( + self, left_ch: str, right_ch: str, start_pos: int | None = None + ) -> int | None: + """ + Find the left bracket enclosing current position. Return the relative + position to the cursor position. + + When `start_pos` is given, don't look past the position. + """ + if self.current_char == left_ch: + return 0 + + if start_pos is None: + start_pos = 0 + else: + start_pos = max(0, start_pos) + + stack = 1 + + # Look backward. + for i in range(self.cursor_position - 1, start_pos - 1, -1): + c = self.text[i] + + if c == right_ch: + stack += 1 + elif c == left_ch: + stack -= 1 + + if stack == 0: + return i - self.cursor_position + + return None + + def find_matching_bracket_position( + self, start_pos: int | None = None, end_pos: int | None = None + ) -> int: + """ + Return relative cursor position of matching [, (, { or < bracket. + + When `start_pos` or `end_pos` are given. Don't look past the positions. + """ + + # Look for a match. + for pair in "()", "[]", "{}", "<>": + A = pair[0] + B = pair[1] + if self.current_char == A: + return self.find_enclosing_bracket_right(A, B, end_pos=end_pos) or 0 + elif self.current_char == B: + return self.find_enclosing_bracket_left(A, B, start_pos=start_pos) or 0 + + return 0 + + def get_start_of_document_position(self) -> int: + """Relative position for the start of the document.""" + return -self.cursor_position + + def get_end_of_document_position(self) -> int: + """Relative position for the end of the document.""" + return len(self.text) - self.cursor_position + + def get_start_of_line_position(self, after_whitespace: bool = False) -> int: + """Relative position for the start of this line.""" + if after_whitespace: + current_line = self.current_line + return ( + len(current_line) + - len(current_line.lstrip()) + - self.cursor_position_col + ) + else: + return -len(self.current_line_before_cursor) + + def get_end_of_line_position(self) -> int: + """Relative position for the end of this line.""" + return len(self.current_line_after_cursor) + + def last_non_blank_of_current_line_position(self) -> int: + """ + Relative position for the last non blank character of this line. + """ + return len(self.current_line.rstrip()) - self.cursor_position_col - 1 + + def get_column_cursor_position(self, column: int) -> int: + """ + Return the relative cursor position for this column at the current + line. (It will stay between the boundaries of the line in case of a + larger number.) + """ + line_length = len(self.current_line) + current_column = self.cursor_position_col + column = max(0, min(line_length, column)) + + return column - current_column + + def selection_range( + self, + ) -> tuple[ + int, int + ]: # XXX: shouldn't this return `None` if there is no selection??? + """ + Return (from, to) tuple of the selection. + start and end position are included. + + This doesn't take the selection type into account. Use + `selection_ranges` instead. + """ + if self.selection: + from_, to = sorted( + [self.cursor_position, self.selection.original_cursor_position] + ) + else: + from_, to = self.cursor_position, self.cursor_position + + return from_, to + + def selection_ranges(self) -> Iterable[tuple[int, int]]: + """ + Return a list of `(from, to)` tuples for the selection or none if + nothing was selected. The upper boundary is not included. + + This will yield several (from, to) tuples in case of a BLOCK selection. + This will return zero ranges, like (8,8) for empty lines in a block + selection. + """ + if self.selection: + from_, to = sorted( + [self.cursor_position, self.selection.original_cursor_position] + ) + + if self.selection.type == SelectionType.BLOCK: + from_line, from_column = self.translate_index_to_position(from_) + to_line, to_column = self.translate_index_to_position(to) + from_column, to_column = sorted([from_column, to_column]) + lines = self.lines + + if vi_mode(): + to_column += 1 + + for l in range(from_line, to_line + 1): + line_length = len(lines[l]) + + if from_column <= line_length: + yield ( + self.translate_row_col_to_index(l, from_column), + self.translate_row_col_to_index( + l, min(line_length, to_column) + ), + ) + else: + # In case of a LINES selection, go to the start/end of the lines. + if self.selection.type == SelectionType.LINES: + from_ = max(0, self.text.rfind("\n", 0, from_) + 1) + + if self.text.find("\n", to) >= 0: + to = self.text.find("\n", to) + else: + to = len(self.text) - 1 + + # In Vi mode, the upper boundary is always included. For Emacs, + # that's not the case. + if vi_mode(): + to += 1 + + yield from_, to + + def selection_range_at_line(self, row: int) -> tuple[int, int] | None: + """ + If the selection spans a portion of the given line, return a (from, to) tuple. + + The returned upper boundary is not included in the selection, so + `(0, 0)` is an empty selection. `(0, 1)`, is a one character selection. + + Returns None if the selection doesn't cover this line at all. + """ + if self.selection: + line = self.lines[row] + + row_start = self.translate_row_col_to_index(row, 0) + row_end = self.translate_row_col_to_index(row, len(line)) + + from_, to = sorted( + [self.cursor_position, self.selection.original_cursor_position] + ) + + # Take the intersection of the current line and the selection. + intersection_start = max(row_start, from_) + intersection_end = min(row_end, to) + + if intersection_start <= intersection_end: + if self.selection.type == SelectionType.LINES: + intersection_start = row_start + intersection_end = row_end + + elif self.selection.type == SelectionType.BLOCK: + _, col1 = self.translate_index_to_position(from_) + _, col2 = self.translate_index_to_position(to) + col1, col2 = sorted([col1, col2]) + + if col1 > len(line): + return None # Block selection doesn't cross this line. + + intersection_start = self.translate_row_col_to_index(row, col1) + intersection_end = self.translate_row_col_to_index(row, col2) + + _, from_column = self.translate_index_to_position(intersection_start) + _, to_column = self.translate_index_to_position(intersection_end) + + # In Vi mode, the upper boundary is always included. For Emacs + # mode, that's not the case. + if vi_mode(): + to_column += 1 + + return from_column, to_column + return None + + def cut_selection(self) -> tuple[Document, ClipboardData]: + """ + Return a (:class:`.Document`, :class:`.ClipboardData`) tuple, where the + document represents the new document when the selection is cut, and the + clipboard data, represents whatever has to be put on the clipboard. + """ + if self.selection: + cut_parts = [] + remaining_parts = [] + new_cursor_position = self.cursor_position + + last_to = 0 + for from_, to in self.selection_ranges(): + if last_to == 0: + new_cursor_position = from_ + + remaining_parts.append(self.text[last_to:from_]) + cut_parts.append(self.text[from_:to]) + last_to = to + + remaining_parts.append(self.text[last_to:]) + + cut_text = "\n".join(cut_parts) + remaining_text = "".join(remaining_parts) + + # In case of a LINES selection, don't include the trailing newline. + if self.selection.type == SelectionType.LINES and cut_text.endswith("\n"): + cut_text = cut_text[:-1] + + return ( + Document(text=remaining_text, cursor_position=new_cursor_position), + ClipboardData(cut_text, self.selection.type), + ) + else: + return self, ClipboardData("") + + def paste_clipboard_data( + self, + data: ClipboardData, + paste_mode: PasteMode = PasteMode.EMACS, + count: int = 1, + ) -> Document: + """ + Return a new :class:`.Document` instance which contains the result if + we would paste this data at the current cursor position. + + :param paste_mode: Where to paste. (Before/after/emacs.) + :param count: When >1, Paste multiple times. + """ + before = paste_mode == PasteMode.VI_BEFORE + after = paste_mode == PasteMode.VI_AFTER + + if data.type == SelectionType.CHARACTERS: + if after: + new_text = ( + self.text[: self.cursor_position + 1] + + data.text * count + + self.text[self.cursor_position + 1 :] + ) + else: + new_text = ( + self.text_before_cursor + data.text * count + self.text_after_cursor + ) + + new_cursor_position = self.cursor_position + len(data.text) * count + if before: + new_cursor_position -= 1 + + elif data.type == SelectionType.LINES: + l = self.cursor_position_row + if before: + lines = self.lines[:l] + [data.text] * count + self.lines[l:] + new_text = "\n".join(lines) + new_cursor_position = len("".join(self.lines[:l])) + l + else: + lines = self.lines[: l + 1] + [data.text] * count + self.lines[l + 1 :] + new_cursor_position = len("".join(self.lines[: l + 1])) + l + 1 + new_text = "\n".join(lines) + + elif data.type == SelectionType.BLOCK: + lines = self.lines[:] + start_line = self.cursor_position_row + start_column = self.cursor_position_col + (0 if before else 1) + + for i, line in enumerate(data.text.split("\n")): + index = i + start_line + if index >= len(lines): + lines.append("") + + lines[index] = lines[index].ljust(start_column) + lines[index] = ( + lines[index][:start_column] + + line * count + + lines[index][start_column:] + ) + + new_text = "\n".join(lines) + new_cursor_position = self.cursor_position + (0 if before else 1) + + return Document(text=new_text, cursor_position=new_cursor_position) + + def empty_line_count_at_the_end(self) -> int: + """ + Return number of empty lines at the end of the document. + """ + count = 0 + for line in self.lines[::-1]: + if not line or line.isspace(): + count += 1 + else: + break + + return count + + def start_of_paragraph(self, count: int = 1, before: bool = False) -> int: + """ + Return the start of the current paragraph. (Relative cursor position.) + """ + + def match_func(text: str) -> bool: + return not text or text.isspace() + + line_index = self.find_previous_matching_line( + match_func=match_func, count=count + ) + + if line_index: + add = 0 if before else 1 + return min(0, self.get_cursor_up_position(count=-line_index) + add) + else: + return -self.cursor_position + + def end_of_paragraph(self, count: int = 1, after: bool = False) -> int: + """ + Return the end of the current paragraph. (Relative cursor position.) + """ + + def match_func(text: str) -> bool: + return not text or text.isspace() + + line_index = self.find_next_matching_line(match_func=match_func, count=count) + + if line_index: + add = 0 if after else 1 + return max(0, self.get_cursor_down_position(count=line_index) - add) + else: + return len(self.text_after_cursor) + + # Modifiers. + + def insert_after(self, text: str) -> Document: + """ + Create a new document, with this text inserted after the buffer. + It keeps selection ranges and cursor position in sync. + """ + return Document( + text=self.text + text, + cursor_position=self.cursor_position, + selection=self.selection, + ) + + def insert_before(self, text: str) -> Document: + """ + Create a new document, with this text inserted before the buffer. + It keeps selection ranges and cursor position in sync. + """ + selection_state = self.selection + + if selection_state: + selection_state = SelectionState( + original_cursor_position=selection_state.original_cursor_position + + len(text), + type=selection_state.type, + ) + + return Document( + text=text + self.text, + cursor_position=self.cursor_position + len(text), + selection=selection_state, + ) diff --git a/lib/prompt_toolkit/enums.py b/lib/prompt_toolkit/enums.py new file mode 100644 index 0000000..da03633 --- /dev/null +++ b/lib/prompt_toolkit/enums.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from enum import Enum + + +class EditingMode(Enum): + # The set of key bindings that is active. + VI = "VI" + EMACS = "EMACS" + + +#: Name of the search buffer. +SEARCH_BUFFER = "SEARCH_BUFFER" + +#: Name of the default buffer. +DEFAULT_BUFFER = "DEFAULT_BUFFER" + +#: Name of the system buffer. +SYSTEM_BUFFER = "SYSTEM_BUFFER" diff --git a/lib/prompt_toolkit/eventloop/__init__.py b/lib/prompt_toolkit/eventloop/__init__.py new file mode 100644 index 0000000..5df623b --- /dev/null +++ b/lib/prompt_toolkit/eventloop/__init__.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from .async_generator import aclosing, generator_to_async_generator +from .inputhook import ( + InputHook, + InputHookContext, + InputHookSelector, + new_eventloop_with_inputhook, + set_eventloop_with_inputhook, +) +from .utils import ( + call_soon_threadsafe, + get_traceback_from_context, + run_in_executor_with_context, +) + +__all__ = [ + # Async generator + "generator_to_async_generator", + "aclosing", + # Utils. + "run_in_executor_with_context", + "call_soon_threadsafe", + "get_traceback_from_context", + # Inputhooks. + "InputHook", + "new_eventloop_with_inputhook", + "set_eventloop_with_inputhook", + "InputHookSelector", + "InputHookContext", +] diff --git a/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..631ac76 Binary files /dev/null and b/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc b/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc new file mode 100644 index 0000000..74a5bf0 Binary files /dev/null and b/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc b/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc new file mode 100644 index 0000000..e5c05e7 Binary files /dev/null and b/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc b/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..c92b6cb Binary files /dev/null and b/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc b/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc new file mode 100644 index 0000000..287670d Binary files /dev/null and b/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/eventloop/async_generator.py b/lib/prompt_toolkit/eventloop/async_generator.py new file mode 100644 index 0000000..32e5f88 --- /dev/null +++ b/lib/prompt_toolkit/eventloop/async_generator.py @@ -0,0 +1,125 @@ +""" +Implementation for async generators. +""" + +from __future__ import annotations + +from asyncio import get_running_loop +from contextlib import asynccontextmanager +from queue import Empty, Full, Queue +from typing import Any, AsyncGenerator, Callable, Iterable, TypeVar + +from .utils import run_in_executor_with_context + +__all__ = [ + "aclosing", + "generator_to_async_generator", +] + +_T_Generator = TypeVar("_T_Generator", bound=AsyncGenerator[Any, None]) + + +@asynccontextmanager +async def aclosing( + thing: _T_Generator, +) -> AsyncGenerator[_T_Generator, None]: + "Similar to `contextlib.aclosing`, in Python 3.10." + try: + yield thing + finally: + await thing.aclose() + + +# By default, choose a buffer size that's a good balance between having enough +# throughput, but not consuming too much memory. We use this to consume a sync +# generator of completions as an async generator. If the queue size is very +# small (like 1), consuming the completions goes really slow (when there are a +# lot of items). If the queue size would be unlimited or too big, this can +# cause overconsumption of memory, and cause CPU time spent producing items +# that are no longer needed (if the consumption of the async generator stops at +# some point). We need a fixed size in order to get some back pressure from the +# async consumer to the sync producer. We choose 1000 by default here. If we +# have around 50k completions, measurements show that 1000 is still +# significantly faster than a buffer of 100. +DEFAULT_BUFFER_SIZE: int = 1000 + +_T = TypeVar("_T") + + +class _Done: + pass + + +async def generator_to_async_generator( + get_iterable: Callable[[], Iterable[_T]], + buffer_size: int = DEFAULT_BUFFER_SIZE, +) -> AsyncGenerator[_T, None]: + """ + Turn a generator or iterable into an async generator. + + This works by running the generator in a background thread. + + :param get_iterable: Function that returns a generator or iterable when + called. + :param buffer_size: Size of the queue between the async consumer and the + synchronous generator that produces items. + """ + quitting = False + # NOTE: We are limiting the queue size in order to have back-pressure. + q: Queue[_T | _Done] = Queue(maxsize=buffer_size) + loop = get_running_loop() + + def runner() -> None: + """ + Consume the generator in background thread. + When items are received, they'll be pushed to the queue. + """ + try: + for item in get_iterable(): + # When this async generator was cancelled (closed), stop this + # thread. + if quitting: + return + + while True: + try: + q.put(item, timeout=1) + except Full: + if quitting: + return + continue + else: + break + + finally: + while True: + try: + q.put(_Done(), timeout=1) + except Full: + if quitting: + return + continue + else: + break + + # Start background thread. + runner_f = run_in_executor_with_context(runner) + + try: + while True: + try: + item = q.get_nowait() + except Empty: + item = await loop.run_in_executor(None, q.get) + if isinstance(item, _Done): + break + else: + yield item + finally: + # When this async generator is closed (GeneratorExit exception, stop + # the background thread as well. - we don't need that anymore.) + quitting = True + + # Wait for the background thread to finish. (should happen right after + # the last item is yielded). + await runner_f diff --git a/lib/prompt_toolkit/eventloop/inputhook.py b/lib/prompt_toolkit/eventloop/inputhook.py new file mode 100644 index 0000000..40016e8 --- /dev/null +++ b/lib/prompt_toolkit/eventloop/inputhook.py @@ -0,0 +1,191 @@ +""" +Similar to `PyOS_InputHook` of the Python API, we can plug in an input hook in +the asyncio event loop. + +The way this works is by using a custom 'selector' that runs the other event +loop until the real selector is ready. + +It's the responsibility of this event hook to return when there is input ready. +There are two ways to detect when input is ready: + +The inputhook itself is a callable that receives an `InputHookContext`. This +callable should run the other event loop, and return when the main loop has +stuff to do. There are two ways to detect when to return: + +- Call the `input_is_ready` method periodically. Quit when this returns `True`. + +- Add the `fileno` as a watch to the external eventloop. Quit when file descriptor + becomes readable. (But don't read from it.) + + Note that this is not the same as checking for `sys.stdin.fileno()`. The + eventloop of prompt-toolkit allows thread-based executors, for example for + asynchronous autocompletion. When the completion for instance is ready, we + also want prompt-toolkit to gain control again in order to display that. +""" + +from __future__ import annotations + +import asyncio +import os +import select +import selectors +import sys +import threading +from asyncio import AbstractEventLoop, get_running_loop +from selectors import BaseSelector, SelectorKey +from typing import TYPE_CHECKING, Any, Callable, Mapping + +__all__ = [ + "new_eventloop_with_inputhook", + "set_eventloop_with_inputhook", + "InputHookSelector", + "InputHookContext", + "InputHook", +] + +if TYPE_CHECKING: + from _typeshed import FileDescriptorLike + from typing_extensions import TypeAlias + + _EventMask = int + + +class InputHookContext: + """ + Given as a parameter to the inputhook. + """ + + def __init__(self, fileno: int, input_is_ready: Callable[[], bool]) -> None: + self._fileno = fileno + self.input_is_ready = input_is_ready + + def fileno(self) -> int: + return self._fileno + + +InputHook: TypeAlias = Callable[[InputHookContext], None] + + +def new_eventloop_with_inputhook( + inputhook: Callable[[InputHookContext], None], +) -> AbstractEventLoop: + """ + Create a new event loop with the given inputhook. + """ + selector = InputHookSelector(selectors.DefaultSelector(), inputhook) + loop = asyncio.SelectorEventLoop(selector) + return loop + + +def set_eventloop_with_inputhook( + inputhook: Callable[[InputHookContext], None], +) -> AbstractEventLoop: + """ + Create a new event loop with the given inputhook, and activate it. + """ + # Deprecated! + + loop = new_eventloop_with_inputhook(inputhook) + asyncio.set_event_loop(loop) + return loop + + +class InputHookSelector(BaseSelector): + """ + Usage: + + selector = selectors.SelectSelector() + loop = asyncio.SelectorEventLoop(InputHookSelector(selector, inputhook)) + asyncio.set_event_loop(loop) + """ + + def __init__( + self, selector: BaseSelector, inputhook: Callable[[InputHookContext], None] + ) -> None: + self.selector = selector + self.inputhook = inputhook + self._r, self._w = os.pipe() + + def register( + self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None + ) -> SelectorKey: + return self.selector.register(fileobj, events, data=data) + + def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: + return self.selector.unregister(fileobj) + + def modify( + self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None + ) -> SelectorKey: + return self.selector.modify(fileobj, events, data=None) + + def select( + self, timeout: float | None = None + ) -> list[tuple[SelectorKey, _EventMask]]: + # If there are tasks in the current event loop, + # don't run the input hook. + if len(getattr(get_running_loop(), "_ready", [])) > 0: + return self.selector.select(timeout=timeout) + + ready = False + result = None + + # Run selector in other thread. + def run_selector() -> None: + nonlocal ready, result + result = self.selector.select(timeout=timeout) + os.write(self._w, b"x") + ready = True + + th = threading.Thread(target=run_selector) + th.start() + + def input_is_ready() -> bool: + return ready + + # Call inputhook. + # The inputhook function is supposed to return when our selector + # becomes ready. The inputhook can do that by registering the fd in its + # own loop, or by checking the `input_is_ready` function regularly. + self.inputhook(InputHookContext(self._r, input_is_ready)) + + # Flush the read end of the pipe. + try: + # Before calling 'os.read', call select.select. This is required + # when the gevent monkey patch has been applied. 'os.read' is never + # monkey patched and won't be cooperative, so that would block all + # other select() calls otherwise. + # See: http://www.gevent.org/gevent.os.html + + # Note: On Windows, this is apparently not an issue. + # However, if we would ever want to add a select call, it + # should use `windll.kernel32.WaitForMultipleObjects`, + # because `select.select` can't wait for a pipe on Windows. + if sys.platform != "win32": + select.select([self._r], [], [], None) + + os.read(self._r, 1024) + except OSError: + # This happens when the window resizes and a SIGWINCH was received. + # We get 'Error: [Errno 4] Interrupted system call' + # Just ignore. + pass + + # Wait for the real selector to be done. + th.join() + assert result is not None + return result + + def close(self) -> None: + """ + Clean up resources. + """ + if self._r: + os.close(self._r) + os.close(self._w) + + self._r = self._w = -1 + self.selector.close() + + def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: + return self.selector.get_map() diff --git a/lib/prompt_toolkit/eventloop/utils.py b/lib/prompt_toolkit/eventloop/utils.py new file mode 100644 index 0000000..3138361 --- /dev/null +++ b/lib/prompt_toolkit/eventloop/utils.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import asyncio +import contextvars +import sys +import time +from asyncio import get_running_loop +from types import TracebackType +from typing import Any, Awaitable, Callable, TypeVar, cast + +__all__ = [ + "run_in_executor_with_context", + "call_soon_threadsafe", + "get_traceback_from_context", +] + +_T = TypeVar("_T") + + +def run_in_executor_with_context( + func: Callable[..., _T], + *args: Any, + loop: asyncio.AbstractEventLoop | None = None, +) -> Awaitable[_T]: + """ + Run a function in an executor, but make sure it uses the same contextvars. + This is required so that the function will see the right application. + + See also: https://bugs.python.org/issue34014 + """ + loop = loop or get_running_loop() + ctx: contextvars.Context = contextvars.copy_context() + + return loop.run_in_executor(None, ctx.run, func, *args) + + +def call_soon_threadsafe( + func: Callable[[], None], + max_postpone_time: float | None = None, + loop: asyncio.AbstractEventLoop | None = None, +) -> None: + """ + Wrapper around asyncio's `call_soon_threadsafe`. + + This takes a `max_postpone_time` which can be used to tune the urgency of + the method. + + Asyncio runs tasks in first-in-first-out. However, this is not what we + want for the render function of the prompt_toolkit UI. Rendering is + expensive, but since the UI is invalidated very often, in some situations + we render the UI too often, so much that the rendering CPU usage slows down + the rest of the processing of the application. (Pymux is an example where + we have to balance the CPU time spend on rendering the UI, and parsing + process output.) + However, we want to set a deadline value, for when the rendering should + happen. (The UI should stay responsive). + """ + loop2 = loop or get_running_loop() + + # If no `max_postpone_time` has been given, schedule right now. + if max_postpone_time is None: + loop2.call_soon_threadsafe(func) + return + + max_postpone_until = time.time() + max_postpone_time + + def schedule() -> None: + # When there are no other tasks scheduled in the event loop. Run it + # now. + # Notice: uvloop doesn't have this _ready attribute. In that case, + # always call immediately. + if not getattr(loop2, "_ready", []): + func() + return + + # If the timeout expired, run this now. + if time.time() > max_postpone_until: + func() + return + + # Schedule again for later. + loop2.call_soon_threadsafe(schedule) + + loop2.call_soon_threadsafe(schedule) + + +def get_traceback_from_context(context: dict[str, Any]) -> TracebackType | None: + """ + Get the traceback object from the context. + """ + exception = context.get("exception") + if exception: + if hasattr(exception, "__traceback__"): + return cast(TracebackType, exception.__traceback__) + else: + # call_exception_handler() is usually called indirectly + # from an except block. If it's not the case, the traceback + # is undefined... + return sys.exc_info()[2] + + return None diff --git a/lib/prompt_toolkit/eventloop/win32.py b/lib/prompt_toolkit/eventloop/win32.py new file mode 100644 index 0000000..56a0c7d --- /dev/null +++ b/lib/prompt_toolkit/eventloop/win32.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import sys + +assert sys.platform == "win32" + +from ctypes import pointer + +from ..utils import SPHINX_AUTODOC_RUNNING + +# Do not import win32-specific stuff when generating documentation. +# Otherwise RTD would be unable to generate docs for this module. +if not SPHINX_AUTODOC_RUNNING: + from ctypes import windll + +from ctypes.wintypes import BOOL, DWORD, HANDLE + +from prompt_toolkit.win32_types import SECURITY_ATTRIBUTES + +__all__ = ["wait_for_handles", "create_win32_event"] + + +WAIT_TIMEOUT = 0x00000102 +INFINITE = -1 + + +def wait_for_handles(handles: list[HANDLE], timeout: int = INFINITE) -> HANDLE | None: + """ + Waits for multiple handles. (Similar to 'select') Returns the handle which is ready. + Returns `None` on timeout. + http://msdn.microsoft.com/en-us/library/windows/desktop/ms687025(v=vs.85).aspx + + Note that handles should be a list of `HANDLE` objects, not integers. See + this comment in the patch by @quark-zju for the reason why: + + ''' Make sure HANDLE on Windows has a correct size + + Previously, the type of various HANDLEs are native Python integer + types. The ctypes library will treat them as 4-byte integer when used + in function arguments. On 64-bit Windows, HANDLE is 8-byte and usually + a small integer. Depending on whether the extra 4 bytes are zero-ed out + or not, things can happen to work, or break. ''' + + This function returns either `None` or one of the given `HANDLE` objects. + (The return value can be tested with the `is` operator.) + """ + arrtype = HANDLE * len(handles) + handle_array = arrtype(*handles) + + ret: int = windll.kernel32.WaitForMultipleObjects( + len(handle_array), handle_array, BOOL(False), DWORD(timeout) + ) + + if ret == WAIT_TIMEOUT: + return None + else: + return handles[ret] + + +def create_win32_event() -> HANDLE: + """ + Creates a Win32 unnamed Event . + http://msdn.microsoft.com/en-us/library/windows/desktop/ms682396(v=vs.85).aspx + """ + return HANDLE( + windll.kernel32.CreateEventA( + pointer(SECURITY_ATTRIBUTES()), + BOOL(True), # Manual reset event. + BOOL(False), # Initial state. + None, # Unnamed event object. + ) + ) diff --git a/lib/prompt_toolkit/filters/__init__.py b/lib/prompt_toolkit/filters/__init__.py new file mode 100644 index 0000000..556ed88 --- /dev/null +++ b/lib/prompt_toolkit/filters/__init__.py @@ -0,0 +1,71 @@ +""" +Filters decide whether something is active or not (they decide about a boolean +state). This is used to enable/disable features, like key bindings, parts of +the layout and other stuff. For instance, we could have a `HasSearch` filter +attached to some part of the layout, in order to show that part of the user +interface only while the user is searching. + +Filters are made to avoid having to attach callbacks to all event in order to +propagate state. However, they are lazy, they don't automatically propagate the +state of what they are observing. Only when a filter is called (it's actually a +callable), it will calculate its value. So, its not really reactive +programming, but it's made to fit for this framework. + +Filters can be chained using ``&`` and ``|`` operations, and inverted using the +``~`` operator, for instance:: + + filter = has_focus('default') & ~ has_selection +""" + +from __future__ import annotations + +from .app import * +from .base import Always, Condition, Filter, FilterOrBool, Never +from .cli import * +from .utils import is_true, to_filter + +__all__ = [ + # app + "has_arg", + "has_completions", + "completion_is_selected", + "has_focus", + "buffer_has_focus", + "has_selection", + "has_validation_error", + "is_done", + "is_read_only", + "is_multiline", + "renderer_height_is_known", + "in_editing_mode", + "in_paste_mode", + "vi_mode", + "vi_navigation_mode", + "vi_insert_mode", + "vi_insert_multiple_mode", + "vi_replace_mode", + "vi_selection_mode", + "vi_waiting_for_text_object_mode", + "vi_digraph_mode", + "vi_recording_macro", + "emacs_mode", + "emacs_insert_mode", + "emacs_selection_mode", + "shift_selection_mode", + "is_searching", + "control_is_searchable", + "vi_search_direction_reversed", + # base. + "Filter", + "Never", + "Always", + "Condition", + "FilterOrBool", + # utils. + "is_true", + "to_filter", +] + +from .cli import __all__ as cli_all + +__all__.extend(cli_all) diff --git a/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..711f826 Binary files /dev/null and b/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc b/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc new file mode 100644 index 0000000..704d37a Binary files /dev/null and b/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..1908b6b Binary files /dev/null and b/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc b/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc new file mode 100644 index 0000000..eaba9d0 Binary files /dev/null and b/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc b/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..9fb9664 Binary files /dev/null and b/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/filters/app.py b/lib/prompt_toolkit/filters/app.py new file mode 100644 index 0000000..b9cc611 --- /dev/null +++ b/lib/prompt_toolkit/filters/app.py @@ -0,0 +1,419 @@ +""" +Filters that accept a `Application` as argument. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.cache import memoized +from prompt_toolkit.enums import EditingMode + +from .base import Condition + +if TYPE_CHECKING: + from prompt_toolkit.layout.layout import FocusableElement + + +__all__ = [ + "has_arg", + "has_completions", + "completion_is_selected", + "has_focus", + "buffer_has_focus", + "has_selection", + "has_suggestion", + "has_validation_error", + "is_done", + "is_read_only", + "is_multiline", + "renderer_height_is_known", + "in_editing_mode", + "in_paste_mode", + "vi_mode", + "vi_navigation_mode", + "vi_insert_mode", + "vi_insert_multiple_mode", + "vi_replace_mode", + "vi_selection_mode", + "vi_waiting_for_text_object_mode", + "vi_digraph_mode", + "vi_recording_macro", + "emacs_mode", + "emacs_insert_mode", + "emacs_selection_mode", + "shift_selection_mode", + "is_searching", + "control_is_searchable", + "vi_search_direction_reversed", +] + + +# NOTE: `has_focus` below should *not* be `memoized`. It can reference any user +# control. For instance, if we would continuously create new +# `PromptSession` instances, then previous instances won't be released, +# because this memoize (which caches results in the global scope) will +# still refer to each instance. +def has_focus(value: FocusableElement) -> Condition: + """ + Enable when this buffer has the focus. + """ + from prompt_toolkit.buffer import Buffer + from prompt_toolkit.layout import walk + from prompt_toolkit.layout.containers import Window, to_container + from prompt_toolkit.layout.controls import UIControl + + if isinstance(value, str): + + def test() -> bool: + return get_app().current_buffer.name == value + + elif isinstance(value, Buffer): + + def test() -> bool: + return get_app().current_buffer == value + + elif isinstance(value, UIControl): + + def test() -> bool: + return get_app().layout.current_control == value + + else: + value = to_container(value) + + if isinstance(value, Window): + + def test() -> bool: + return get_app().layout.current_window == value + + else: + + def test() -> bool: + # Consider focused when any window inside this container is + # focused. + current_window = get_app().layout.current_window + + for c in walk(value): + if isinstance(c, Window) and c == current_window: + return True + return False + + @Condition + def has_focus_filter() -> bool: + return test() + + return has_focus_filter + + +@Condition +def buffer_has_focus() -> bool: + """ + Enabled when the currently focused control is a `BufferControl`. + """ + return get_app().layout.buffer_has_focus + + +@Condition +def has_selection() -> bool: + """ + Enable when the current buffer has a selection. + """ + return bool(get_app().current_buffer.selection_state) + + +@Condition +def has_suggestion() -> bool: + """ + Enable when the current buffer has a suggestion. + """ + buffer = get_app().current_buffer + return buffer.suggestion is not None and buffer.suggestion.text != "" + + +@Condition +def has_completions() -> bool: + """ + Enable when the current buffer has completions. + """ + state = get_app().current_buffer.complete_state + return state is not None and len(state.completions) > 0 + + +@Condition +def completion_is_selected() -> bool: + """ + True when the user selected a completion. + """ + complete_state = get_app().current_buffer.complete_state + return complete_state is not None and complete_state.current_completion is not None + + +@Condition +def is_read_only() -> bool: + """ + True when the current buffer is read only. + """ + return get_app().current_buffer.read_only() + + +@Condition +def is_multiline() -> bool: + """ + True when the current buffer has been marked as multiline. + """ + return get_app().current_buffer.multiline() + + +@Condition +def has_validation_error() -> bool: + "Current buffer has validation error." + return get_app().current_buffer.validation_error is not None + + +@Condition +def has_arg() -> bool: + "Enable when the input processor has an 'arg'." + return get_app().key_processor.arg is not None + + +@Condition +def is_done() -> bool: + """ + True when the CLI is returning, aborting or exiting. + """ + return get_app().is_done + + +@Condition +def renderer_height_is_known() -> bool: + """ + Only True when the renderer knows it's real height. + + (On VT100 terminals, we have to wait for a CPR response, before we can be + sure of the available height between the cursor position and the bottom of + the terminal. And usually it's nicer to wait with drawing bottom toolbars + until we receive the height, in order to avoid flickering -- first drawing + somewhere in the middle, and then again at the bottom.) + """ + return get_app().renderer.height_is_known + + +@memoized() +def in_editing_mode(editing_mode: EditingMode) -> Condition: + """ + Check whether a given editing mode is active. (Vi or Emacs.) + """ + + @Condition + def in_editing_mode_filter() -> bool: + return get_app().editing_mode == editing_mode + + return in_editing_mode_filter + + +@Condition +def in_paste_mode() -> bool: + return get_app().paste_mode() + + +@Condition +def vi_mode() -> bool: + return get_app().editing_mode == EditingMode.VI + + +@Condition +def vi_navigation_mode() -> bool: + """ + Active when the set for Vi navigation key bindings are active. + """ + from prompt_toolkit.key_binding.vi_state import InputMode + + app = get_app() + + if ( + app.editing_mode != EditingMode.VI + or app.vi_state.operator_func + or app.vi_state.waiting_for_digraph + or app.current_buffer.selection_state + ): + return False + + return ( + app.vi_state.input_mode == InputMode.NAVIGATION + or app.vi_state.temporary_navigation_mode + or app.current_buffer.read_only() + ) + + +@Condition +def vi_insert_mode() -> bool: + from prompt_toolkit.key_binding.vi_state import InputMode + + app = get_app() + + if ( + app.editing_mode != EditingMode.VI + or app.vi_state.operator_func + or app.vi_state.waiting_for_digraph + or app.current_buffer.selection_state + or app.vi_state.temporary_navigation_mode + or app.current_buffer.read_only() + ): + return False + + return app.vi_state.input_mode == InputMode.INSERT + + +@Condition +def vi_insert_multiple_mode() -> bool: + from prompt_toolkit.key_binding.vi_state import InputMode + + app = get_app() + + if ( + app.editing_mode != EditingMode.VI + or app.vi_state.operator_func + or app.vi_state.waiting_for_digraph + or app.current_buffer.selection_state + or app.vi_state.temporary_navigation_mode + or app.current_buffer.read_only() + ): + return False + + return app.vi_state.input_mode == InputMode.INSERT_MULTIPLE + + +@Condition +def vi_replace_mode() -> bool: + from prompt_toolkit.key_binding.vi_state import InputMode + + app = get_app() + + if ( + app.editing_mode != EditingMode.VI + or app.vi_state.operator_func + or app.vi_state.waiting_for_digraph + or app.current_buffer.selection_state + or app.vi_state.temporary_navigation_mode + or app.current_buffer.read_only() + ): + return False + + return app.vi_state.input_mode == InputMode.REPLACE + + +@Condition +def vi_replace_single_mode() -> bool: + from prompt_toolkit.key_binding.vi_state import InputMode + + app = get_app() + + if ( + app.editing_mode != EditingMode.VI + or app.vi_state.operator_func + or app.vi_state.waiting_for_digraph + or app.current_buffer.selection_state + or app.vi_state.temporary_navigation_mode + or app.current_buffer.read_only() + ): + return False + + return app.vi_state.input_mode == InputMode.REPLACE_SINGLE + + +@Condition +def vi_selection_mode() -> bool: + app = get_app() + if app.editing_mode != EditingMode.VI: + return False + + return bool(app.current_buffer.selection_state) + + +@Condition +def vi_waiting_for_text_object_mode() -> bool: + app = get_app() + if app.editing_mode != EditingMode.VI: + return False + + return app.vi_state.operator_func is not None + + +@Condition +def vi_digraph_mode() -> bool: + app = get_app() + if app.editing_mode != EditingMode.VI: + return False + + return app.vi_state.waiting_for_digraph + + +@Condition +def vi_recording_macro() -> bool: + "When recording a Vi macro." + app = get_app() + if app.editing_mode != EditingMode.VI: + return False + + return app.vi_state.recording_register is not None + + +@Condition +def emacs_mode() -> bool: + "When the Emacs bindings are active." + return get_app().editing_mode == EditingMode.EMACS + + +@Condition +def emacs_insert_mode() -> bool: + app = get_app() + if ( + app.editing_mode != EditingMode.EMACS + or app.current_buffer.selection_state + or app.current_buffer.read_only() + ): + return False + return True + + +@Condition +def emacs_selection_mode() -> bool: + app = get_app() + return bool( + app.editing_mode == EditingMode.EMACS and app.current_buffer.selection_state + ) + + +@Condition +def shift_selection_mode() -> bool: + app = get_app() + return bool( + app.current_buffer.selection_state + and app.current_buffer.selection_state.shift_mode + ) + + +@Condition +def is_searching() -> bool: + "When we are searching." + app = get_app() + return app.layout.is_searching + + +@Condition +def control_is_searchable() -> bool: + "When the current UIControl is searchable." + from prompt_toolkit.layout.controls import BufferControl + + control = get_app().layout.current_control + + return ( + isinstance(control, BufferControl) and control.search_buffer_control is not None + ) + + +@Condition +def vi_search_direction_reversed() -> bool: + "When the '/' and '?' key bindings for Vi-style searching have been reversed." + return get_app().reverse_vi_search_direction() diff --git a/lib/prompt_toolkit/filters/base.py b/lib/prompt_toolkit/filters/base.py new file mode 100644 index 0000000..cd95424 --- /dev/null +++ b/lib/prompt_toolkit/filters/base.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import Callable, Iterable, Union + +__all__ = ["Filter", "Never", "Always", "Condition", "FilterOrBool"] + + +class Filter(metaclass=ABCMeta): + """ + Base class for any filter to activate/deactivate a feature, depending on a + condition. + + The return value of ``__call__`` will tell if the feature should be active. + """ + + def __init__(self) -> None: + self._and_cache: dict[Filter, Filter] = {} + self._or_cache: dict[Filter, Filter] = {} + self._invert_result: Filter | None = None + + @abstractmethod + def __call__(self) -> bool: + """ + The actual call to evaluate the filter. + """ + return True + + def __and__(self, other: Filter) -> Filter: + """ + Chaining of filters using the & operator. + """ + assert isinstance(other, Filter), f"Expecting filter, got {other!r}" + + if isinstance(other, Always): + return self + if isinstance(other, Never): + return other + + if other in self._and_cache: + return self._and_cache[other] + + result = _AndList.create([self, other]) + self._and_cache[other] = result + return result + + def __or__(self, other: Filter) -> Filter: + """ + Chaining of filters using the | operator. + """ + assert isinstance(other, Filter), f"Expecting filter, got {other!r}" + + if isinstance(other, Always): + return other + if isinstance(other, Never): + return self + + if other in self._or_cache: + return self._or_cache[other] + + result = _OrList.create([self, other]) + self._or_cache[other] = result + return result + + def __invert__(self) -> Filter: + """ + Inverting of filters using the ~ operator. + """ + if self._invert_result is None: + self._invert_result = _Invert(self) + + return self._invert_result + + def __bool__(self) -> None: + """ + By purpose, we don't allow bool(...) operations directly on a filter, + because the meaning is ambiguous. + + Executing a filter has to be done always by calling it. Providing + defaults for `None` values should be done through an `is None` check + instead of for instance ``filter1 or Always()``. + """ + raise ValueError( + "The truth value of a Filter is ambiguous. Instead, call it as a function." + ) + + +def _remove_duplicates(filters: list[Filter]) -> list[Filter]: + result = [] + for f in filters: + if f not in result: + result.append(f) + return result + + +class _AndList(Filter): + """ + Result of &-operation between several filters. + """ + + def __init__(self, filters: list[Filter]) -> None: + super().__init__() + self.filters = filters + + @classmethod + def create(cls, filters: Iterable[Filter]) -> Filter: + """ + Create a new filter by applying an `&` operator between them. + + If there's only one unique filter in the given iterable, it will return + that one filter instead of an `_AndList`. + """ + filters_2: list[Filter] = [] + + for f in filters: + if isinstance(f, _AndList): # Turn nested _AndLists into one. + filters_2.extend(f.filters) + else: + filters_2.append(f) + + # Remove duplicates. This could speed up execution, and doesn't make a + # difference for the evaluation. + filters = _remove_duplicates(filters_2) + + # If only one filter is left, return that without wrapping into an + # `_AndList`. + if len(filters) == 1: + return filters[0] + + return cls(filters) + + def __call__(self) -> bool: + return all(f() for f in self.filters) + + def __repr__(self) -> str: + return "&".join(repr(f) for f in self.filters) + + +class _OrList(Filter): + """ + Result of |-operation between several filters. + """ + + def __init__(self, filters: list[Filter]) -> None: + super().__init__() + self.filters = filters + + @classmethod + def create(cls, filters: Iterable[Filter]) -> Filter: + """ + Create a new filter by applying an `|` operator between them. + + If there's only one unique filter in the given iterable, it will return + that one filter instead of an `_OrList`. + """ + filters_2: list[Filter] = [] + + for f in filters: + if isinstance(f, _OrList): # Turn nested _AndLists into one. + filters_2.extend(f.filters) + else: + filters_2.append(f) + + # Remove duplicates. This could speed up execution, and doesn't make a + # difference for the evaluation. + filters = _remove_duplicates(filters_2) + + # If only one filter is left, return that without wrapping into an + # `_AndList`. + if len(filters) == 1: + return filters[0] + + return cls(filters) + + def __call__(self) -> bool: + return any(f() for f in self.filters) + + def __repr__(self) -> str: + return "|".join(repr(f) for f in self.filters) + + +class _Invert(Filter): + """ + Negation of another filter. + """ + + def __init__(self, filter: Filter) -> None: + super().__init__() + self.filter = filter + + def __call__(self) -> bool: + return not self.filter() + + def __repr__(self) -> str: + return f"~{self.filter!r}" + + +class Always(Filter): + """ + Always enable feature. + """ + + def __call__(self) -> bool: + return True + + def __or__(self, other: Filter) -> Filter: + return self + + def __and__(self, other: Filter) -> Filter: + return other + + def __invert__(self) -> Never: + return Never() + + +class Never(Filter): + """ + Never enable feature. + """ + + def __call__(self) -> bool: + return False + + def __and__(self, other: Filter) -> Filter: + return self + + def __or__(self, other: Filter) -> Filter: + return other + + def __invert__(self) -> Always: + return Always() + + +class Condition(Filter): + """ + Turn any callable into a Filter. The callable is supposed to not take any + arguments. + + This can be used as a decorator:: + + @Condition + def feature_is_active(): # `feature_is_active` becomes a Filter. + return True + + :param func: Callable which takes no inputs and returns a boolean. + """ + + def __init__(self, func: Callable[[], bool]) -> None: + super().__init__() + self.func = func + + def __call__(self) -> bool: + return self.func() + + def __repr__(self) -> str: + return f"Condition({self.func!r})" + + +# Often used as type annotation. +FilterOrBool = Union[Filter, bool] diff --git a/lib/prompt_toolkit/filters/cli.py b/lib/prompt_toolkit/filters/cli.py new file mode 100644 index 0000000..902fbaa --- /dev/null +++ b/lib/prompt_toolkit/filters/cli.py @@ -0,0 +1,65 @@ +""" +For backwards-compatibility. keep this file. +(Many people are going to have key bindings that rely on this file.) +""" + +from __future__ import annotations + +from .app import * + +__all__ = [ + # Old names. + "HasArg", + "HasCompletions", + "HasFocus", + "HasSelection", + "HasValidationError", + "IsDone", + "IsReadOnly", + "IsMultiline", + "RendererHeightIsKnown", + "InEditingMode", + "InPasteMode", + "ViMode", + "ViNavigationMode", + "ViInsertMode", + "ViInsertMultipleMode", + "ViReplaceMode", + "ViSelectionMode", + "ViWaitingForTextObjectMode", + "ViDigraphMode", + "EmacsMode", + "EmacsInsertMode", + "EmacsSelectionMode", + "IsSearching", + "HasSearch", + "ControlIsSearchable", +] + +# Keep the original classnames for backwards compatibility. +HasValidationError = lambda: has_validation_error +HasArg = lambda: has_arg +IsDone = lambda: is_done +RendererHeightIsKnown = lambda: renderer_height_is_known +ViNavigationMode = lambda: vi_navigation_mode +InPasteMode = lambda: in_paste_mode +EmacsMode = lambda: emacs_mode +EmacsInsertMode = lambda: emacs_insert_mode +ViMode = lambda: vi_mode +IsSearching = lambda: is_searching +HasSearch = lambda: is_searching +ControlIsSearchable = lambda: control_is_searchable +EmacsSelectionMode = lambda: emacs_selection_mode +ViDigraphMode = lambda: vi_digraph_mode +ViWaitingForTextObjectMode = lambda: vi_waiting_for_text_object_mode +ViSelectionMode = lambda: vi_selection_mode +ViReplaceMode = lambda: vi_replace_mode +ViInsertMultipleMode = lambda: vi_insert_multiple_mode +ViInsertMode = lambda: vi_insert_mode +HasSelection = lambda: has_selection +HasCompletions = lambda: has_completions +IsReadOnly = lambda: is_read_only +IsMultiline = lambda: is_multiline + +HasFocus = has_focus # No lambda here! (Has_focus is callable that returns a callable.) +InEditingMode = in_editing_mode diff --git a/lib/prompt_toolkit/filters/utils.py b/lib/prompt_toolkit/filters/utils.py new file mode 100644 index 0000000..20e00ee --- /dev/null +++ b/lib/prompt_toolkit/filters/utils.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from .base import Always, Filter, FilterOrBool, Never + +__all__ = [ + "to_filter", + "is_true", +] + + +_always = Always() +_never = Never() + + +_bool_to_filter: dict[bool, Filter] = { + True: _always, + False: _never, +} + + +def to_filter(bool_or_filter: FilterOrBool) -> Filter: + """ + Accept both booleans and Filters as input and + turn it into a Filter. + """ + if isinstance(bool_or_filter, bool): + return _bool_to_filter[bool_or_filter] + + if isinstance(bool_or_filter, Filter): + return bool_or_filter + + raise TypeError(f"Expecting a bool or a Filter instance. Got {bool_or_filter!r}") + + +def is_true(value: FilterOrBool) -> bool: + """ + Test whether `value` is True. In case of a Filter, call it. + + :param value: Boolean or `Filter` instance. + """ + return to_filter(value)() diff --git a/lib/prompt_toolkit/formatted_text/__init__.py b/lib/prompt_toolkit/formatted_text/__init__.py new file mode 100644 index 0000000..0590c81 --- /dev/null +++ b/lib/prompt_toolkit/formatted_text/__init__.py @@ -0,0 +1,59 @@ +""" +Many places in prompt_toolkit can take either plain text, or formatted text. +For instance the :func:`~prompt_toolkit.shortcuts.prompt` function takes either +plain text or formatted text for the prompt. The +:class:`~prompt_toolkit.layout.FormattedTextControl` can also take either plain +text or formatted text. + +In any case, there is an input that can either be just plain text (a string), +an :class:`.HTML` object, an :class:`.ANSI` object or a sequence of +`(style_string, text)` tuples. The :func:`.to_formatted_text` conversion +function takes any of these and turns all of them into such a tuple sequence. +""" + +from __future__ import annotations + +from .ansi import ANSI +from .base import ( + AnyFormattedText, + FormattedText, + OneStyleAndTextTuple, + StyleAndTextTuples, + Template, + is_formatted_text, + merge_formatted_text, + to_formatted_text, +) +from .html import HTML +from .pygments import PygmentsTokens +from .utils import ( + fragment_list_len, + fragment_list_to_text, + fragment_list_width, + split_lines, + to_plain_text, +) + +__all__ = [ + # Base. + "AnyFormattedText", + "OneStyleAndTextTuple", + "to_formatted_text", + "is_formatted_text", + "Template", + "merge_formatted_text", + "FormattedText", + "StyleAndTextTuples", + # HTML. + "HTML", + # ANSI. + "ANSI", + # Pygments. + "PygmentsTokens", + # Utils. + "fragment_list_len", + "fragment_list_width", + "fragment_list_to_text", + "split_lines", + "to_plain_text", +] diff --git a/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..e02f627 Binary files /dev/null and b/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc b/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc new file mode 100644 index 0000000..1d022fe Binary files /dev/null and b/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..65b400a Binary files /dev/null and b/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc b/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc new file mode 100644 index 0000000..81f1efc Binary files /dev/null and b/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc b/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc new file mode 100644 index 0000000..8854292 Binary files /dev/null and b/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc b/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..60a3979 Binary files /dev/null and b/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/formatted_text/ansi.py b/lib/prompt_toolkit/formatted_text/ansi.py new file mode 100644 index 0000000..8cc37a6 --- /dev/null +++ b/lib/prompt_toolkit/formatted_text/ansi.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +from string import Formatter +from typing import Generator + +from prompt_toolkit.output.vt100 import BG_ANSI_COLORS, FG_ANSI_COLORS +from prompt_toolkit.output.vt100 import _256_colors as _256_colors_table + +from .base import StyleAndTextTuples + +__all__ = [ + "ANSI", + "ansi_escape", +] + + +class ANSI: + """ + ANSI formatted text. + Take something ANSI escaped text, for use as a formatted string. E.g. + + :: + + ANSI('\\x1b[31mhello \\x1b[32mworld') + + Characters between ``\\001`` and ``\\002`` are supposed to have a zero width + when printed, but these are literally sent to the terminal output. This can + be used for instance, for inserting Final Term prompt commands. They will + be translated into a prompt_toolkit '[ZeroWidthEscape]' fragment. + """ + + def __init__(self, value: str) -> None: + self.value = value + self._formatted_text: StyleAndTextTuples = [] + + # Default style attributes. + self._color: str | None = None + self._bgcolor: str | None = None + self._bold = False + self._dim = False + self._underline = False + self._strike = False + self._italic = False + self._blink = False + self._reverse = False + self._hidden = False + + # Process received text. + parser = self._parse_corot() + parser.send(None) # type: ignore + for c in value: + parser.send(c) + + def _parse_corot(self) -> Generator[None, str, None]: + """ + Coroutine that parses the ANSI escape sequences. + """ + style = "" + formatted_text = self._formatted_text + + while True: + # NOTE: CSI is a special token within a stream of characters that + # introduces an ANSI control sequence used to set the + # style attributes of the following characters. + csi = False + + c = yield + + # Everything between \001 and \002 should become a ZeroWidthEscape. + if c == "\001": + escaped_text = "" + while c != "\002": + c = yield + if c == "\002": + formatted_text.append(("[ZeroWidthEscape]", escaped_text)) + c = yield + break + else: + escaped_text += c + + # Check for CSI + if c == "\x1b": + # Start of color escape sequence. + square_bracket = yield + if square_bracket == "[": + csi = True + else: + continue + elif c == "\x9b": + csi = True + + if csi: + # Got a CSI sequence. Color codes are following. + current = "" + params = [] + + while True: + char = yield + + # Construct number + if char.isdigit(): + current += char + + # Eval number + else: + # Limit and save number value + params.append(min(int(current or 0), 9999)) + + # Get delimiter token if present + if char == ";": + current = "" + + # Check and evaluate color codes + elif char == "m": + # Set attributes and token. + self._select_graphic_rendition(params) + style = self._create_style_string() + break + + # Check and evaluate cursor forward + elif char == "C": + for i in range(params[0]): + # add using current style + formatted_text.append((style, " ")) + break + + else: + # Ignore unsupported sequence. + break + else: + # Add current character. + # NOTE: At this point, we could merge the current character + # into the previous tuple if the style did not change, + # however, it's not worth the effort given that it will + # be "Exploded" once again when it's rendered to the + # output. + formatted_text.append((style, c)) + + def _select_graphic_rendition(self, attrs: list[int]) -> None: + """ + Taken a list of graphics attributes and apply changes. + """ + if not attrs: + attrs = [0] + else: + attrs = list(attrs[::-1]) + + while attrs: + attr = attrs.pop() + + if attr in _fg_colors: + self._color = _fg_colors[attr] + elif attr in _bg_colors: + self._bgcolor = _bg_colors[attr] + elif attr == 1: + self._bold = True + elif attr == 2: + self._dim = True + elif attr == 3: + self._italic = True + elif attr == 4: + self._underline = True + elif attr == 5: + self._blink = True # Slow blink + elif attr == 6: + self._blink = True # Fast blink + elif attr == 7: + self._reverse = True + elif attr == 8: + self._hidden = True + elif attr == 9: + self._strike = True + elif attr == 22: + self._bold = False # Normal intensity + self._dim = False + elif attr == 23: + self._italic = False + elif attr == 24: + self._underline = False + elif attr == 25: + self._blink = False + elif attr == 27: + self._reverse = False + elif attr == 28: + self._hidden = False + elif attr == 29: + self._strike = False + elif not attr: + # Reset all style attributes + self._color = None + self._bgcolor = None + self._bold = False + self._dim = False + self._underline = False + self._strike = False + self._italic = False + self._blink = False + self._reverse = False + self._hidden = False + + elif attr in (38, 48) and len(attrs) > 1: + n = attrs.pop() + + # 256 colors. + if n == 5 and len(attrs) >= 1: + if attr == 38: + m = attrs.pop() + self._color = _256_colors.get(m) + elif attr == 48: + m = attrs.pop() + self._bgcolor = _256_colors.get(m) + + # True colors. + if n == 2 and len(attrs) >= 3: + try: + color_str = ( + f"#{attrs.pop():02x}{attrs.pop():02x}{attrs.pop():02x}" + ) + except IndexError: + pass + else: + if attr == 38: + self._color = color_str + elif attr == 48: + self._bgcolor = color_str + + def _create_style_string(self) -> str: + """ + Turn current style flags into a string for usage in a formatted text. + """ + result = [] + if self._color: + result.append(self._color) + if self._bgcolor: + result.append("bg:" + self._bgcolor) + if self._bold: + result.append("bold") + if self._dim: + result.append("dim") + if self._underline: + result.append("underline") + if self._strike: + result.append("strike") + if self._italic: + result.append("italic") + if self._blink: + result.append("blink") + if self._reverse: + result.append("reverse") + if self._hidden: + result.append("hidden") + + return " ".join(result) + + def __repr__(self) -> str: + return f"ANSI({self.value!r})" + + def __pt_formatted_text__(self) -> StyleAndTextTuples: + return self._formatted_text + + def format(self, *args: str, **kwargs: str) -> ANSI: + """ + Like `str.format`, but make sure that the arguments are properly + escaped. (No ANSI escapes can be injected.) + """ + return ANSI(FORMATTER.vformat(self.value, args, kwargs)) + + def __mod__(self, value: object) -> ANSI: + """ + ANSI('%s') % value + """ + if not isinstance(value, tuple): + value = (value,) + + value = tuple(ansi_escape(i) for i in value) + return ANSI(self.value % value) + + +# Mapping of the ANSI color codes to their names. +_fg_colors = {v: k for k, v in FG_ANSI_COLORS.items()} +_bg_colors = {v: k for k, v in BG_ANSI_COLORS.items()} + +# Mapping of the escape codes for 256colors to their 'ffffff' value. +_256_colors = {} + +for i, (r, g, b) in enumerate(_256_colors_table.colors): + _256_colors[i] = f"#{r:02x}{g:02x}{b:02x}" + + +def ansi_escape(text: object) -> str: + """ + Replace characters with a special meaning. + """ + return str(text).replace("\x1b", "?").replace("\b", "?") + + +class ANSIFormatter(Formatter): + def format_field(self, value: object, format_spec: str) -> str: + return ansi_escape(format(value, format_spec)) + + +FORMATTER = ANSIFormatter() diff --git a/lib/prompt_toolkit/formatted_text/base.py b/lib/prompt_toolkit/formatted_text/base.py new file mode 100644 index 0000000..5fee1f8 --- /dev/null +++ b/lib/prompt_toolkit/formatted_text/base.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, Iterable, List, Tuple, Union, cast + +from prompt_toolkit.mouse_events import MouseEvent + +if TYPE_CHECKING: + from typing_extensions import Protocol + + from prompt_toolkit.key_binding.key_bindings import NotImplementedOrNone + +__all__ = [ + "OneStyleAndTextTuple", + "StyleAndTextTuples", + "MagicFormattedText", + "AnyFormattedText", + "to_formatted_text", + "is_formatted_text", + "Template", + "merge_formatted_text", + "FormattedText", +] + +OneStyleAndTextTuple = Union[ + Tuple[str, str], Tuple[str, str, Callable[[MouseEvent], "NotImplementedOrNone"]] +] + +# List of (style, text) tuples. +StyleAndTextTuples = List[OneStyleAndTextTuple] + + +if TYPE_CHECKING: + from typing_extensions import TypeGuard + + class MagicFormattedText(Protocol): + """ + Any object that implements ``__pt_formatted_text__`` represents formatted + text. + """ + + def __pt_formatted_text__(self) -> StyleAndTextTuples: ... + + +AnyFormattedText = Union[ + str, + "MagicFormattedText", + StyleAndTextTuples, + # Callable[[], 'AnyFormattedText'] # Recursive definition not supported by mypy. + Callable[[], Any], + None, +] + + +def to_formatted_text( + value: AnyFormattedText, style: str = "", auto_convert: bool = False +) -> FormattedText: + """ + Convert the given value (which can be formatted text) into a list of text + fragments. (Which is the canonical form of formatted text.) The outcome is + always a `FormattedText` instance, which is a list of (style, text) tuples. + + It can take a plain text string, an `HTML` or `ANSI` object, anything that + implements `__pt_formatted_text__` or a callable that takes no arguments and + returns one of those. + + :param style: An additional style string which is applied to all text + fragments. + :param auto_convert: If `True`, also accept other types, and convert them + to a string first. + """ + result: FormattedText | StyleAndTextTuples + + if value is None: + result = [] + elif isinstance(value, str): + result = [("", value)] + elif isinstance(value, list): + result = value # StyleAndTextTuples + elif hasattr(value, "__pt_formatted_text__"): + result = cast("MagicFormattedText", value).__pt_formatted_text__() + elif callable(value): + return to_formatted_text(value(), style=style) + elif auto_convert: + result = [("", f"{value}")] + else: + raise ValueError( + "No formatted text. Expecting a unicode object, " + f"HTML, ANSI or a FormattedText instance. Got {value!r}" + ) + + # Apply extra style. + if style: + result = cast( + StyleAndTextTuples, + [(style + " " + item_style, *rest) for item_style, *rest in result], + ) + + # Make sure the result is wrapped in a `FormattedText`. Among other + # reasons, this is important for `print_formatted_text` to work correctly + # and distinguish between lists and formatted text. + if isinstance(result, FormattedText): + return result + else: + return FormattedText(result) + + +def is_formatted_text(value: object) -> TypeGuard[AnyFormattedText]: + """ + Check whether the input is valid formatted text (for use in assert + statements). + In case of a callable, it doesn't check the return type. + """ + if callable(value): + return True + if isinstance(value, (str, list)): + return True + if hasattr(value, "__pt_formatted_text__"): + return True + return False + + +class FormattedText(StyleAndTextTuples): + """ + A list of ``(style, text)`` tuples. + + (In some situations, this can also be ``(style, text, mouse_handler)`` + tuples.) + """ + + def __pt_formatted_text__(self) -> StyleAndTextTuples: + return self + + def __repr__(self) -> str: + return f"FormattedText({super().__repr__()})" + + +class Template: + """ + Template for string interpolation with formatted text. + + Example:: + + Template(' ... {} ... ').format(HTML(...)) + + :param text: Plain text. + """ + + def __init__(self, text: str) -> None: + assert "{0}" not in text + self.text = text + + def format(self, *values: AnyFormattedText) -> AnyFormattedText: + def get_result() -> AnyFormattedText: + # Split the template in parts. + parts = self.text.split("{}") + assert len(parts) - 1 == len(values) + + result = FormattedText() + for part, val in zip(parts, values): + result.append(("", part)) + result.extend(to_formatted_text(val)) + result.append(("", parts[-1])) + return result + + return get_result + + +def merge_formatted_text(items: Iterable[AnyFormattedText]) -> AnyFormattedText: + """ + Merge (Concatenate) several pieces of formatted text together. + """ + + def _merge_formatted_text() -> AnyFormattedText: + result = FormattedText() + for i in items: + result.extend(to_formatted_text(i)) + return result + + return _merge_formatted_text diff --git a/lib/prompt_toolkit/formatted_text/html.py b/lib/prompt_toolkit/formatted_text/html.py new file mode 100644 index 0000000..a940ac8 --- /dev/null +++ b/lib/prompt_toolkit/formatted_text/html.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import xml.dom.minidom as minidom +from string import Formatter +from typing import Any + +from .base import FormattedText, StyleAndTextTuples + +__all__ = ["HTML"] + + +class HTML: + """ + HTML formatted text. + Take something HTML-like, for use as a formatted string. + + :: + + # Turn something into red. + HTML('') + + # Italic, bold, underline and strike. + HTML('...') + HTML('...') + HTML('...') + HTML('...') + + All HTML elements become available as a "class" in the style sheet. + E.g. ``...`` can be styled, by setting a style for + ``username``. + """ + + def __init__(self, value: str) -> None: + self.value = value + document = minidom.parseString(f"{value}") + + result: StyleAndTextTuples = [] + name_stack: list[str] = [] + fg_stack: list[str] = [] + bg_stack: list[str] = [] + + def get_current_style() -> str: + "Build style string for current node." + parts = [] + if name_stack: + parts.append("class:" + ",".join(name_stack)) + + if fg_stack: + parts.append("fg:" + fg_stack[-1]) + if bg_stack: + parts.append("bg:" + bg_stack[-1]) + return " ".join(parts) + + def process_node(node: Any) -> None: + "Process node recursively." + for child in node.childNodes: + if child.nodeType == child.TEXT_NODE: + result.append((get_current_style(), child.data)) + else: + add_to_name_stack = child.nodeName not in ( + "#document", + "html-root", + "style", + ) + fg = bg = "" + + for k, v in child.attributes.items(): + if k == "fg": + fg = v + if k == "bg": + bg = v + if k == "color": + fg = v # Alias for 'fg'. + + # Check for spaces in attributes. This would result in + # invalid style strings otherwise. + if " " in fg: + raise ValueError('"fg" attribute contains a space.') + if " " in bg: + raise ValueError('"bg" attribute contains a space.') + + if add_to_name_stack: + name_stack.append(child.nodeName) + if fg: + fg_stack.append(fg) + if bg: + bg_stack.append(bg) + + process_node(child) + + if add_to_name_stack: + name_stack.pop() + if fg: + fg_stack.pop() + if bg: + bg_stack.pop() + + process_node(document) + + self.formatted_text = FormattedText(result) + + def __repr__(self) -> str: + return f"HTML({self.value!r})" + + def __pt_formatted_text__(self) -> StyleAndTextTuples: + return self.formatted_text + + def format(self, *args: object, **kwargs: object) -> HTML: + """ + Like `str.format`, but make sure that the arguments are properly + escaped. + """ + return HTML(FORMATTER.vformat(self.value, args, kwargs)) + + def __mod__(self, value: object) -> HTML: + """ + HTML('%s') % value + """ + if not isinstance(value, tuple): + value = (value,) + + value = tuple(html_escape(i) for i in value) + return HTML(self.value % value) + + +class HTMLFormatter(Formatter): + def format_field(self, value: object, format_spec: str) -> str: + return html_escape(format(value, format_spec)) + + +def html_escape(text: object) -> str: + # The string interpolation functions also take integers and other types. + # Convert to string first. + if not isinstance(text, str): + text = f"{text}" + + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +FORMATTER = HTMLFormatter() diff --git a/lib/prompt_toolkit/formatted_text/pygments.py b/lib/prompt_toolkit/formatted_text/pygments.py new file mode 100644 index 0000000..d4ef3ad --- /dev/null +++ b/lib/prompt_toolkit/formatted_text/pygments.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from prompt_toolkit.styles.pygments import pygments_token_to_classname + +from .base import StyleAndTextTuples + +if TYPE_CHECKING: + from pygments.token import Token + +__all__ = [ + "PygmentsTokens", +] + + +class PygmentsTokens: + """ + Turn a pygments token list into a list of prompt_toolkit text fragments + (``(style_str, text)`` tuples). + """ + + def __init__(self, token_list: list[tuple[Token, str]]) -> None: + self.token_list = token_list + + def __pt_formatted_text__(self) -> StyleAndTextTuples: + result: StyleAndTextTuples = [] + + for token, text in self.token_list: + result.append(("class:" + pygments_token_to_classname(token), text)) + + return result diff --git a/lib/prompt_toolkit/formatted_text/utils.py b/lib/prompt_toolkit/formatted_text/utils.py new file mode 100644 index 0000000..a6f78cb --- /dev/null +++ b/lib/prompt_toolkit/formatted_text/utils.py @@ -0,0 +1,102 @@ +""" +Utilities for manipulating formatted text. + +When ``to_formatted_text`` has been called, we get a list of ``(style, text)`` +tuples. This file contains functions for manipulating such a list. +""" + +from __future__ import annotations + +from typing import Iterable, cast + +from prompt_toolkit.utils import get_cwidth + +from .base import ( + AnyFormattedText, + OneStyleAndTextTuple, + StyleAndTextTuples, + to_formatted_text, +) + +__all__ = [ + "to_plain_text", + "fragment_list_len", + "fragment_list_width", + "fragment_list_to_text", + "split_lines", +] + + +def to_plain_text(value: AnyFormattedText) -> str: + """ + Turn any kind of formatted text back into plain text. + """ + return fragment_list_to_text(to_formatted_text(value)) + + +def fragment_list_len(fragments: StyleAndTextTuples) -> int: + """ + Return the amount of characters in this text fragment list. + + :param fragments: List of ``(style_str, text)`` or + ``(style_str, text, mouse_handler)`` tuples. + """ + ZeroWidthEscape = "[ZeroWidthEscape]" + return sum(len(item[1]) for item in fragments if ZeroWidthEscape not in item[0]) + + +def fragment_list_width(fragments: StyleAndTextTuples) -> int: + """ + Return the character width of this text fragment list. + (Take double width characters into account.) + + :param fragments: List of ``(style_str, text)`` or + ``(style_str, text, mouse_handler)`` tuples. + """ + ZeroWidthEscape = "[ZeroWidthEscape]" + return sum( + get_cwidth(c) + for item in fragments + for c in item[1] + if ZeroWidthEscape not in item[0] + ) + + +def fragment_list_to_text(fragments: StyleAndTextTuples) -> str: + """ + Concatenate all the text parts again. + + :param fragments: List of ``(style_str, text)`` or + ``(style_str, text, mouse_handler)`` tuples. + """ + ZeroWidthEscape = "[ZeroWidthEscape]" + return "".join(item[1] for item in fragments if ZeroWidthEscape not in item[0]) + + +def split_lines( + fragments: Iterable[OneStyleAndTextTuple], +) -> Iterable[StyleAndTextTuples]: + """ + Take a single list of (style_str, text) tuples and yield one such list for each + line. Just like str.split, this will yield at least one item. + + :param fragments: Iterable of ``(style_str, text)`` or + ``(style_str, text, mouse_handler)`` tuples. + """ + line: StyleAndTextTuples = [] + + for style, string, *mouse_handler in fragments: + parts = string.split("\n") + + for part in parts[:-1]: + line.append(cast(OneStyleAndTextTuple, (style, part, *mouse_handler))) + yield line + line = [] + + line.append(cast(OneStyleAndTextTuple, (style, parts[-1], *mouse_handler))) + + # Always yield the last line, even when this is an empty line. This ensures + # that when `fragments` ends with a newline character, an additional empty + # line is yielded. (Otherwise, there's no way to differentiate between the + # cases where `fragments` does and doesn't end with a newline.) + yield line diff --git a/lib/prompt_toolkit/history.py b/lib/prompt_toolkit/history.py new file mode 100644 index 0000000..2d497a0 --- /dev/null +++ b/lib/prompt_toolkit/history.py @@ -0,0 +1,306 @@ +""" +Implementations for the history of a `Buffer`. + +NOTE: There is no `DynamicHistory`: + This doesn't work well, because the `Buffer` needs to be able to attach + an event handler to the event when a history entry is loaded. This + loading can be done asynchronously and making the history swappable would + probably break this. +""" + +from __future__ import annotations + +import datetime +import os +import threading +from abc import ABCMeta, abstractmethod +from asyncio import get_running_loop +from typing import AsyncGenerator, Iterable, Sequence, Union + +__all__ = [ + "History", + "ThreadedHistory", + "DummyHistory", + "FileHistory", + "InMemoryHistory", +] + + +class History(metaclass=ABCMeta): + """ + Base ``History`` class. + + This also includes abstract methods for loading/storing history. + """ + + def __init__(self) -> None: + # In memory storage for strings. + self._loaded = False + + # History that's loaded already, in reverse order. Latest, most recent + # item first. + self._loaded_strings: list[str] = [] + + # + # Methods expected by `Buffer`. + # + + async def load(self) -> AsyncGenerator[str, None]: + """ + Load the history and yield all the entries in reverse order (latest, + most recent history entry first). + + This method can be called multiple times from the `Buffer` to + repopulate the history when prompting for a new input. So we are + responsible here for both caching, and making sure that strings that + were were appended to the history will be incorporated next time this + method is called. + """ + if not self._loaded: + self._loaded_strings = list(self.load_history_strings()) + self._loaded = True + + for item in self._loaded_strings: + yield item + + def get_strings(self) -> list[str]: + """ + Get the strings from the history that are loaded so far. + (In order. Oldest item first.) + """ + return self._loaded_strings[::-1] + + def append_string(self, string: str) -> None: + "Add string to the history." + self._loaded_strings.insert(0, string) + self.store_string(string) + + # + # Implementation for specific backends. + # + + @abstractmethod + def load_history_strings(self) -> Iterable[str]: + """ + This should be a generator that yields `str` instances. + + It should yield the most recent items first, because they are the most + important. (The history can already be used, even when it's only + partially loaded.) + """ + while False: + yield + + @abstractmethod + def store_string(self, string: str) -> None: + """ + Store the string in persistent storage. + """ + + +class ThreadedHistory(History): + """ + Wrapper around `History` implementations that run the `load()` generator in + a thread. + + Use this to increase the start-up time of prompt_toolkit applications. + History entries are available as soon as they are loaded. We don't have to + wait for everything to be loaded. + """ + + def __init__(self, history: History) -> None: + super().__init__() + + self.history = history + + self._load_thread: threading.Thread | None = None + + # Lock for accessing/manipulating `_loaded_strings` and `_loaded` + # together in a consistent state. + self._lock = threading.Lock() + + # Events created by each `load()` call. Used to wait for new history + # entries from the loader thread. + self._string_load_events: list[threading.Event] = [] + + async def load(self) -> AsyncGenerator[str, None]: + """ + Like `History.load(), but call `self.load_history_strings()` in a + background thread. + """ + # Start the load thread, if this is called for the first time. + if not self._load_thread: + self._load_thread = threading.Thread( + target=self._in_load_thread, + daemon=True, + ) + self._load_thread.start() + + # Consume the `_loaded_strings` list, using asyncio. + loop = get_running_loop() + + # Create threading Event so that we can wait for new items. + event = threading.Event() + event.set() + self._string_load_events.append(event) + + items_yielded = 0 + + try: + while True: + # Wait for new items to be available. + # (Use a timeout, because the executor thread is not a daemon + # thread. The "slow-history.py" example would otherwise hang if + # Control-C is pressed before the history is fully loaded, + # because there's still this non-daemon executor thread waiting + # for this event.) + got_timeout = await loop.run_in_executor( + None, lambda: event.wait(timeout=0.5) + ) + if not got_timeout: + continue + + # Read new items (in lock). + def in_executor() -> tuple[list[str], bool]: + with self._lock: + new_items = self._loaded_strings[items_yielded:] + done = self._loaded + event.clear() + return new_items, done + + new_items, done = await loop.run_in_executor(None, in_executor) + + items_yielded += len(new_items) + + for item in new_items: + yield item + + if done: + break + finally: + self._string_load_events.remove(event) + + def _in_load_thread(self) -> None: + try: + # Start with an empty list. In case `append_string()` was called + # before `load()` happened. Then `.store_string()` will have + # written these entries back to disk and we will reload it. + self._loaded_strings = [] + + for item in self.history.load_history_strings(): + with self._lock: + self._loaded_strings.append(item) + + for event in self._string_load_events: + event.set() + finally: + with self._lock: + self._loaded = True + for event in self._string_load_events: + event.set() + + def append_string(self, string: str) -> None: + with self._lock: + self._loaded_strings.insert(0, string) + self.store_string(string) + + # All of the following are proxied to `self.history`. + + def load_history_strings(self) -> Iterable[str]: + return self.history.load_history_strings() + + def store_string(self, string: str) -> None: + self.history.store_string(string) + + def __repr__(self) -> str: + return f"ThreadedHistory({self.history!r})" + + +class InMemoryHistory(History): + """ + :class:`.History` class that keeps a list of all strings in memory. + + In order to prepopulate the history, it's possible to call either + `append_string` for all items or pass a list of strings to `__init__` here. + """ + + def __init__(self, history_strings: Sequence[str] | None = None) -> None: + super().__init__() + # Emulating disk storage. + if history_strings is None: + self._storage = [] + else: + self._storage = list(history_strings) + + def load_history_strings(self) -> Iterable[str]: + yield from self._storage[::-1] + + def store_string(self, string: str) -> None: + self._storage.append(string) + + +class DummyHistory(History): + """ + :class:`.History` object that doesn't remember anything. + """ + + def load_history_strings(self) -> Iterable[str]: + return [] + + def store_string(self, string: str) -> None: + pass + + def append_string(self, string: str) -> None: + # Don't remember this. + pass + + +_StrOrBytesPath = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] + + +class FileHistory(History): + """ + :class:`.History` class that stores all strings in a file. + """ + + def __init__(self, filename: _StrOrBytesPath) -> None: + self.filename = filename + super().__init__() + + def load_history_strings(self) -> Iterable[str]: + strings: list[str] = [] + lines: list[str] = [] + + def add() -> None: + if lines: + # Join and drop trailing newline. + string = "".join(lines)[:-1] + + strings.append(string) + + if os.path.exists(self.filename): + with open(self.filename, "rb") as f: + for line_bytes in f: + line = line_bytes.decode("utf-8", errors="replace") + + if line.startswith("+"): + lines.append(line[1:]) + else: + add() + lines = [] + + add() + + # Reverse the order, because newest items have to go first. + return reversed(strings) + + def store_string(self, string: str) -> None: + # Save to file. + with open(self.filename, "ab") as f: + + def write(t: str) -> None: + f.write(t.encode("utf-8")) + + write(f"\n# {datetime.datetime.now()}\n") + for line in string.split("\n"): + write(f"+{line}\n") diff --git a/lib/prompt_toolkit/input/__init__.py b/lib/prompt_toolkit/input/__init__.py new file mode 100644 index 0000000..ed8631b --- /dev/null +++ b/lib/prompt_toolkit/input/__init__.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from .base import DummyInput, Input, PipeInput +from .defaults import create_input, create_pipe_input + +__all__ = [ + # Base. + "Input", + "PipeInput", + "DummyInput", + # Defaults. + "create_input", + "create_pipe_input", +] diff --git a/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..73fe942 Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc new file mode 100644 index 0000000..adbcbb5 Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..436db5a Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc new file mode 100644 index 0000000..86d5f02 Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc new file mode 100644 index 0000000..ab6f01e Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc new file mode 100644 index 0000000..74bec4f Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc new file mode 100644 index 0000000..bd1db40 Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc new file mode 100644 index 0000000..9df515c Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc new file mode 100644 index 0000000..ef9f849 Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc new file mode 100644 index 0000000..a5a3f7e Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc b/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc new file mode 100644 index 0000000..12e041d Binary files /dev/null and b/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/input/ansi_escape_sequences.py b/lib/prompt_toolkit/input/ansi_escape_sequences.py new file mode 100644 index 0000000..1fba418 --- /dev/null +++ b/lib/prompt_toolkit/input/ansi_escape_sequences.py @@ -0,0 +1,344 @@ +""" +Mappings from VT100 (ANSI) escape sequences to the corresponding prompt_toolkit +keys. + +We are not using the terminfo/termcap databases to detect the ANSI escape +sequences for the input. Instead, we recognize 99% of the most common +sequences. This works well, because in practice, every modern terminal is +mostly Xterm compatible. + +Some useful docs: +- Mintty: https://github.com/mintty/mintty/blob/master/wiki/Keycodes.md +""" + +from __future__ import annotations + +from ..keys import Keys + +__all__ = [ + "ANSI_SEQUENCES", + "REVERSE_ANSI_SEQUENCES", +] + +# Mapping of vt100 escape codes to Keys. +ANSI_SEQUENCES: dict[str, Keys | tuple[Keys, ...]] = { + # Control keys. + "\x00": Keys.ControlAt, # Control-At (Also for Ctrl-Space) + "\x01": Keys.ControlA, # Control-A (home) + "\x02": Keys.ControlB, # Control-B (emacs cursor left) + "\x03": Keys.ControlC, # Control-C (interrupt) + "\x04": Keys.ControlD, # Control-D (exit) + "\x05": Keys.ControlE, # Control-E (end) + "\x06": Keys.ControlF, # Control-F (cursor forward) + "\x07": Keys.ControlG, # Control-G + "\x08": Keys.ControlH, # Control-H (8) (Identical to '\b') + "\x09": Keys.ControlI, # Control-I (9) (Identical to '\t') + "\x0a": Keys.ControlJ, # Control-J (10) (Identical to '\n') + "\x0b": Keys.ControlK, # Control-K (delete until end of line; vertical tab) + "\x0c": Keys.ControlL, # Control-L (clear; form feed) + "\x0d": Keys.ControlM, # Control-M (13) (Identical to '\r') + "\x0e": Keys.ControlN, # Control-N (14) (history forward) + "\x0f": Keys.ControlO, # Control-O (15) + "\x10": Keys.ControlP, # Control-P (16) (history back) + "\x11": Keys.ControlQ, # Control-Q + "\x12": Keys.ControlR, # Control-R (18) (reverse search) + "\x13": Keys.ControlS, # Control-S (19) (forward search) + "\x14": Keys.ControlT, # Control-T + "\x15": Keys.ControlU, # Control-U + "\x16": Keys.ControlV, # Control-V + "\x17": Keys.ControlW, # Control-W + "\x18": Keys.ControlX, # Control-X + "\x19": Keys.ControlY, # Control-Y (25) + "\x1a": Keys.ControlZ, # Control-Z + "\x1b": Keys.Escape, # Also Control-[ + "\x9b": Keys.ShiftEscape, + "\x1c": Keys.ControlBackslash, # Both Control-\ (also Ctrl-| ) + "\x1d": Keys.ControlSquareClose, # Control-] + "\x1e": Keys.ControlCircumflex, # Control-^ + "\x1f": Keys.ControlUnderscore, # Control-underscore (Also for Ctrl-hyphen.) + # ASCII Delete (0x7f) + # Vt220 (and Linux terminal) send this when pressing backspace. We map this + # to ControlH, because that will make it easier to create key bindings that + # work everywhere, with the trade-off that it's no longer possible to + # handle backspace and control-h individually for the few terminals that + # support it. (Most terminals send ControlH when backspace is pressed.) + # See: http://www.ibb.net/~anne/keyboard.html + "\x7f": Keys.ControlH, + # -- + # Various + "\x1b[1~": Keys.Home, # tmux + "\x1b[2~": Keys.Insert, + "\x1b[3~": Keys.Delete, + "\x1b[4~": Keys.End, # tmux + "\x1b[5~": Keys.PageUp, + "\x1b[6~": Keys.PageDown, + "\x1b[7~": Keys.Home, # xrvt + "\x1b[8~": Keys.End, # xrvt + "\x1b[Z": Keys.BackTab, # shift + tab + "\x1b\x09": Keys.BackTab, # Linux console + "\x1b[~": Keys.BackTab, # Windows console + # -- + # Function keys. + "\x1bOP": Keys.F1, + "\x1bOQ": Keys.F2, + "\x1bOR": Keys.F3, + "\x1bOS": Keys.F4, + "\x1b[[A": Keys.F1, # Linux console. + "\x1b[[B": Keys.F2, # Linux console. + "\x1b[[C": Keys.F3, # Linux console. + "\x1b[[D": Keys.F4, # Linux console. + "\x1b[[E": Keys.F5, # Linux console. + "\x1b[11~": Keys.F1, # rxvt-unicode + "\x1b[12~": Keys.F2, # rxvt-unicode + "\x1b[13~": Keys.F3, # rxvt-unicode + "\x1b[14~": Keys.F4, # rxvt-unicode + "\x1b[15~": Keys.F5, + "\x1b[17~": Keys.F6, + "\x1b[18~": Keys.F7, + "\x1b[19~": Keys.F8, + "\x1b[20~": Keys.F9, + "\x1b[21~": Keys.F10, + "\x1b[23~": Keys.F11, + "\x1b[24~": Keys.F12, + "\x1b[25~": Keys.F13, + "\x1b[26~": Keys.F14, + "\x1b[28~": Keys.F15, + "\x1b[29~": Keys.F16, + "\x1b[31~": Keys.F17, + "\x1b[32~": Keys.F18, + "\x1b[33~": Keys.F19, + "\x1b[34~": Keys.F20, + # Xterm + "\x1b[1;2P": Keys.F13, + "\x1b[1;2Q": Keys.F14, + # '\x1b[1;2R': Keys.F15, # Conflicts with CPR response. + "\x1b[1;2S": Keys.F16, + "\x1b[15;2~": Keys.F17, + "\x1b[17;2~": Keys.F18, + "\x1b[18;2~": Keys.F19, + "\x1b[19;2~": Keys.F20, + "\x1b[20;2~": Keys.F21, + "\x1b[21;2~": Keys.F22, + "\x1b[23;2~": Keys.F23, + "\x1b[24;2~": Keys.F24, + # -- + # CSI 27 disambiguated modified "other" keys (xterm) + # Ref: https://invisible-island.net/xterm/modified-keys.html + # These are currently unsupported, so just re-map some common ones to the + # unmodified versions + "\x1b[27;2;13~": Keys.ControlM, # Shift + Enter + "\x1b[27;5;13~": Keys.ControlM, # Ctrl + Enter + "\x1b[27;6;13~": Keys.ControlM, # Ctrl + Shift + Enter + # -- + # Control + function keys. + "\x1b[1;5P": Keys.ControlF1, + "\x1b[1;5Q": Keys.ControlF2, + # "\x1b[1;5R": Keys.ControlF3, # Conflicts with CPR response. + "\x1b[1;5S": Keys.ControlF4, + "\x1b[15;5~": Keys.ControlF5, + "\x1b[17;5~": Keys.ControlF6, + "\x1b[18;5~": Keys.ControlF7, + "\x1b[19;5~": Keys.ControlF8, + "\x1b[20;5~": Keys.ControlF9, + "\x1b[21;5~": Keys.ControlF10, + "\x1b[23;5~": Keys.ControlF11, + "\x1b[24;5~": Keys.ControlF12, + "\x1b[1;6P": Keys.ControlF13, + "\x1b[1;6Q": Keys.ControlF14, + # "\x1b[1;6R": Keys.ControlF15, # Conflicts with CPR response. + "\x1b[1;6S": Keys.ControlF16, + "\x1b[15;6~": Keys.ControlF17, + "\x1b[17;6~": Keys.ControlF18, + "\x1b[18;6~": Keys.ControlF19, + "\x1b[19;6~": Keys.ControlF20, + "\x1b[20;6~": Keys.ControlF21, + "\x1b[21;6~": Keys.ControlF22, + "\x1b[23;6~": Keys.ControlF23, + "\x1b[24;6~": Keys.ControlF24, + # -- + # Tmux (Win32 subsystem) sends the following scroll events. + "\x1b[62~": Keys.ScrollUp, + "\x1b[63~": Keys.ScrollDown, + "\x1b[200~": Keys.BracketedPaste, # Start of bracketed paste. + # -- + # Sequences generated by numpad 5. Not sure what it means. (It doesn't + # appear in 'infocmp'. Just ignore. + "\x1b[E": Keys.Ignore, # Xterm. + "\x1b[G": Keys.Ignore, # Linux console. + # -- + # Meta/control/escape + pageup/pagedown/insert/delete. + "\x1b[3;2~": Keys.ShiftDelete, # xterm, gnome-terminal. + "\x1b[5;2~": Keys.ShiftPageUp, + "\x1b[6;2~": Keys.ShiftPageDown, + "\x1b[2;3~": (Keys.Escape, Keys.Insert), + "\x1b[3;3~": (Keys.Escape, Keys.Delete), + "\x1b[5;3~": (Keys.Escape, Keys.PageUp), + "\x1b[6;3~": (Keys.Escape, Keys.PageDown), + "\x1b[2;4~": (Keys.Escape, Keys.ShiftInsert), + "\x1b[3;4~": (Keys.Escape, Keys.ShiftDelete), + "\x1b[5;4~": (Keys.Escape, Keys.ShiftPageUp), + "\x1b[6;4~": (Keys.Escape, Keys.ShiftPageDown), + "\x1b[3;5~": Keys.ControlDelete, # xterm, gnome-terminal. + "\x1b[5;5~": Keys.ControlPageUp, + "\x1b[6;5~": Keys.ControlPageDown, + "\x1b[3;6~": Keys.ControlShiftDelete, + "\x1b[5;6~": Keys.ControlShiftPageUp, + "\x1b[6;6~": Keys.ControlShiftPageDown, + "\x1b[2;7~": (Keys.Escape, Keys.ControlInsert), + "\x1b[5;7~": (Keys.Escape, Keys.ControlPageDown), + "\x1b[6;7~": (Keys.Escape, Keys.ControlPageDown), + "\x1b[2;8~": (Keys.Escape, Keys.ControlShiftInsert), + "\x1b[5;8~": (Keys.Escape, Keys.ControlShiftPageDown), + "\x1b[6;8~": (Keys.Escape, Keys.ControlShiftPageDown), + # -- + # Arrows. + # (Normal cursor mode). + "\x1b[A": Keys.Up, + "\x1b[B": Keys.Down, + "\x1b[C": Keys.Right, + "\x1b[D": Keys.Left, + "\x1b[H": Keys.Home, + "\x1b[F": Keys.End, + # Tmux sends following keystrokes when control+arrow is pressed, but for + # Emacs ansi-term sends the same sequences for normal arrow keys. Consider + # it a normal arrow press, because that's more important. + # (Application cursor mode). + "\x1bOA": Keys.Up, + "\x1bOB": Keys.Down, + "\x1bOC": Keys.Right, + "\x1bOD": Keys.Left, + "\x1bOF": Keys.End, + "\x1bOH": Keys.Home, + # Shift + arrows. + "\x1b[1;2A": Keys.ShiftUp, + "\x1b[1;2B": Keys.ShiftDown, + "\x1b[1;2C": Keys.ShiftRight, + "\x1b[1;2D": Keys.ShiftLeft, + "\x1b[1;2F": Keys.ShiftEnd, + "\x1b[1;2H": Keys.ShiftHome, + # Meta + arrow keys. Several terminals handle this differently. + # The following sequences are for xterm and gnome-terminal. + # (Iterm sends ESC followed by the normal arrow_up/down/left/right + # sequences, and the OSX Terminal sends ESCb and ESCf for "alt + # arrow_left" and "alt arrow_right." We don't handle these + # explicitly, in here, because would could not distinguish between + # pressing ESC (to go to Vi navigation mode), followed by just the + # 'b' or 'f' key. These combinations are handled in + # the input processor.) + "\x1b[1;3A": (Keys.Escape, Keys.Up), + "\x1b[1;3B": (Keys.Escape, Keys.Down), + "\x1b[1;3C": (Keys.Escape, Keys.Right), + "\x1b[1;3D": (Keys.Escape, Keys.Left), + "\x1b[1;3F": (Keys.Escape, Keys.End), + "\x1b[1;3H": (Keys.Escape, Keys.Home), + # Alt+shift+number. + "\x1b[1;4A": (Keys.Escape, Keys.ShiftDown), + "\x1b[1;4B": (Keys.Escape, Keys.ShiftUp), + "\x1b[1;4C": (Keys.Escape, Keys.ShiftRight), + "\x1b[1;4D": (Keys.Escape, Keys.ShiftLeft), + "\x1b[1;4F": (Keys.Escape, Keys.ShiftEnd), + "\x1b[1;4H": (Keys.Escape, Keys.ShiftHome), + # Control + arrows. + "\x1b[1;5A": Keys.ControlUp, # Cursor Mode + "\x1b[1;5B": Keys.ControlDown, # Cursor Mode + "\x1b[1;5C": Keys.ControlRight, # Cursor Mode + "\x1b[1;5D": Keys.ControlLeft, # Cursor Mode + "\x1b[1;5F": Keys.ControlEnd, + "\x1b[1;5H": Keys.ControlHome, + # Tmux sends following keystrokes when control+arrow is pressed, but for + # Emacs ansi-term sends the same sequences for normal arrow keys. Consider + # it a normal arrow press, because that's more important. + "\x1b[5A": Keys.ControlUp, + "\x1b[5B": Keys.ControlDown, + "\x1b[5C": Keys.ControlRight, + "\x1b[5D": Keys.ControlLeft, + "\x1bOc": Keys.ControlRight, # rxvt + "\x1bOd": Keys.ControlLeft, # rxvt + # Control + shift + arrows. + "\x1b[1;6A": Keys.ControlShiftDown, + "\x1b[1;6B": Keys.ControlShiftUp, + "\x1b[1;6C": Keys.ControlShiftRight, + "\x1b[1;6D": Keys.ControlShiftLeft, + "\x1b[1;6F": Keys.ControlShiftEnd, + "\x1b[1;6H": Keys.ControlShiftHome, + # Control + Meta + arrows. + "\x1b[1;7A": (Keys.Escape, Keys.ControlDown), + "\x1b[1;7B": (Keys.Escape, Keys.ControlUp), + "\x1b[1;7C": (Keys.Escape, Keys.ControlRight), + "\x1b[1;7D": (Keys.Escape, Keys.ControlLeft), + "\x1b[1;7F": (Keys.Escape, Keys.ControlEnd), + "\x1b[1;7H": (Keys.Escape, Keys.ControlHome), + # Meta + Shift + arrows. + "\x1b[1;8A": (Keys.Escape, Keys.ControlShiftDown), + "\x1b[1;8B": (Keys.Escape, Keys.ControlShiftUp), + "\x1b[1;8C": (Keys.Escape, Keys.ControlShiftRight), + "\x1b[1;8D": (Keys.Escape, Keys.ControlShiftLeft), + "\x1b[1;8F": (Keys.Escape, Keys.ControlShiftEnd), + "\x1b[1;8H": (Keys.Escape, Keys.ControlShiftHome), + # Meta + arrow on (some?) Macs when using iTerm defaults (see issue #483). + "\x1b[1;9A": (Keys.Escape, Keys.Up), + "\x1b[1;9B": (Keys.Escape, Keys.Down), + "\x1b[1;9C": (Keys.Escape, Keys.Right), + "\x1b[1;9D": (Keys.Escape, Keys.Left), + # -- + # Control/shift/meta + number in mintty. + # (c-2 will actually send c-@ and c-6 will send c-^.) + "\x1b[1;5p": Keys.Control0, + "\x1b[1;5q": Keys.Control1, + "\x1b[1;5r": Keys.Control2, + "\x1b[1;5s": Keys.Control3, + "\x1b[1;5t": Keys.Control4, + "\x1b[1;5u": Keys.Control5, + "\x1b[1;5v": Keys.Control6, + "\x1b[1;5w": Keys.Control7, + "\x1b[1;5x": Keys.Control8, + "\x1b[1;5y": Keys.Control9, + "\x1b[1;6p": Keys.ControlShift0, + "\x1b[1;6q": Keys.ControlShift1, + "\x1b[1;6r": Keys.ControlShift2, + "\x1b[1;6s": Keys.ControlShift3, + "\x1b[1;6t": Keys.ControlShift4, + "\x1b[1;6u": Keys.ControlShift5, + "\x1b[1;6v": Keys.ControlShift6, + "\x1b[1;6w": Keys.ControlShift7, + "\x1b[1;6x": Keys.ControlShift8, + "\x1b[1;6y": Keys.ControlShift9, + "\x1b[1;7p": (Keys.Escape, Keys.Control0), + "\x1b[1;7q": (Keys.Escape, Keys.Control1), + "\x1b[1;7r": (Keys.Escape, Keys.Control2), + "\x1b[1;7s": (Keys.Escape, Keys.Control3), + "\x1b[1;7t": (Keys.Escape, Keys.Control4), + "\x1b[1;7u": (Keys.Escape, Keys.Control5), + "\x1b[1;7v": (Keys.Escape, Keys.Control6), + "\x1b[1;7w": (Keys.Escape, Keys.Control7), + "\x1b[1;7x": (Keys.Escape, Keys.Control8), + "\x1b[1;7y": (Keys.Escape, Keys.Control9), + "\x1b[1;8p": (Keys.Escape, Keys.ControlShift0), + "\x1b[1;8q": (Keys.Escape, Keys.ControlShift1), + "\x1b[1;8r": (Keys.Escape, Keys.ControlShift2), + "\x1b[1;8s": (Keys.Escape, Keys.ControlShift3), + "\x1b[1;8t": (Keys.Escape, Keys.ControlShift4), + "\x1b[1;8u": (Keys.Escape, Keys.ControlShift5), + "\x1b[1;8v": (Keys.Escape, Keys.ControlShift6), + "\x1b[1;8w": (Keys.Escape, Keys.ControlShift7), + "\x1b[1;8x": (Keys.Escape, Keys.ControlShift8), + "\x1b[1;8y": (Keys.Escape, Keys.ControlShift9), +} + + +def _get_reverse_ansi_sequences() -> dict[Keys, str]: + """ + Create a dictionary that maps prompt_toolkit keys back to the VT100 escape + sequences. + """ + result: dict[Keys, str] = {} + + for sequence, key in ANSI_SEQUENCES.items(): + if not isinstance(key, tuple): + if key not in result: + result[key] = sequence + + return result + + +REVERSE_ANSI_SEQUENCES = _get_reverse_ansi_sequences() diff --git a/lib/prompt_toolkit/input/base.py b/lib/prompt_toolkit/input/base.py new file mode 100644 index 0000000..40ffd9d --- /dev/null +++ b/lib/prompt_toolkit/input/base.py @@ -0,0 +1,154 @@ +""" +Abstraction of CLI Input. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from contextlib import contextmanager +from typing import Callable, ContextManager, Generator + +from prompt_toolkit.key_binding import KeyPress + +__all__ = [ + "Input", + "PipeInput", + "DummyInput", +] + + +class Input(metaclass=ABCMeta): + """ + Abstraction for any input. + + An instance of this class can be given to the constructor of a + :class:`~prompt_toolkit.application.Application` and will also be + passed to the :class:`~prompt_toolkit.eventloop.base.EventLoop`. + """ + + @abstractmethod + def fileno(self) -> int: + """ + Fileno for putting this in an event loop. + """ + + @abstractmethod + def typeahead_hash(self) -> str: + """ + Identifier for storing type ahead key presses. + """ + + @abstractmethod + def read_keys(self) -> list[KeyPress]: + """ + Return a list of Key objects which are read/parsed from the input. + """ + + def flush_keys(self) -> list[KeyPress]: + """ + Flush the underlying parser. and return the pending keys. + (Used for vt100 input.) + """ + return [] + + def flush(self) -> None: + "The event loop can call this when the input has to be flushed." + pass + + @property + @abstractmethod + def closed(self) -> bool: + "Should be true when the input stream is closed." + return False + + @abstractmethod + def raw_mode(self) -> ContextManager[None]: + """ + Context manager that turns the input into raw mode. + """ + + @abstractmethod + def cooked_mode(self) -> ContextManager[None]: + """ + Context manager that turns the input into cooked mode. + """ + + @abstractmethod + def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]: + """ + Return a context manager that makes this input active in the current + event loop. + """ + + @abstractmethod + def detach(self) -> ContextManager[None]: + """ + Return a context manager that makes sure that this input is not active + in the current event loop. + """ + + def close(self) -> None: + "Close input." + pass + + +class PipeInput(Input): + """ + Abstraction for pipe input. + """ + + @abstractmethod + def send_bytes(self, data: bytes) -> None: + """Feed byte string into the pipe""" + + @abstractmethod + def send_text(self, data: str) -> None: + """Feed a text string into the pipe""" + + +class DummyInput(Input): + """ + Input for use in a `DummyApplication` + + If used in an actual application, it will make the application render + itself once and exit immediately, due to an `EOFError`. + """ + + def fileno(self) -> int: + raise NotImplementedError + + def typeahead_hash(self) -> str: + return f"dummy-{id(self)}" + + def read_keys(self) -> list[KeyPress]: + return [] + + @property + def closed(self) -> bool: + # This needs to be true, so that the dummy input will trigger an + # `EOFError` immediately in the application. + return True + + def raw_mode(self) -> ContextManager[None]: + return _dummy_context_manager() + + def cooked_mode(self) -> ContextManager[None]: + return _dummy_context_manager() + + def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]: + # Call the callback immediately once after attaching. + # This tells the callback to call `read_keys` and check the + # `input.closed` flag, after which it won't receive any keys, but knows + # that `EOFError` should be raised. This unblocks `read_from_input` in + # `application.py`. + input_ready_callback() + + return _dummy_context_manager() + + def detach(self) -> ContextManager[None]: + return _dummy_context_manager() + + +@contextmanager +def _dummy_context_manager() -> Generator[None, None, None]: + yield diff --git a/lib/prompt_toolkit/input/defaults.py b/lib/prompt_toolkit/input/defaults.py new file mode 100644 index 0000000..483eeb2 --- /dev/null +++ b/lib/prompt_toolkit/input/defaults.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import io +import sys +from typing import ContextManager, TextIO + +from .base import DummyInput, Input, PipeInput + +__all__ = [ + "create_input", + "create_pipe_input", +] + + +def create_input(stdin: TextIO | None = None, always_prefer_tty: bool = False) -> Input: + """ + Create the appropriate `Input` object for the current os/environment. + + :param always_prefer_tty: When set, if `sys.stdin` is connected to a Unix + `pipe`, check whether `sys.stdout` or `sys.stderr` are connected to a + pseudo terminal. If so, open the tty for reading instead of reading for + `sys.stdin`. (We can open `stdout` or `stderr` for reading, this is how + a `$PAGER` works.) + """ + if sys.platform == "win32": + from .win32 import Win32Input + + # If `stdin` was assigned `None` (which happens with pythonw.exe), use + # a `DummyInput`. This triggers `EOFError` in the application code. + if stdin is None and sys.stdin is None: + return DummyInput() + + return Win32Input(stdin or sys.stdin) + else: + from .vt100 import Vt100Input + + # If no input TextIO is given, use stdin/stdout. + if stdin is None: + stdin = sys.stdin + + if always_prefer_tty: + for obj in [sys.stdin, sys.stdout, sys.stderr]: + if obj.isatty(): + stdin = obj + break + + # If we can't access the file descriptor for the selected stdin, return + # a `DummyInput` instead. This can happen for instance in unit tests, + # when `sys.stdin` is patched by something that's not an actual file. + # (Instantiating `Vt100Input` would fail in this case.) + try: + stdin.fileno() + except io.UnsupportedOperation: + return DummyInput() + + return Vt100Input(stdin) + + +def create_pipe_input() -> ContextManager[PipeInput]: + """ + Create an input pipe. + This is mostly useful for unit testing. + + Usage:: + + with create_pipe_input() as input: + input.send_text('inputdata') + + Breaking change: In prompt_toolkit 3.0.28 and earlier, this was returning + the `PipeInput` directly, rather than through a context manager. + """ + if sys.platform == "win32": + from .win32_pipe import Win32PipeInput + + return Win32PipeInput.create() + else: + from .posix_pipe import PosixPipeInput + + return PosixPipeInput.create() diff --git a/lib/prompt_toolkit/input/posix_pipe.py b/lib/prompt_toolkit/input/posix_pipe.py new file mode 100644 index 0000000..c131fb8 --- /dev/null +++ b/lib/prompt_toolkit/input/posix_pipe.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import sys + +assert sys.platform != "win32" + +import os +from contextlib import contextmanager +from typing import ContextManager, Iterator, TextIO, cast + +from ..utils import DummyContext +from .base import PipeInput +from .vt100 import Vt100Input + +__all__ = [ + "PosixPipeInput", +] + + +class _Pipe: + "Wrapper around os.pipe, that ensures we don't double close any end." + + def __init__(self) -> None: + self.read_fd, self.write_fd = os.pipe() + self._read_closed = False + self._write_closed = False + + def close_read(self) -> None: + "Close read-end if not yet closed." + if self._read_closed: + return + + os.close(self.read_fd) + self._read_closed = True + + def close_write(self) -> None: + "Close write-end if not yet closed." + if self._write_closed: + return + + os.close(self.write_fd) + self._write_closed = True + + def close(self) -> None: + "Close both read and write ends." + self.close_read() + self.close_write() + + +class PosixPipeInput(Vt100Input, PipeInput): + """ + Input that is send through a pipe. + This is useful if we want to send the input programmatically into the + application. Mostly useful for unit testing. + + Usage:: + + with PosixPipeInput.create() as input: + input.send_text('inputdata') + """ + + _id = 0 + + def __init__(self, _pipe: _Pipe, _text: str = "") -> None: + # Private constructor. Users should use the public `.create()` method. + self.pipe = _pipe + + class Stdin: + encoding = "utf-8" + + def isatty(stdin) -> bool: + return True + + def fileno(stdin) -> int: + return self.pipe.read_fd + + super().__init__(cast(TextIO, Stdin())) + self.send_text(_text) + + # Identifier for every PipeInput for the hash. + self.__class__._id += 1 + self._id = self.__class__._id + + @classmethod + @contextmanager + def create(cls, text: str = "") -> Iterator[PosixPipeInput]: + pipe = _Pipe() + try: + yield PosixPipeInput(_pipe=pipe, _text=text) + finally: + pipe.close() + + def send_bytes(self, data: bytes) -> None: + os.write(self.pipe.write_fd, data) + + def send_text(self, data: str) -> None: + "Send text to the input." + os.write(self.pipe.write_fd, data.encode("utf-8")) + + def raw_mode(self) -> ContextManager[None]: + return DummyContext() + + def cooked_mode(self) -> ContextManager[None]: + return DummyContext() + + def close(self) -> None: + "Close pipe fds." + # Only close the write-end of the pipe. This will unblock the reader + # callback (in vt100.py > _attached_input), which eventually will raise + # `EOFError`. If we'd also close the read-end, then the event loop + # won't wake up the corresponding callback because of this. + self.pipe.close_write() + + def typeahead_hash(self) -> str: + """ + This needs to be unique for every `PipeInput`. + """ + return f"pipe-input-{self._id}" diff --git a/lib/prompt_toolkit/input/posix_utils.py b/lib/prompt_toolkit/input/posix_utils.py new file mode 100644 index 0000000..4a78dc4 --- /dev/null +++ b/lib/prompt_toolkit/input/posix_utils.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import os +import select +from codecs import getincrementaldecoder + +__all__ = [ + "PosixStdinReader", +] + + +class PosixStdinReader: + """ + Wrapper around stdin which reads (nonblocking) the next available 1024 + bytes and decodes it. + + Note that you can't be sure that the input file is closed if the ``read`` + function returns an empty string. When ``errors=ignore`` is passed, + ``read`` can return an empty string if all malformed input was replaced by + an empty string. (We can't block here and wait for more input.) So, because + of that, check the ``closed`` attribute, to be sure that the file has been + closed. + + :param stdin_fd: File descriptor from which we read. + :param errors: Can be 'ignore', 'strict' or 'replace'. + On Python3, this can be 'surrogateescape', which is the default. + + 'surrogateescape' is preferred, because this allows us to transfer + unrecognized bytes to the key bindings. Some terminals, like lxterminal + and Guake, use the 'Mxx' notation to send mouse events, where each 'x' + can be any possible byte. + """ + + # By default, we want to 'ignore' errors here. The input stream can be full + # of junk. One occurrence of this that I had was when using iTerm2 on OS X, + # with "Option as Meta" checked (You should choose "Option as +Esc".) + + def __init__( + self, stdin_fd: int, errors: str = "surrogateescape", encoding: str = "utf-8" + ) -> None: + self.stdin_fd = stdin_fd + self.errors = errors + + # Create incremental decoder for decoding stdin. + # We can not just do `os.read(stdin.fileno(), 1024).decode('utf-8')`, because + # it could be that we are in the middle of a utf-8 byte sequence. + self._stdin_decoder_cls = getincrementaldecoder(encoding) + self._stdin_decoder = self._stdin_decoder_cls(errors=errors) + + #: True when there is nothing anymore to read. + self.closed = False + + def read(self, count: int = 1024) -> str: + # By default we choose a rather small chunk size, because reading + # big amounts of input at once, causes the event loop to process + # all these key bindings also at once without going back to the + # loop. This will make the application feel unresponsive. + """ + Read the input and return it as a string. + + Return the text. Note that this can return an empty string, even when + the input stream was not yet closed. This means that something went + wrong during the decoding. + """ + if self.closed: + return "" + + # Check whether there is some input to read. `os.read` would block + # otherwise. + # (Actually, the event loop is responsible to make sure that this + # function is only called when there is something to read, but for some + # reason this happens in certain situations.) + try: + if not select.select([self.stdin_fd], [], [], 0)[0]: + return "" + except OSError: + # Happens for instance when the file descriptor was closed. + # (We had this in ptterm, where the FD became ready, a callback was + # scheduled, but in the meantime another callback closed it already.) + self.closed = True + + # Note: the following works better than wrapping `self.stdin` like + # `codecs.getreader('utf-8')(stdin)` and doing `read(1)`. + # Somehow that causes some latency when the escape + # character is pressed. (Especially on combination with the `select`.) + try: + data = os.read(self.stdin_fd, count) + + # Nothing more to read, stream is closed. + if data == b"": + self.closed = True + return "" + except OSError: + # In case of SIGWINCH + data = b"" + + return self._stdin_decoder.decode(data) diff --git a/lib/prompt_toolkit/input/typeahead.py b/lib/prompt_toolkit/input/typeahead.py new file mode 100644 index 0000000..f8faa93 --- /dev/null +++ b/lib/prompt_toolkit/input/typeahead.py @@ -0,0 +1,78 @@ +r""" +Store input key strokes if we did read more than was required. + +The input classes `Vt100Input` and `Win32Input` read the input text in chunks +of a few kilobytes. This means that if we read input from stdin, it could be +that we read a couple of lines (with newlines in between) at once. + +This creates a problem: potentially, we read too much from stdin. Sometimes +people paste several lines at once because they paste input in a REPL and +expect each input() call to process one line. Or they rely on type ahead +because the application can't keep up with the processing. + +However, we need to read input in bigger chunks. We need this mostly to support +pasting of larger chunks of text. We don't want everything to become +unresponsive because we: + - read one character; + - parse one character; + - call the key binding, which does a string operation with one character; + - and render the user interface. +Doing text operations on single characters is very inefficient in Python, so we +prefer to work on bigger chunks of text. This is why we have to read the input +in bigger chunks. + +Further, line buffering is also not an option, because it doesn't work well in +the architecture. We use lower level Posix APIs, that work better with the +event loop and so on. In fact, there is also nothing that defines that only \n +can accept the input, you could create a key binding for any key to accept the +input. + +To support type ahead, this module will store all the key strokes that were +read too early, so that they can be feed into to the next `prompt()` call or to +the next prompt_toolkit `Application`. +""" + +from __future__ import annotations + +from collections import defaultdict + +from ..key_binding import KeyPress +from .base import Input + +__all__ = [ + "store_typeahead", + "get_typeahead", + "clear_typeahead", +] + +_buffer: dict[str, list[KeyPress]] = defaultdict(list) + + +def store_typeahead(input_obj: Input, key_presses: list[KeyPress]) -> None: + """ + Insert typeahead key presses for the given input. + """ + global _buffer + key = input_obj.typeahead_hash() + _buffer[key].extend(key_presses) + + +def get_typeahead(input_obj: Input) -> list[KeyPress]: + """ + Retrieve typeahead and reset the buffer for this input. + """ + global _buffer + + key = input_obj.typeahead_hash() + result = _buffer[key] + _buffer[key] = [] + return result + + +def clear_typeahead(input_obj: Input) -> None: + """ + Clear typeahead buffer. + """ + global _buffer + key = input_obj.typeahead_hash() + _buffer[key] = [] diff --git a/lib/prompt_toolkit/input/vt100.py b/lib/prompt_toolkit/input/vt100.py new file mode 100644 index 0000000..c1660de --- /dev/null +++ b/lib/prompt_toolkit/input/vt100.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import sys + +assert sys.platform != "win32" + +import contextlib +import io +import termios +import tty +from asyncio import AbstractEventLoop, get_running_loop +from typing import Callable, ContextManager, Generator, TextIO + +from ..key_binding import KeyPress +from .base import Input +from .posix_utils import PosixStdinReader +from .vt100_parser import Vt100Parser + +__all__ = [ + "Vt100Input", + "raw_mode", + "cooked_mode", +] + + +class Vt100Input(Input): + """ + Vt100 input for Posix systems. + (This uses a posix file descriptor that can be registered in the event loop.) + """ + + # For the error messages. Only display "Input is not a terminal" once per + # file descriptor. + _fds_not_a_terminal: set[int] = set() + + def __init__(self, stdin: TextIO) -> None: + # Test whether the given input object has a file descriptor. + # (Idle reports stdin to be a TTY, but fileno() is not implemented.) + try: + # This should not raise, but can return 0. + stdin.fileno() + except io.UnsupportedOperation as e: + if "idlelib.run" in sys.modules: + raise io.UnsupportedOperation( + "Stdin is not a terminal. Running from Idle is not supported." + ) from e + else: + raise io.UnsupportedOperation("Stdin is not a terminal.") from e + + # Even when we have a file descriptor, it doesn't mean it's a TTY. + # Normally, this requires a real TTY device, but people instantiate + # this class often during unit tests as well. They use for instance + # pexpect to pipe data into an application. For convenience, we print + # an error message and go on. + isatty = stdin.isatty() + fd = stdin.fileno() + + if not isatty and fd not in Vt100Input._fds_not_a_terminal: + msg = "Warning: Input is not a terminal (fd=%r).\n" + sys.stderr.write(msg % fd) + sys.stderr.flush() + Vt100Input._fds_not_a_terminal.add(fd) + + # + self.stdin = stdin + + # Create a backup of the fileno(). We want this to work even if the + # underlying file is closed, so that `typeahead_hash()` keeps working. + self._fileno = stdin.fileno() + + self._buffer: list[KeyPress] = [] # Buffer to collect the Key objects. + self.stdin_reader = PosixStdinReader(self._fileno, encoding=stdin.encoding) + self.vt100_parser = Vt100Parser( + lambda key_press: self._buffer.append(key_press) + ) + + def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]: + """ + Return a context manager that makes this input active in the current + event loop. + """ + return _attached_input(self, input_ready_callback) + + def detach(self) -> ContextManager[None]: + """ + Return a context manager that makes sure that this input is not active + in the current event loop. + """ + return _detached_input(self) + + def read_keys(self) -> list[KeyPress]: + "Read list of KeyPress." + # Read text from stdin. + data = self.stdin_reader.read() + + # Pass it through our vt100 parser. + self.vt100_parser.feed(data) + + # Return result. + result = self._buffer + self._buffer = [] + return result + + def flush_keys(self) -> list[KeyPress]: + """ + Flush pending keys and return them. + (Used for flushing the 'escape' key.) + """ + # Flush all pending keys. (This is most important to flush the vt100 + # 'Escape' key early when nothing else follows.) + self.vt100_parser.flush() + + # Return result. + result = self._buffer + self._buffer = [] + return result + + @property + def closed(self) -> bool: + return self.stdin_reader.closed + + def raw_mode(self) -> ContextManager[None]: + return raw_mode(self.stdin.fileno()) + + def cooked_mode(self) -> ContextManager[None]: + return cooked_mode(self.stdin.fileno()) + + def fileno(self) -> int: + return self.stdin.fileno() + + def typeahead_hash(self) -> str: + return f"fd-{self._fileno}" + + +_current_callbacks: dict[ + tuple[AbstractEventLoop, int], Callable[[], None] | None +] = {} # (loop, fd) -> current callback + + +@contextlib.contextmanager +def _attached_input( + input: Vt100Input, callback: Callable[[], None] +) -> Generator[None, None, None]: + """ + Context manager that makes this input active in the current event loop. + + :param input: :class:`~prompt_toolkit.input.Input` object. + :param callback: Called when the input is ready to read. + """ + loop = get_running_loop() + fd = input.fileno() + previous = _current_callbacks.get((loop, fd)) + + def callback_wrapper() -> None: + """Wrapper around the callback that already removes the reader when + the input is closed. Otherwise, we keep continuously calling this + callback, until we leave the context manager (which can happen a bit + later). This fixes issues when piping /dev/null into a prompt_toolkit + application.""" + if input.closed: + loop.remove_reader(fd) + callback() + + try: + loop.add_reader(fd, callback_wrapper) + except PermissionError: + # For `EPollSelector`, adding /dev/null to the event loop will raise + # `PermissionError` (that doesn't happen for `SelectSelector` + # apparently). Whenever we get a `PermissionError`, we can raise + # `EOFError`, because there's not more to be read anyway. `EOFError` is + # an exception that people expect in + # `prompt_toolkit.application.Application.run()`. + # To reproduce, do: `ptpython 0< /dev/null 1< /dev/null` + raise EOFError + + _current_callbacks[loop, fd] = callback + + try: + yield + finally: + loop.remove_reader(fd) + + if previous: + loop.add_reader(fd, previous) + _current_callbacks[loop, fd] = previous + else: + del _current_callbacks[loop, fd] + + +@contextlib.contextmanager +def _detached_input(input: Vt100Input) -> Generator[None, None, None]: + loop = get_running_loop() + fd = input.fileno() + previous = _current_callbacks.get((loop, fd)) + + if previous: + loop.remove_reader(fd) + _current_callbacks[loop, fd] = None + + try: + yield + finally: + if previous: + loop.add_reader(fd, previous) + _current_callbacks[loop, fd] = previous + + +class raw_mode: + """ + :: + + with raw_mode(stdin): + ''' the pseudo-terminal stdin is now used in raw mode ''' + + We ignore errors when executing `tcgetattr` fails. + """ + + # There are several reasons for ignoring errors: + # 1. To avoid the "Inappropriate ioctl for device" crash if somebody would + # execute this code (In a Python REPL, for instance): + # + # import os; f = open(os.devnull); os.dup2(f.fileno(), 0) + # + # The result is that the eventloop will stop correctly, because it has + # to logic to quit when stdin is closed. However, we should not fail at + # this point. See: + # https://github.com/jonathanslenders/python-prompt-toolkit/pull/393 + # https://github.com/jonathanslenders/python-prompt-toolkit/issues/392 + + # 2. Related, when stdin is an SSH pipe, and no full terminal was allocated. + # See: https://github.com/jonathanslenders/python-prompt-toolkit/pull/165 + def __init__(self, fileno: int) -> None: + self.fileno = fileno + self.attrs_before: list[int | list[bytes | int]] | None + try: + self.attrs_before = termios.tcgetattr(fileno) + except termios.error: + # Ignore attribute errors. + self.attrs_before = None + + def __enter__(self) -> None: + # NOTE: On os X systems, using pty.setraw() fails. Therefor we are using this: + try: + newattr = termios.tcgetattr(self.fileno) + except termios.error: + pass + else: + newattr[tty.LFLAG] = self._patch_lflag(newattr[tty.LFLAG]) + newattr[tty.IFLAG] = self._patch_iflag(newattr[tty.IFLAG]) + + # VMIN defines the number of characters read at a time in + # non-canonical mode. It seems to default to 1 on Linux, but on + # Solaris and derived operating systems it defaults to 4. (This is + # because the VMIN slot is the same as the VEOF slot, which + # defaults to ASCII EOT = Ctrl-D = 4.) + newattr[tty.CC][termios.VMIN] = 1 + + termios.tcsetattr(self.fileno, termios.TCSANOW, newattr) + + @classmethod + def _patch_lflag(cls, attrs: int) -> int: + return attrs & ~(termios.ECHO | termios.ICANON | termios.IEXTEN | termios.ISIG) + + @classmethod + def _patch_iflag(cls, attrs: int) -> int: + return attrs & ~( + # Disable XON/XOFF flow control on output and input. + # (Don't capture Ctrl-S and Ctrl-Q.) + # Like executing: "stty -ixon." + termios.IXON + | termios.IXOFF + | + # Don't translate carriage return into newline on input. + termios.ICRNL + | termios.INLCR + | termios.IGNCR + ) + + def __exit__(self, *a: object) -> None: + if self.attrs_before is not None: + try: + termios.tcsetattr(self.fileno, termios.TCSANOW, self.attrs_before) + except termios.error: + pass + + # # Put the terminal in application mode. + # self._stdout.write('\x1b[?1h') + + +class cooked_mode(raw_mode): + """ + The opposite of ``raw_mode``, used when we need cooked mode inside a + `raw_mode` block. Used in `Application.run_in_terminal`.:: + + with cooked_mode(stdin): + ''' the pseudo-terminal stdin is now used in cooked mode. ''' + """ + + @classmethod + def _patch_lflag(cls, attrs: int) -> int: + return attrs | (termios.ECHO | termios.ICANON | termios.IEXTEN | termios.ISIG) + + @classmethod + def _patch_iflag(cls, attrs: int) -> int: + # Turn the ICRNL flag back on. (Without this, calling `input()` in + # run_in_terminal doesn't work and displays ^M instead. Ptpython + # evaluates commands using `run_in_terminal`, so it's important that + # they translate ^M back into ^J.) + return attrs | termios.ICRNL diff --git a/lib/prompt_toolkit/input/vt100_parser.py b/lib/prompt_toolkit/input/vt100_parser.py new file mode 100644 index 0000000..73dbce3 --- /dev/null +++ b/lib/prompt_toolkit/input/vt100_parser.py @@ -0,0 +1,250 @@ +""" +Parser for VT100 input stream. +""" + +from __future__ import annotations + +import re +from typing import Callable, Dict, Generator + +from ..key_binding.key_processor import KeyPress +from ..keys import Keys +from .ansi_escape_sequences import ANSI_SEQUENCES + +__all__ = [ + "Vt100Parser", +] + + +# Regex matching any CPR response +# (Note that we use '\Z' instead of '$', because '$' could include a trailing +# newline.) +_cpr_response_re = re.compile("^" + re.escape("\x1b[") + r"\d+;\d+R\Z") + +# Mouse events: +# Typical: "Esc[MaB*" Urxvt: "Esc[96;14;13M" and for Xterm SGR: "Esc[<64;85;12M" +_mouse_event_re = re.compile("^" + re.escape("\x1b[") + r"( bool: + # (hard coded) If this could be a prefix of a CPR response, return + # True. + if _cpr_response_prefix_re.match(prefix) or _mouse_event_prefix_re.match( + prefix + ): + result = True + else: + # If this could be a prefix of anything else, also return True. + result = any( + v + for k, v in ANSI_SEQUENCES.items() + if k.startswith(prefix) and k != prefix + ) + + self[prefix] = result + return result + + +_IS_PREFIX_OF_LONGER_MATCH_CACHE = _IsPrefixOfLongerMatchCache() + + +class Vt100Parser: + """ + Parser for VT100 input stream. + Data can be fed through the `feed` method and the given callback will be + called with KeyPress objects. + + :: + + def callback(key): + pass + i = Vt100Parser(callback) + i.feed('data\x01...') + + :attr feed_key_callback: Function that will be called when a key is parsed. + """ + + # Lookup table of ANSI escape sequences for a VT100 terminal + # Hint: in order to know what sequences your terminal writes to stdin, run + # "od -c" and start typing. + def __init__(self, feed_key_callback: Callable[[KeyPress], None]) -> None: + self.feed_key_callback = feed_key_callback + self.reset() + + def reset(self, request: bool = False) -> None: + self._in_bracketed_paste = False + self._start_parser() + + def _start_parser(self) -> None: + """ + Start the parser coroutine. + """ + self._input_parser = self._input_parser_generator() + self._input_parser.send(None) # type: ignore + + def _get_match(self, prefix: str) -> None | Keys | tuple[Keys, ...]: + """ + Return the key (or keys) that maps to this prefix. + """ + # (hard coded) If we match a CPR response, return Keys.CPRResponse. + # (This one doesn't fit in the ANSI_SEQUENCES, because it contains + # integer variables.) + if _cpr_response_re.match(prefix): + return Keys.CPRResponse + + elif _mouse_event_re.match(prefix): + return Keys.Vt100MouseEvent + + # Otherwise, use the mappings. + try: + return ANSI_SEQUENCES[prefix] + except KeyError: + return None + + def _input_parser_generator(self) -> Generator[None, str | _Flush, None]: + """ + Coroutine (state machine) for the input parser. + """ + prefix = "" + retry = False + flush = False + + while True: + flush = False + + if retry: + retry = False + else: + # Get next character. + c = yield + + if isinstance(c, _Flush): + flush = True + else: + prefix += c + + # If we have some data, check for matches. + if prefix: + is_prefix_of_longer_match = _IS_PREFIX_OF_LONGER_MATCH_CACHE[prefix] + match = self._get_match(prefix) + + # Exact matches found, call handlers.. + if (flush or not is_prefix_of_longer_match) and match: + self._call_handler(match, prefix) + prefix = "" + + # No exact match found. + elif (flush or not is_prefix_of_longer_match) and not match: + found = False + retry = True + + # Loop over the input, try the longest match first and + # shift. + for i in range(len(prefix), 0, -1): + match = self._get_match(prefix[:i]) + if match: + self._call_handler(match, prefix[:i]) + prefix = prefix[i:] + found = True + + if not found: + self._call_handler(prefix[0], prefix[0]) + prefix = prefix[1:] + + def _call_handler( + self, key: str | Keys | tuple[Keys, ...], insert_text: str + ) -> None: + """ + Callback to handler. + """ + if isinstance(key, tuple): + # Received ANSI sequence that corresponds with multiple keys + # (probably alt+something). Handle keys individually, but only pass + # data payload to first KeyPress (so that we won't insert it + # multiple times). + for i, k in enumerate(key): + self._call_handler(k, insert_text if i == 0 else "") + else: + if key == Keys.BracketedPaste: + self._in_bracketed_paste = True + self._paste_buffer = "" + else: + self.feed_key_callback(KeyPress(key, insert_text)) + + def feed(self, data: str) -> None: + """ + Feed the input stream. + + :param data: Input string (unicode). + """ + # Handle bracketed paste. (We bypass the parser that matches all other + # key presses and keep reading input until we see the end mark.) + # This is much faster then parsing character by character. + if self._in_bracketed_paste: + self._paste_buffer += data + end_mark = "\x1b[201~" + + if end_mark in self._paste_buffer: + end_index = self._paste_buffer.index(end_mark) + + # Feed content to key bindings. + paste_content = self._paste_buffer[:end_index] + self.feed_key_callback(KeyPress(Keys.BracketedPaste, paste_content)) + + # Quit bracketed paste mode and handle remaining input. + self._in_bracketed_paste = False + remaining = self._paste_buffer[end_index + len(end_mark) :] + self._paste_buffer = "" + + self.feed(remaining) + + # Handle normal input character by character. + else: + for i, c in enumerate(data): + if self._in_bracketed_paste: + # Quit loop and process from this position when the parser + # entered bracketed paste. + self.feed(data[i:]) + break + else: + self._input_parser.send(c) + + def flush(self) -> None: + """ + Flush the buffer of the input stream. + + This will allow us to handle the escape key (or maybe meta) sooner. + The input received by the escape key is actually the same as the first + characters of e.g. Arrow-Up, so without knowing what follows the escape + sequence, we don't know whether escape has been pressed, or whether + it's something else. This flush function should be called after a + timeout, and processes everything that's still in the buffer as-is, so + without assuming any characters will follow. + """ + self._input_parser.send(_Flush()) + + def feed_and_flush(self, data: str) -> None: + """ + Wrapper around ``feed`` and ``flush``. + """ + self.feed(data) + self.flush() diff --git a/lib/prompt_toolkit/input/win32.py b/lib/prompt_toolkit/input/win32.py new file mode 100644 index 0000000..616df8e --- /dev/null +++ b/lib/prompt_toolkit/input/win32.py @@ -0,0 +1,904 @@ +from __future__ import annotations + +import os +import sys +from abc import abstractmethod +from asyncio import get_running_loop +from contextlib import contextmanager + +from ..utils import SPHINX_AUTODOC_RUNNING + +assert sys.platform == "win32" + +# Do not import win32-specific stuff when generating documentation. +# Otherwise RTD would be unable to generate docs for this module. +if not SPHINX_AUTODOC_RUNNING: + import msvcrt + from ctypes import windll + +from ctypes import Array, byref, pointer +from ctypes.wintypes import DWORD, HANDLE +from typing import Callable, ContextManager, Iterable, Iterator, TextIO + +from prompt_toolkit.eventloop import run_in_executor_with_context +from prompt_toolkit.eventloop.win32 import create_win32_event, wait_for_handles +from prompt_toolkit.key_binding.key_processor import KeyPress +from prompt_toolkit.keys import Keys +from prompt_toolkit.mouse_events import MouseButton, MouseEventType +from prompt_toolkit.win32_types import ( + INPUT_RECORD, + KEY_EVENT_RECORD, + MOUSE_EVENT_RECORD, + STD_INPUT_HANDLE, + EventTypes, +) + +from .ansi_escape_sequences import REVERSE_ANSI_SEQUENCES +from .base import Input +from .vt100_parser import Vt100Parser + +__all__ = [ + "Win32Input", + "ConsoleInputReader", + "raw_mode", + "cooked_mode", + "attach_win32_input", + "detach_win32_input", +] + +# Win32 Constants for MOUSE_EVENT_RECORD. +# See: https://docs.microsoft.com/en-us/windows/console/mouse-event-record-str +FROM_LEFT_1ST_BUTTON_PRESSED = 0x1 +RIGHTMOST_BUTTON_PRESSED = 0x2 +MOUSE_MOVED = 0x0001 +MOUSE_WHEELED = 0x0004 + +# See: https://msdn.microsoft.com/pl-pl/library/windows/desktop/ms686033(v=vs.85).aspx +ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200 + + +class _Win32InputBase(Input): + """ + Base class for `Win32Input` and `Win32PipeInput`. + """ + + def __init__(self) -> None: + self.win32_handles = _Win32Handles() + + @property + @abstractmethod + def handle(self) -> HANDLE: + pass + + +class Win32Input(_Win32InputBase): + """ + `Input` class that reads from the Windows console. + """ + + def __init__(self, stdin: TextIO | None = None) -> None: + super().__init__() + self._use_virtual_terminal_input = _is_win_vt100_input_enabled() + + self.console_input_reader: Vt100ConsoleInputReader | ConsoleInputReader + + if self._use_virtual_terminal_input: + self.console_input_reader = Vt100ConsoleInputReader() + else: + self.console_input_reader = ConsoleInputReader() + + def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]: + """ + Return a context manager that makes this input active in the current + event loop. + """ + return attach_win32_input(self, input_ready_callback) + + def detach(self) -> ContextManager[None]: + """ + Return a context manager that makes sure that this input is not active + in the current event loop. + """ + return detach_win32_input(self) + + def read_keys(self) -> list[KeyPress]: + return list(self.console_input_reader.read()) + + def flush_keys(self) -> list[KeyPress]: + return self.console_input_reader.flush_keys() + + @property + def closed(self) -> bool: + return False + + def raw_mode(self) -> ContextManager[None]: + return raw_mode( + use_win10_virtual_terminal_input=self._use_virtual_terminal_input + ) + + def cooked_mode(self) -> ContextManager[None]: + return cooked_mode() + + def fileno(self) -> int: + # The windows console doesn't depend on the file handle, so + # this is not used for the event loop (which uses the + # handle instead). But it's used in `Application.run_system_command` + # which opens a subprocess with a given stdin/stdout. + return sys.stdin.fileno() + + def typeahead_hash(self) -> str: + return "win32-input" + + def close(self) -> None: + self.console_input_reader.close() + + @property + def handle(self) -> HANDLE: + return self.console_input_reader.handle + + +class ConsoleInputReader: + """ + :param recognize_paste: When True, try to discover paste actions and turn + the event into a BracketedPaste. + """ + + # Keys with character data. + mappings = { + b"\x1b": Keys.Escape, + b"\x00": Keys.ControlSpace, # Control-Space (Also for Ctrl-@) + b"\x01": Keys.ControlA, # Control-A (home) + b"\x02": Keys.ControlB, # Control-B (emacs cursor left) + b"\x03": Keys.ControlC, # Control-C (interrupt) + b"\x04": Keys.ControlD, # Control-D (exit) + b"\x05": Keys.ControlE, # Control-E (end) + b"\x06": Keys.ControlF, # Control-F (cursor forward) + b"\x07": Keys.ControlG, # Control-G + b"\x08": Keys.ControlH, # Control-H (8) (Identical to '\b') + b"\x09": Keys.ControlI, # Control-I (9) (Identical to '\t') + b"\x0a": Keys.ControlJ, # Control-J (10) (Identical to '\n') + b"\x0b": Keys.ControlK, # Control-K (delete until end of line; vertical tab) + b"\x0c": Keys.ControlL, # Control-L (clear; form feed) + b"\x0d": Keys.ControlM, # Control-M (enter) + b"\x0e": Keys.ControlN, # Control-N (14) (history forward) + b"\x0f": Keys.ControlO, # Control-O (15) + b"\x10": Keys.ControlP, # Control-P (16) (history back) + b"\x11": Keys.ControlQ, # Control-Q + b"\x12": Keys.ControlR, # Control-R (18) (reverse search) + b"\x13": Keys.ControlS, # Control-S (19) (forward search) + b"\x14": Keys.ControlT, # Control-T + b"\x15": Keys.ControlU, # Control-U + b"\x16": Keys.ControlV, # Control-V + b"\x17": Keys.ControlW, # Control-W + b"\x18": Keys.ControlX, # Control-X + b"\x19": Keys.ControlY, # Control-Y (25) + b"\x1a": Keys.ControlZ, # Control-Z + b"\x1c": Keys.ControlBackslash, # Both Control-\ and Ctrl-| + b"\x1d": Keys.ControlSquareClose, # Control-] + b"\x1e": Keys.ControlCircumflex, # Control-^ + b"\x1f": Keys.ControlUnderscore, # Control-underscore (Also for Ctrl-hyphen.) + b"\x7f": Keys.Backspace, # (127) Backspace (ASCII Delete.) + } + + # Keys that don't carry character data. + keycodes = { + # Home/End + 33: Keys.PageUp, + 34: Keys.PageDown, + 35: Keys.End, + 36: Keys.Home, + # Arrows + 37: Keys.Left, + 38: Keys.Up, + 39: Keys.Right, + 40: Keys.Down, + 45: Keys.Insert, + 46: Keys.Delete, + # F-keys. + 112: Keys.F1, + 113: Keys.F2, + 114: Keys.F3, + 115: Keys.F4, + 116: Keys.F5, + 117: Keys.F6, + 118: Keys.F7, + 119: Keys.F8, + 120: Keys.F9, + 121: Keys.F10, + 122: Keys.F11, + 123: Keys.F12, + } + + LEFT_ALT_PRESSED = 0x0002 + RIGHT_ALT_PRESSED = 0x0001 + SHIFT_PRESSED = 0x0010 + LEFT_CTRL_PRESSED = 0x0008 + RIGHT_CTRL_PRESSED = 0x0004 + + def __init__(self, recognize_paste: bool = True) -> None: + self._fdcon = None + self.recognize_paste = recognize_paste + + # When stdin is a tty, use that handle, otherwise, create a handle from + # CONIN$. + self.handle: HANDLE + if sys.stdin.isatty(): + self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + else: + self._fdcon = os.open("CONIN$", os.O_RDWR | os.O_BINARY) + self.handle = HANDLE(msvcrt.get_osfhandle(self._fdcon)) + + def close(self) -> None: + "Close fdcon." + if self._fdcon is not None: + os.close(self._fdcon) + + def read(self) -> Iterable[KeyPress]: + """ + Return a list of `KeyPress` instances. It won't return anything when + there was nothing to read. (This function doesn't block.) + + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684961(v=vs.85).aspx + """ + max_count = 2048 # Max events to read at the same time. + + read = DWORD(0) + arrtype = INPUT_RECORD * max_count + input_records = arrtype() + + # Check whether there is some input to read. `ReadConsoleInputW` would + # block otherwise. + # (Actually, the event loop is responsible to make sure that this + # function is only called when there is something to read, but for some + # reason this happened in the asyncio_win32 loop, and it's better to be + # safe anyway.) + if not wait_for_handles([self.handle], timeout=0): + return + + # Get next batch of input event. + windll.kernel32.ReadConsoleInputW( + self.handle, pointer(input_records), max_count, pointer(read) + ) + + # First, get all the keys from the input buffer, in order to determine + # whether we should consider this a paste event or not. + all_keys = list(self._get_keys(read, input_records)) + + # Fill in 'data' for key presses. + all_keys = [self._insert_key_data(key) for key in all_keys] + + # Correct non-bmp characters that are passed as separate surrogate codes + all_keys = list(self._merge_paired_surrogates(all_keys)) + + if self.recognize_paste and self._is_paste(all_keys): + gen = iter(all_keys) + k: KeyPress | None + + for k in gen: + # Pasting: if the current key consists of text or \n, turn it + # into a BracketedPaste. + data = [] + while k and ( + not isinstance(k.key, Keys) + or k.key in {Keys.ControlJ, Keys.ControlM} + ): + data.append(k.data) + try: + k = next(gen) + except StopIteration: + k = None + + if data: + yield KeyPress(Keys.BracketedPaste, "".join(data)) + if k is not None: + yield k + else: + yield from all_keys + + def flush_keys(self) -> list[KeyPress]: + # Method only needed for structural compatibility with `Vt100ConsoleInputReader`. + return [] + + def _insert_key_data(self, key_press: KeyPress) -> KeyPress: + """ + Insert KeyPress data, for vt100 compatibility. + """ + if key_press.data: + return key_press + + if isinstance(key_press.key, Keys): + data = REVERSE_ANSI_SEQUENCES.get(key_press.key, "") + else: + data = "" + + return KeyPress(key_press.key, data) + + def _get_keys( + self, read: DWORD, input_records: Array[INPUT_RECORD] + ) -> Iterator[KeyPress]: + """ + Generator that yields `KeyPress` objects from the input records. + """ + for i in range(read.value): + ir = input_records[i] + + # Get the right EventType from the EVENT_RECORD. + # (For some reason the Windows console application 'cmder' + # [http://gooseberrycreative.com/cmder/] can return '0' for + # ir.EventType. -- Just ignore that.) + if ir.EventType in EventTypes: + ev = getattr(ir.Event, EventTypes[ir.EventType]) + + # Process if this is a key event. (We also have mouse, menu and + # focus events.) + if isinstance(ev, KEY_EVENT_RECORD) and ev.KeyDown: + yield from self._event_to_key_presses(ev) + + elif isinstance(ev, MOUSE_EVENT_RECORD): + yield from self._handle_mouse(ev) + + @staticmethod + def _merge_paired_surrogates(key_presses: list[KeyPress]) -> Iterator[KeyPress]: + """ + Combines consecutive KeyPresses with high and low surrogates into + single characters + """ + buffered_high_surrogate = None + for key in key_presses: + is_text = not isinstance(key.key, Keys) + is_high_surrogate = is_text and "\ud800" <= key.key <= "\udbff" + is_low_surrogate = is_text and "\udc00" <= key.key <= "\udfff" + + if buffered_high_surrogate: + if is_low_surrogate: + # convert high surrogate + low surrogate to single character + fullchar = ( + (buffered_high_surrogate.key + key.key) + .encode("utf-16-le", "surrogatepass") + .decode("utf-16-le") + ) + key = KeyPress(fullchar, fullchar) + else: + yield buffered_high_surrogate + buffered_high_surrogate = None + + if is_high_surrogate: + buffered_high_surrogate = key + else: + yield key + + if buffered_high_surrogate: + yield buffered_high_surrogate + + @staticmethod + def _is_paste(keys: list[KeyPress]) -> bool: + """ + Return `True` when we should consider this list of keys as a paste + event. Pasted text on windows will be turned into a + `Keys.BracketedPaste` event. (It's not 100% correct, but it is probably + the best possible way to detect pasting of text and handle that + correctly.) + """ + # Consider paste when it contains at least one newline and at least one + # other character. + text_count = 0 + newline_count = 0 + + for k in keys: + if not isinstance(k.key, Keys): + text_count += 1 + if k.key == Keys.ControlM: + newline_count += 1 + + return newline_count >= 1 and text_count >= 1 + + def _event_to_key_presses(self, ev: KEY_EVENT_RECORD) -> list[KeyPress]: + """ + For this `KEY_EVENT_RECORD`, return a list of `KeyPress` instances. + """ + assert isinstance(ev, KEY_EVENT_RECORD) and ev.KeyDown + + result: KeyPress | None = None + + control_key_state = ev.ControlKeyState + u_char = ev.uChar.UnicodeChar + # Use surrogatepass because u_char may be an unmatched surrogate + ascii_char = u_char.encode("utf-8", "surrogatepass") + + # NOTE: We don't use `ev.uChar.AsciiChar`. That appears to be the + # unicode code point truncated to 1 byte. See also: + # https://github.com/ipython/ipython/issues/10004 + # https://github.com/jonathanslenders/python-prompt-toolkit/issues/389 + + if u_char == "\x00": + if ev.VirtualKeyCode in self.keycodes: + result = KeyPress(self.keycodes[ev.VirtualKeyCode], "") + else: + if ascii_char in self.mappings: + if self.mappings[ascii_char] == Keys.ControlJ: + u_char = ( + "\n" # Windows sends \n, turn into \r for unix compatibility. + ) + result = KeyPress(self.mappings[ascii_char], u_char) + else: + result = KeyPress(u_char, u_char) + + # First we handle Shift-Control-Arrow/Home/End (need to do this first) + if ( + ( + control_key_state & self.LEFT_CTRL_PRESSED + or control_key_state & self.RIGHT_CTRL_PRESSED + ) + and control_key_state & self.SHIFT_PRESSED + and result + ): + mapping: dict[str, str] = { + Keys.Left: Keys.ControlShiftLeft, + Keys.Right: Keys.ControlShiftRight, + Keys.Up: Keys.ControlShiftUp, + Keys.Down: Keys.ControlShiftDown, + Keys.Home: Keys.ControlShiftHome, + Keys.End: Keys.ControlShiftEnd, + Keys.Insert: Keys.ControlShiftInsert, + Keys.PageUp: Keys.ControlShiftPageUp, + Keys.PageDown: Keys.ControlShiftPageDown, + } + result.key = mapping.get(result.key, result.key) + + # Correctly handle Control-Arrow/Home/End and Control-Insert/Delete keys. + if ( + control_key_state & self.LEFT_CTRL_PRESSED + or control_key_state & self.RIGHT_CTRL_PRESSED + ) and result: + mapping = { + Keys.Left: Keys.ControlLeft, + Keys.Right: Keys.ControlRight, + Keys.Up: Keys.ControlUp, + Keys.Down: Keys.ControlDown, + Keys.Home: Keys.ControlHome, + Keys.End: Keys.ControlEnd, + Keys.Insert: Keys.ControlInsert, + Keys.Delete: Keys.ControlDelete, + Keys.PageUp: Keys.ControlPageUp, + Keys.PageDown: Keys.ControlPageDown, + } + result.key = mapping.get(result.key, result.key) + + # Turn 'Tab' into 'BackTab' when shift was pressed. + # Also handle other shift-key combination + if control_key_state & self.SHIFT_PRESSED and result: + mapping = { + Keys.Tab: Keys.BackTab, + Keys.Left: Keys.ShiftLeft, + Keys.Right: Keys.ShiftRight, + Keys.Up: Keys.ShiftUp, + Keys.Down: Keys.ShiftDown, + Keys.Home: Keys.ShiftHome, + Keys.End: Keys.ShiftEnd, + Keys.Insert: Keys.ShiftInsert, + Keys.Delete: Keys.ShiftDelete, + Keys.PageUp: Keys.ShiftPageUp, + Keys.PageDown: Keys.ShiftPageDown, + } + result.key = mapping.get(result.key, result.key) + + # Turn 'Space' into 'ControlSpace' when control was pressed. + if ( + ( + control_key_state & self.LEFT_CTRL_PRESSED + or control_key_state & self.RIGHT_CTRL_PRESSED + ) + and result + and result.data == " " + ): + result = KeyPress(Keys.ControlSpace, " ") + + # Turn Control-Enter into META-Enter. (On a vt100 terminal, we cannot + # detect this combination. But it's really practical on Windows.) + if ( + ( + control_key_state & self.LEFT_CTRL_PRESSED + or control_key_state & self.RIGHT_CTRL_PRESSED + ) + and result + and result.key == Keys.ControlJ + ): + return [KeyPress(Keys.Escape, ""), result] + + # Return result. If alt was pressed, prefix the result with an + # 'Escape' key, just like unix VT100 terminals do. + + # NOTE: Only replace the left alt with escape. The right alt key often + # acts as altgr and is used in many non US keyboard layouts for + # typing some special characters, like a backslash. We don't want + # all backslashes to be prefixed with escape. (Esc-\ has a + # meaning in E-macs, for instance.) + if result: + meta_pressed = control_key_state & self.LEFT_ALT_PRESSED + + if meta_pressed: + return [KeyPress(Keys.Escape, ""), result] + else: + return [result] + + else: + return [] + + def _handle_mouse(self, ev: MOUSE_EVENT_RECORD) -> list[KeyPress]: + """ + Handle mouse events. Return a list of KeyPress instances. + """ + event_flags = ev.EventFlags + button_state = ev.ButtonState + + event_type: MouseEventType | None = None + button: MouseButton = MouseButton.NONE + + # Scroll events. + if event_flags & MOUSE_WHEELED: + if button_state > 0: + event_type = MouseEventType.SCROLL_UP + else: + event_type = MouseEventType.SCROLL_DOWN + else: + # Handle button state for non-scroll events. + if button_state == FROM_LEFT_1ST_BUTTON_PRESSED: + button = MouseButton.LEFT + + elif button_state == RIGHTMOST_BUTTON_PRESSED: + button = MouseButton.RIGHT + + # Move events. + if event_flags & MOUSE_MOVED: + event_type = MouseEventType.MOUSE_MOVE + + # No key pressed anymore: mouse up. + if event_type is None: + if button_state > 0: + # Some button pressed. + event_type = MouseEventType.MOUSE_DOWN + else: + # No button pressed. + event_type = MouseEventType.MOUSE_UP + + data = ";".join( + [ + button.value, + event_type.value, + str(ev.MousePosition.X), + str(ev.MousePosition.Y), + ] + ) + return [KeyPress(Keys.WindowsMouseEvent, data)] + + +class Vt100ConsoleInputReader: + """ + Similar to `ConsoleInputReader`, but for usage when + `ENABLE_VIRTUAL_TERMINAL_INPUT` is enabled. This assumes that Windows sends + us the right vt100 escape sequences and we parse those with our vt100 + parser. + + (Using this instead of `ConsoleInputReader` results in the "data" attribute + from the `KeyPress` instances to be more correct in edge cases, because + this responds to for instance the terminal being in application cursor keys + mode.) + """ + + def __init__(self) -> None: + self._fdcon = None + + self._buffer: list[KeyPress] = [] # Buffer to collect the Key objects. + self._vt100_parser = Vt100Parser( + lambda key_press: self._buffer.append(key_press) + ) + + # When stdin is a tty, use that handle, otherwise, create a handle from + # CONIN$. + self.handle: HANDLE + if sys.stdin.isatty(): + self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + else: + self._fdcon = os.open("CONIN$", os.O_RDWR | os.O_BINARY) + self.handle = HANDLE(msvcrt.get_osfhandle(self._fdcon)) + + def close(self) -> None: + "Close fdcon." + if self._fdcon is not None: + os.close(self._fdcon) + + def read(self) -> Iterable[KeyPress]: + """ + Return a list of `KeyPress` instances. It won't return anything when + there was nothing to read. (This function doesn't block.) + + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684961(v=vs.85).aspx + """ + max_count = 2048 # Max events to read at the same time. + + read = DWORD(0) + arrtype = INPUT_RECORD * max_count + input_records = arrtype() + + # Check whether there is some input to read. `ReadConsoleInputW` would + # block otherwise. + # (Actually, the event loop is responsible to make sure that this + # function is only called when there is something to read, but for some + # reason this happened in the asyncio_win32 loop, and it's better to be + # safe anyway.) + if not wait_for_handles([self.handle], timeout=0): + return [] + + # Get next batch of input event. + windll.kernel32.ReadConsoleInputW( + self.handle, pointer(input_records), max_count, pointer(read) + ) + + # First, get all the keys from the input buffer, in order to determine + # whether we should consider this a paste event or not. + for key_data in self._get_keys(read, input_records): + self._vt100_parser.feed(key_data) + + # Return result. + result = self._buffer + self._buffer = [] + return result + + def flush_keys(self) -> list[KeyPress]: + """ + Flush pending keys and return them. + (Used for flushing the 'escape' key.) + """ + # Flush all pending keys. (This is most important to flush the vt100 + # 'Escape' key early when nothing else follows.) + self._vt100_parser.flush() + + # Return result. + result = self._buffer + self._buffer = [] + return result + + def _get_keys( + self, read: DWORD, input_records: Array[INPUT_RECORD] + ) -> Iterator[str]: + """ + Generator that yields `KeyPress` objects from the input records. + """ + for i in range(read.value): + ir = input_records[i] + + # Get the right EventType from the EVENT_RECORD. + # (For some reason the Windows console application 'cmder' + # [http://gooseberrycreative.com/cmder/] can return '0' for + # ir.EventType. -- Just ignore that.) + if ir.EventType in EventTypes: + ev = getattr(ir.Event, EventTypes[ir.EventType]) + + # Process if this is a key event. (We also have mouse, menu and + # focus events.) + if isinstance(ev, KEY_EVENT_RECORD) and ev.KeyDown: + u_char = ev.uChar.UnicodeChar + if u_char != "\x00": + yield u_char + + +class _Win32Handles: + """ + Utility to keep track of which handles are connectod to which callbacks. + + `add_win32_handle` starts a tiny event loop in another thread which waits + for the Win32 handle to become ready. When this happens, the callback will + be called in the current asyncio event loop using `call_soon_threadsafe`. + + `remove_win32_handle` will stop this tiny event loop. + + NOTE: We use this technique, so that we don't have to use the + `ProactorEventLoop` on Windows and we can wait for things like stdin + in a `SelectorEventLoop`. This is important, because our inputhook + mechanism (used by IPython), only works with the `SelectorEventLoop`. + """ + + def __init__(self) -> None: + self._handle_callbacks: dict[int, Callable[[], None]] = {} + + # Windows Events that are triggered when we have to stop watching this + # handle. + self._remove_events: dict[int, HANDLE] = {} + + def add_win32_handle(self, handle: HANDLE, callback: Callable[[], None]) -> None: + """ + Add a Win32 handle to the event loop. + """ + handle_value = handle.value + + if handle_value is None: + raise ValueError("Invalid handle.") + + # Make sure to remove a previous registered handler first. + self.remove_win32_handle(handle) + + loop = get_running_loop() + self._handle_callbacks[handle_value] = callback + + # Create remove event. + remove_event = create_win32_event() + self._remove_events[handle_value] = remove_event + + # Add reader. + def ready() -> None: + # Tell the callback that input's ready. + try: + callback() + finally: + run_in_executor_with_context(wait, loop=loop) + + # Wait for the input to become ready. + # (Use an executor for this, the Windows asyncio event loop doesn't + # allow us to wait for handles like stdin.) + def wait() -> None: + # Wait until either the handle becomes ready, or the remove event + # has been set. + result = wait_for_handles([remove_event, handle]) + + if result is remove_event: + windll.kernel32.CloseHandle(remove_event) + return + else: + loop.call_soon_threadsafe(ready) + + run_in_executor_with_context(wait, loop=loop) + + def remove_win32_handle(self, handle: HANDLE) -> Callable[[], None] | None: + """ + Remove a Win32 handle from the event loop. + Return either the registered handler or `None`. + """ + if handle.value is None: + return None # Ignore. + + # Trigger remove events, so that the reader knows to stop. + try: + event = self._remove_events.pop(handle.value) + except KeyError: + pass + else: + windll.kernel32.SetEvent(event) + + try: + return self._handle_callbacks.pop(handle.value) + except KeyError: + return None + + +@contextmanager +def attach_win32_input( + input: _Win32InputBase, callback: Callable[[], None] +) -> Iterator[None]: + """ + Context manager that makes this input active in the current event loop. + + :param input: :class:`~prompt_toolkit.input.Input` object. + :param input_ready_callback: Called when the input is ready to read. + """ + win32_handles = input.win32_handles + handle = input.handle + + if handle.value is None: + raise ValueError("Invalid handle.") + + # Add reader. + previous_callback = win32_handles.remove_win32_handle(handle) + win32_handles.add_win32_handle(handle, callback) + + try: + yield + finally: + win32_handles.remove_win32_handle(handle) + + if previous_callback: + win32_handles.add_win32_handle(handle, previous_callback) + + +@contextmanager +def detach_win32_input(input: _Win32InputBase) -> Iterator[None]: + win32_handles = input.win32_handles + handle = input.handle + + if handle.value is None: + raise ValueError("Invalid handle.") + + previous_callback = win32_handles.remove_win32_handle(handle) + + try: + yield + finally: + if previous_callback: + win32_handles.add_win32_handle(handle, previous_callback) + + +class raw_mode: + """ + :: + + with raw_mode(stdin): + ''' the windows terminal is now in 'raw' mode. ''' + + The ``fileno`` attribute is ignored. This is to be compatible with the + `raw_input` method of `.vt100_input`. + """ + + def __init__( + self, fileno: int | None = None, use_win10_virtual_terminal_input: bool = False + ) -> None: + self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + self.use_win10_virtual_terminal_input = use_win10_virtual_terminal_input + + def __enter__(self) -> None: + # Remember original mode. + original_mode = DWORD() + windll.kernel32.GetConsoleMode(self.handle, pointer(original_mode)) + self.original_mode = original_mode + + self._patch() + + def _patch(self) -> None: + # Set raw + ENABLE_ECHO_INPUT = 0x0004 + ENABLE_LINE_INPUT = 0x0002 + ENABLE_PROCESSED_INPUT = 0x0001 + + new_mode = self.original_mode.value & ~( + ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT + ) + + if self.use_win10_virtual_terminal_input: + new_mode |= ENABLE_VIRTUAL_TERMINAL_INPUT + + windll.kernel32.SetConsoleMode(self.handle, new_mode) + + def __exit__(self, *a: object) -> None: + # Restore original mode + windll.kernel32.SetConsoleMode(self.handle, self.original_mode) + + +class cooked_mode(raw_mode): + """ + :: + + with cooked_mode(stdin): + ''' The pseudo-terminal stdin is now used in cooked mode. ''' + """ + + def _patch(self) -> None: + # Set cooked. + ENABLE_ECHO_INPUT = 0x0004 + ENABLE_LINE_INPUT = 0x0002 + ENABLE_PROCESSED_INPUT = 0x0001 + + windll.kernel32.SetConsoleMode( + self.handle, + self.original_mode.value + | (ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT), + ) + + +def _is_win_vt100_input_enabled() -> bool: + """ + Returns True when we're running Windows and VT100 escape sequences are + supported. + """ + hconsole = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + + # Get original console mode. + original_mode = DWORD(0) + windll.kernel32.GetConsoleMode(hconsole, byref(original_mode)) + + try: + # Try to enable VT100 sequences. + result: int = windll.kernel32.SetConsoleMode( + hconsole, DWORD(ENABLE_VIRTUAL_TERMINAL_INPUT) + ) + + return result == 1 + finally: + windll.kernel32.SetConsoleMode(hconsole, original_mode) diff --git a/lib/prompt_toolkit/input/win32_pipe.py b/lib/prompt_toolkit/input/win32_pipe.py new file mode 100644 index 0000000..0bafa49 --- /dev/null +++ b/lib/prompt_toolkit/input/win32_pipe.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import sys + +assert sys.platform == "win32" + +from contextlib import contextmanager +from ctypes import windll +from ctypes.wintypes import HANDLE +from typing import Callable, ContextManager, Iterator + +from prompt_toolkit.eventloop.win32 import create_win32_event + +from ..key_binding import KeyPress +from ..utils import DummyContext +from .base import PipeInput +from .vt100_parser import Vt100Parser +from .win32 import _Win32InputBase, attach_win32_input, detach_win32_input + +__all__ = ["Win32PipeInput"] + + +class Win32PipeInput(_Win32InputBase, PipeInput): + """ + This is an input pipe that works on Windows. + Text or bytes can be feed into the pipe, and key strokes can be read from + the pipe. This is useful if we want to send the input programmatically into + the application. Mostly useful for unit testing. + + Notice that even though it's Windows, we use vt100 escape sequences over + the pipe. + + Usage:: + + input = Win32PipeInput() + input.send_text('inputdata') + """ + + _id = 0 + + def __init__(self, _event: HANDLE) -> None: + super().__init__() + # Event (handle) for registering this input in the event loop. + # This event is set when there is data available to read from the pipe. + # Note: We use this approach instead of using a regular pipe, like + # returned from `os.pipe()`, because making such a regular pipe + # non-blocking is tricky and this works really well. + self._event = create_win32_event() + + self._closed = False + + # Parser for incoming keys. + self._buffer: list[KeyPress] = [] # Buffer to collect the Key objects. + self.vt100_parser = Vt100Parser(lambda key: self._buffer.append(key)) + + # Identifier for every PipeInput for the hash. + self.__class__._id += 1 + self._id = self.__class__._id + + @classmethod + @contextmanager + def create(cls) -> Iterator[Win32PipeInput]: + event = create_win32_event() + try: + yield Win32PipeInput(_event=event) + finally: + windll.kernel32.CloseHandle(event) + + @property + def closed(self) -> bool: + return self._closed + + def fileno(self) -> int: + """ + The windows pipe doesn't depend on the file handle. + """ + raise NotImplementedError + + @property + def handle(self) -> HANDLE: + "The handle used for registering this pipe in the event loop." + return self._event + + def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]: + """ + Return a context manager that makes this input active in the current + event loop. + """ + return attach_win32_input(self, input_ready_callback) + + def detach(self) -> ContextManager[None]: + """ + Return a context manager that makes sure that this input is not active + in the current event loop. + """ + return detach_win32_input(self) + + def read_keys(self) -> list[KeyPress]: + "Read list of KeyPress." + + # Return result. + result = self._buffer + self._buffer = [] + + # Reset event. + if not self._closed: + # (If closed, the event should not reset.) + windll.kernel32.ResetEvent(self._event) + + return result + + def flush_keys(self) -> list[KeyPress]: + """ + Flush pending keys and return them. + (Used for flushing the 'escape' key.) + """ + # Flush all pending keys. (This is most important to flush the vt100 + # 'Escape' key early when nothing else follows.) + self.vt100_parser.flush() + + # Return result. + result = self._buffer + self._buffer = [] + return result + + def send_bytes(self, data: bytes) -> None: + "Send bytes to the input." + self.send_text(data.decode("utf-8", "ignore")) + + def send_text(self, text: str) -> None: + "Send text to the input." + if self._closed: + raise ValueError("Attempt to write into a closed pipe.") + + # Pass it through our vt100 parser. + self.vt100_parser.feed(text) + + # Set event. + windll.kernel32.SetEvent(self._event) + + def raw_mode(self) -> ContextManager[None]: + return DummyContext() + + def cooked_mode(self) -> ContextManager[None]: + return DummyContext() + + def close(self) -> None: + "Close write-end of the pipe." + self._closed = True + windll.kernel32.SetEvent(self._event) + + def typeahead_hash(self) -> str: + """ + This needs to be unique for every `PipeInput`. + """ + return f"pipe-input-{self._id}" diff --git a/lib/prompt_toolkit/key_binding/__init__.py b/lib/prompt_toolkit/key_binding/__init__.py new file mode 100644 index 0000000..c31746a --- /dev/null +++ b/lib/prompt_toolkit/key_binding/__init__.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from .key_bindings import ( + ConditionalKeyBindings, + DynamicKeyBindings, + KeyBindings, + KeyBindingsBase, + merge_key_bindings, +) +from .key_processor import KeyPress, KeyPressEvent + +__all__ = [ + # key_bindings. + "ConditionalKeyBindings", + "DynamicKeyBindings", + "KeyBindings", + "KeyBindingsBase", + "merge_key_bindings", + # key_processor + "KeyPress", + "KeyPressEvent", +] diff --git a/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..8d5eafd Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc new file mode 100644 index 0000000..02fea8e Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc new file mode 100644 index 0000000..b6d36ec Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc new file mode 100644 index 0000000..7312d83 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc new file mode 100644 index 0000000..f764bb9 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc new file mode 100644 index 0000000..67679a2 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc b/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc new file mode 100644 index 0000000..24636f3 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__init__.py b/lib/prompt_toolkit/key_binding/bindings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..51ea47f Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc new file mode 100644 index 0000000..bb37b11 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc new file mode 100644 index 0000000..e40229e Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc new file mode 100644 index 0000000..4cb913c Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc new file mode 100644 index 0000000..3739958 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc new file mode 100644 index 0000000..5969e2b Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc new file mode 100644 index 0000000..b54bf08 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc new file mode 100644 index 0000000..1c3ab0d Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc new file mode 100644 index 0000000..a6dddc5 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc new file mode 100644 index 0000000..ce51fa1 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc new file mode 100644 index 0000000..254bd2e Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc new file mode 100644 index 0000000..c98d26f Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc new file mode 100644 index 0000000..8bb345c Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc b/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc new file mode 100644 index 0000000..c078b58 Binary files /dev/null and b/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py b/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py new file mode 100644 index 0000000..b487f14 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py @@ -0,0 +1,66 @@ +""" +Key bindings for auto suggestion (for fish-style auto suggestion). +""" + +from __future__ import annotations + +import re + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.filters import Condition, emacs_mode +from prompt_toolkit.key_binding.key_bindings import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPressEvent + +__all__ = [ + "load_auto_suggest_bindings", +] + +E = KeyPressEvent + + +def load_auto_suggest_bindings() -> KeyBindings: + """ + Key bindings for accepting auto suggestion text. + + (This has to come after the Vi bindings, because they also have an + implementation for the "right arrow", but we really want the suggestion + binding when a suggestion is available.) + """ + key_bindings = KeyBindings() + handle = key_bindings.add + + @Condition + def suggestion_available() -> bool: + app = get_app() + return ( + app.current_buffer.suggestion is not None + and len(app.current_buffer.suggestion.text) > 0 + and app.current_buffer.document.is_cursor_at_the_end + ) + + @handle("c-f", filter=suggestion_available) + @handle("c-e", filter=suggestion_available) + @handle("right", filter=suggestion_available) + def _accept(event: E) -> None: + """ + Accept suggestion. + """ + b = event.current_buffer + suggestion = b.suggestion + + if suggestion: + b.insert_text(suggestion.text) + + @handle("escape", "f", filter=suggestion_available & emacs_mode) + def _fill(event: E) -> None: + """ + Fill partial suggestion. + """ + b = event.current_buffer + suggestion = b.suggestion + + if suggestion: + t = re.split(r"([^\s/]+(?:\s+|/))", suggestion.text) + b.insert_text(next(x for x in t if x)) + + return key_bindings diff --git a/lib/prompt_toolkit/key_binding/bindings/basic.py b/lib/prompt_toolkit/key_binding/bindings/basic.py new file mode 100644 index 0000000..ad18df9 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/basic.py @@ -0,0 +1,257 @@ +# pylint: disable=function-redefined +from __future__ import annotations + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.filters import ( + Condition, + emacs_insert_mode, + has_selection, + in_paste_mode, + is_multiline, + vi_insert_mode, +) +from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent +from prompt_toolkit.keys import Keys + +from ..key_bindings import KeyBindings +from .named_commands import get_by_name + +__all__ = [ + "load_basic_bindings", +] + +E = KeyPressEvent + + +def if_no_repeat(event: E) -> bool: + """Callable that returns True when the previous event was delivered to + another handler.""" + return not event.is_repeat + + +@Condition +def has_text_before_cursor() -> bool: + return bool(get_app().current_buffer.text) + + +@Condition +def in_quoted_insert() -> bool: + return get_app().quoted_insert + + +def load_basic_bindings() -> KeyBindings: + key_bindings = KeyBindings() + insert_mode = vi_insert_mode | emacs_insert_mode + handle = key_bindings.add + + @handle("c-a") + @handle("c-b") + @handle("c-c") + @handle("c-d") + @handle("c-e") + @handle("c-f") + @handle("c-g") + @handle("c-h") + @handle("c-i") + @handle("c-j") + @handle("c-k") + @handle("c-l") + @handle("c-m") + @handle("c-n") + @handle("c-o") + @handle("c-p") + @handle("c-q") + @handle("c-r") + @handle("c-s") + @handle("c-t") + @handle("c-u") + @handle("c-v") + @handle("c-w") + @handle("c-x") + @handle("c-y") + @handle("c-z") + @handle("f1") + @handle("f2") + @handle("f3") + @handle("f4") + @handle("f5") + @handle("f6") + @handle("f7") + @handle("f8") + @handle("f9") + @handle("f10") + @handle("f11") + @handle("f12") + @handle("f13") + @handle("f14") + @handle("f15") + @handle("f16") + @handle("f17") + @handle("f18") + @handle("f19") + @handle("f20") + @handle("f21") + @handle("f22") + @handle("f23") + @handle("f24") + @handle("c-@") # Also c-space. + @handle("c-\\") + @handle("c-]") + @handle("c-^") + @handle("c-_") + @handle("backspace") + @handle("up") + @handle("down") + @handle("right") + @handle("left") + @handle("s-up") + @handle("s-down") + @handle("s-right") + @handle("s-left") + @handle("home") + @handle("end") + @handle("s-home") + @handle("s-end") + @handle("delete") + @handle("s-delete") + @handle("c-delete") + @handle("pageup") + @handle("pagedown") + @handle("s-tab") + @handle("tab") + @handle("c-s-left") + @handle("c-s-right") + @handle("c-s-home") + @handle("c-s-end") + @handle("c-left") + @handle("c-right") + @handle("c-up") + @handle("c-down") + @handle("c-home") + @handle("c-end") + @handle("insert") + @handle("s-insert") + @handle("c-insert") + @handle("") + @handle(Keys.Ignore) + def _ignore(event: E) -> None: + """ + First, for any of these keys, Don't do anything by default. Also don't + catch them in the 'Any' handler which will insert them as data. + + If people want to insert these characters as a literal, they can always + do by doing a quoted insert. (ControlQ in emacs mode, ControlV in Vi + mode.) + """ + pass + + # Readline-style bindings. + handle("home")(get_by_name("beginning-of-line")) + handle("end")(get_by_name("end-of-line")) + handle("left")(get_by_name("backward-char")) + handle("right")(get_by_name("forward-char")) + handle("c-up")(get_by_name("previous-history")) + handle("c-down")(get_by_name("next-history")) + handle("c-l")(get_by_name("clear-screen")) + + handle("c-k", filter=insert_mode)(get_by_name("kill-line")) + handle("c-u", filter=insert_mode)(get_by_name("unix-line-discard")) + handle("backspace", filter=insert_mode, save_before=if_no_repeat)( + get_by_name("backward-delete-char") + ) + handle("delete", filter=insert_mode, save_before=if_no_repeat)( + get_by_name("delete-char") + ) + handle("c-delete", filter=insert_mode, save_before=if_no_repeat)( + get_by_name("delete-char") + ) + handle(Keys.Any, filter=insert_mode, save_before=if_no_repeat)( + get_by_name("self-insert") + ) + handle("c-t", filter=insert_mode)(get_by_name("transpose-chars")) + handle("c-i", filter=insert_mode)(get_by_name("menu-complete")) + handle("s-tab", filter=insert_mode)(get_by_name("menu-complete-backward")) + + # Control-W should delete, using whitespace as separator, while M-Del + # should delete using [^a-zA-Z0-9] as a boundary. + handle("c-w", filter=insert_mode)(get_by_name("unix-word-rubout")) + + handle("pageup", filter=~has_selection)(get_by_name("previous-history")) + handle("pagedown", filter=~has_selection)(get_by_name("next-history")) + + # CTRL keys. + + handle("c-d", filter=has_text_before_cursor & insert_mode)( + get_by_name("delete-char") + ) + + @handle("enter", filter=insert_mode & is_multiline) + def _newline(event: E) -> None: + """ + Newline (in case of multiline input. + """ + event.current_buffer.newline(copy_margin=not in_paste_mode()) + + @handle("c-j") + def _newline2(event: E) -> None: + r""" + By default, handle \n as if it were a \r (enter). + (It appears that some terminals send \n instead of \r when pressing + enter. - at least the Linux subsystem for Windows.) + """ + event.key_processor.feed(KeyPress(Keys.ControlM, "\r"), first=True) + + # Delete the word before the cursor. + + @handle("up") + def _go_up(event: E) -> None: + event.current_buffer.auto_up(count=event.arg) + + @handle("down") + def _go_down(event: E) -> None: + event.current_buffer.auto_down(count=event.arg) + + @handle("delete", filter=has_selection) + def _cut(event: E) -> None: + data = event.current_buffer.cut_selection() + event.app.clipboard.set_data(data) + + # Global bindings. + + @handle("c-z") + def _insert_ctrl_z(event: E) -> None: + """ + By default, control-Z should literally insert Ctrl-Z. + (Ansi Ctrl-Z, code 26 in MSDOS means End-Of-File. + In a Python REPL for instance, it's possible to type + Control-Z followed by enter to quit.) + + When the system bindings are loaded and suspend-to-background is + supported, that will override this binding. + """ + event.current_buffer.insert_text(event.data) + + @handle(Keys.BracketedPaste) + def _paste(event: E) -> None: + """ + Pasting from clipboard. + """ + data = event.data + + # Be sure to use \n as line ending. + # Some terminals (Like iTerm2) seem to paste \r\n line endings in a + # bracketed paste. See: https://github.com/ipython/ipython/issues/9737 + data = data.replace("\r\n", "\n") + data = data.replace("\r", "\n") + + event.current_buffer.insert_text(data) + + @handle(Keys.Any, filter=in_quoted_insert, eager=True) + def _insert_text(event: E) -> None: + """ + Handle quoted insert. + """ + event.current_buffer.insert_text(event.data, overwrite=False) + event.app.quoted_insert = False + + return key_bindings diff --git a/lib/prompt_toolkit/key_binding/bindings/completion.py b/lib/prompt_toolkit/key_binding/bindings/completion.py new file mode 100644 index 0000000..8c5e005 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/completion.py @@ -0,0 +1,206 @@ +""" +Key binding handlers for displaying completions. +""" + +from __future__ import annotations + +import asyncio +import math +from typing import TYPE_CHECKING + +from prompt_toolkit.application.run_in_terminal import in_terminal +from prompt_toolkit.completion import ( + CompleteEvent, + Completion, + get_common_complete_suffix, +) +from prompt_toolkit.formatted_text import StyleAndTextTuples +from prompt_toolkit.key_binding.key_bindings import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.utils import get_cwidth + +if TYPE_CHECKING: + from prompt_toolkit.application import Application + from prompt_toolkit.shortcuts import PromptSession + +__all__ = [ + "generate_completions", + "display_completions_like_readline", +] + +E = KeyPressEvent + + +def generate_completions(event: E) -> None: + r""" + Tab-completion: where the first tab completes the common suffix and the + second tab lists all the completions. + """ + b = event.current_buffer + + # When already navigating through completions, select the next one. + if b.complete_state: + b.complete_next() + else: + b.start_completion(insert_common_part=True) + + +def display_completions_like_readline(event: E) -> None: + """ + Key binding handler for readline-style tab completion. + This is meant to be as similar as possible to the way how readline displays + completions. + + Generate the completions immediately (blocking) and display them above the + prompt in columns. + + Usage:: + + # Call this handler when 'Tab' has been pressed. + key_bindings.add(Keys.ControlI)(display_completions_like_readline) + """ + # Request completions. + b = event.current_buffer + if b.completer is None: + return + complete_event = CompleteEvent(completion_requested=True) + completions = list(b.completer.get_completions(b.document, complete_event)) + + # Calculate the common suffix. + common_suffix = get_common_complete_suffix(b.document, completions) + + # One completion: insert it. + if len(completions) == 1: + b.delete_before_cursor(-completions[0].start_position) + b.insert_text(completions[0].text) + # Multiple completions with common part. + elif common_suffix: + b.insert_text(common_suffix) + # Otherwise: display all completions. + elif completions: + _display_completions_like_readline(event.app, completions) + + +def _display_completions_like_readline( + app: Application[object], completions: list[Completion] +) -> asyncio.Task[None]: + """ + Display the list of completions in columns above the prompt. + This will ask for a confirmation if there are too many completions to fit + on a single page and provide a paginator to walk through them. + """ + from prompt_toolkit.formatted_text import to_formatted_text + from prompt_toolkit.shortcuts.prompt import create_confirm_session + + # Get terminal dimensions. + term_size = app.output.get_size() + term_width = term_size.columns + term_height = term_size.rows + + # Calculate amount of required columns/rows for displaying the + # completions. (Keep in mind that completions are displayed + # alphabetically column-wise.) + max_compl_width = min( + term_width, max(get_cwidth(c.display_text) for c in completions) + 1 + ) + column_count = max(1, term_width // max_compl_width) + completions_per_page = column_count * (term_height - 1) + page_count = int(math.ceil(len(completions) / float(completions_per_page))) + # Note: math.ceil can return float on Python2. + + def display(page: int) -> None: + # Display completions. + page_completions = completions[ + page * completions_per_page : (page + 1) * completions_per_page + ] + + page_row_count = int(math.ceil(len(page_completions) / float(column_count))) + page_columns = [ + page_completions[i * page_row_count : (i + 1) * page_row_count] + for i in range(column_count) + ] + + result: StyleAndTextTuples = [] + + for r in range(page_row_count): + for c in range(column_count): + try: + completion = page_columns[c][r] + style = "class:readline-like-completions.completion " + ( + completion.style or "" + ) + + result.extend(to_formatted_text(completion.display, style=style)) + + # Add padding. + padding = max_compl_width - get_cwidth(completion.display_text) + result.append((completion.style, " " * padding)) + except IndexError: + pass + result.append(("", "\n")) + + app.print_text(to_formatted_text(result, "class:readline-like-completions")) + + # User interaction through an application generator function. + async def run_compl() -> None: + "Coroutine." + async with in_terminal(render_cli_done=True): + if len(completions) > completions_per_page: + # Ask confirmation if it doesn't fit on the screen. + confirm = await create_confirm_session( + f"Display all {len(completions)} possibilities?", + ).prompt_async() + + if confirm: + # Display pages. + for page in range(page_count): + display(page) + + if page != page_count - 1: + # Display --MORE-- and go to the next page. + show_more = await _create_more_session( + "--MORE--" + ).prompt_async() + + if not show_more: + return + else: + app.output.flush() + else: + # Display all completions. + display(0) + + return app.create_background_task(run_compl()) + + +def _create_more_session(message: str = "--MORE--") -> PromptSession[bool]: + """ + Create a `PromptSession` object for displaying the "--MORE--". + """ + from prompt_toolkit.shortcuts import PromptSession + + bindings = KeyBindings() + + @bindings.add(" ") + @bindings.add("y") + @bindings.add("Y") + @bindings.add(Keys.ControlJ) + @bindings.add(Keys.ControlM) + @bindings.add(Keys.ControlI) # Tab. + def _yes(event: E) -> None: + event.app.exit(result=True) + + @bindings.add("n") + @bindings.add("N") + @bindings.add("q") + @bindings.add("Q") + @bindings.add(Keys.ControlC) + def _no(event: E) -> None: + event.app.exit(result=False) + + @bindings.add(Keys.Any) + def _ignore(event: E) -> None: + "Disable inserting of text." + + return PromptSession(message, key_bindings=bindings, erase_when_done=True) diff --git a/lib/prompt_toolkit/key_binding/bindings/cpr.py b/lib/prompt_toolkit/key_binding/bindings/cpr.py new file mode 100644 index 0000000..cd9df0a --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/cpr.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.keys import Keys + +from ..key_bindings import KeyBindings + +__all__ = [ + "load_cpr_bindings", +] + +E = KeyPressEvent + + +def load_cpr_bindings() -> KeyBindings: + key_bindings = KeyBindings() + + @key_bindings.add(Keys.CPRResponse, save_before=lambda e: False) + def _(event: E) -> None: + """ + Handle incoming Cursor-Position-Request response. + """ + # The incoming data looks like u'\x1b[35;1R' + # Parse row/col information. + row, col = map(int, event.data[2:-1].split(";")) + + # Report absolute cursor position to the renderer. + event.app.renderer.report_absolute_cursor_row(row) + + return key_bindings diff --git a/lib/prompt_toolkit/key_binding/bindings/emacs.py b/lib/prompt_toolkit/key_binding/bindings/emacs.py new file mode 100644 index 0000000..207afba --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/emacs.py @@ -0,0 +1,563 @@ +# pylint: disable=function-redefined +from __future__ import annotations + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.buffer import Buffer, indent, unindent +from prompt_toolkit.completion import CompleteEvent +from prompt_toolkit.filters import ( + Condition, + emacs_insert_mode, + emacs_mode, + has_arg, + has_selection, + in_paste_mode, + is_multiline, + is_read_only, + shift_selection_mode, + vi_search_direction_reversed, +) +from prompt_toolkit.key_binding.key_bindings import Binding +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.selection import SelectionType + +from ..key_bindings import ConditionalKeyBindings, KeyBindings, KeyBindingsBase +from .named_commands import get_by_name + +__all__ = [ + "load_emacs_bindings", + "load_emacs_search_bindings", + "load_emacs_shift_selection_bindings", +] + +E = KeyPressEvent + + +@Condition +def is_returnable() -> bool: + return get_app().current_buffer.is_returnable + + +@Condition +def is_arg() -> bool: + return get_app().key_processor.arg == "-" + + +def load_emacs_bindings() -> KeyBindingsBase: + """ + Some e-macs extensions. + """ + # Overview of Readline emacs commands: + # http://www.catonmat.net/download/readline-emacs-editing-mode-cheat-sheet.pdf + key_bindings = KeyBindings() + handle = key_bindings.add + + insert_mode = emacs_insert_mode + + @handle("escape") + def _esc(event: E) -> None: + """ + By default, ignore escape key. + + (If we don't put this here, and Esc is followed by a key which sequence + is not handled, we'll insert an Escape character in the input stream. + Something we don't want and happens to easily in emacs mode. + Further, people can always use ControlQ to do a quoted insert.) + """ + pass + + handle("c-a")(get_by_name("beginning-of-line")) + handle("c-b")(get_by_name("backward-char")) + handle("c-delete", filter=insert_mode)(get_by_name("kill-word")) + handle("c-e")(get_by_name("end-of-line")) + handle("c-f")(get_by_name("forward-char")) + handle("c-left")(get_by_name("backward-word")) + handle("c-right")(get_by_name("forward-word")) + handle("c-x", "r", "y", filter=insert_mode)(get_by_name("yank")) + handle("c-y", filter=insert_mode)(get_by_name("yank")) + handle("escape", "b")(get_by_name("backward-word")) + handle("escape", "c", filter=insert_mode)(get_by_name("capitalize-word")) + handle("escape", "d", filter=insert_mode)(get_by_name("kill-word")) + handle("escape", "f")(get_by_name("forward-word")) + handle("escape", "l", filter=insert_mode)(get_by_name("downcase-word")) + handle("escape", "u", filter=insert_mode)(get_by_name("uppercase-word")) + handle("escape", "y", filter=insert_mode)(get_by_name("yank-pop")) + handle("escape", "backspace", filter=insert_mode)(get_by_name("backward-kill-word")) + handle("escape", "\\", filter=insert_mode)(get_by_name("delete-horizontal-space")) + + handle("c-home")(get_by_name("beginning-of-buffer")) + handle("c-end")(get_by_name("end-of-buffer")) + + handle("c-_", save_before=(lambda e: False), filter=insert_mode)( + get_by_name("undo") + ) + + handle("c-x", "c-u", save_before=(lambda e: False), filter=insert_mode)( + get_by_name("undo") + ) + + handle("escape", "<", filter=~has_selection)(get_by_name("beginning-of-history")) + handle("escape", ">", filter=~has_selection)(get_by_name("end-of-history")) + + handle("escape", ".", filter=insert_mode)(get_by_name("yank-last-arg")) + handle("escape", "_", filter=insert_mode)(get_by_name("yank-last-arg")) + handle("escape", "c-y", filter=insert_mode)(get_by_name("yank-nth-arg")) + handle("escape", "#", filter=insert_mode)(get_by_name("insert-comment")) + handle("c-o")(get_by_name("operate-and-get-next")) + + # ControlQ does a quoted insert. Not that for vt100 terminals, you have to + # disable flow control by running ``stty -ixon``, otherwise Ctrl-Q and + # Ctrl-S are captured by the terminal. + handle("c-q", filter=~has_selection)(get_by_name("quoted-insert")) + + handle("c-x", "(")(get_by_name("start-kbd-macro")) + handle("c-x", ")")(get_by_name("end-kbd-macro")) + handle("c-x", "e")(get_by_name("call-last-kbd-macro")) + + @handle("c-n") + def _next(event: E) -> None: + "Next line." + event.current_buffer.auto_down() + + @handle("c-p") + def _prev(event: E) -> None: + "Previous line." + event.current_buffer.auto_up(count=event.arg) + + def handle_digit(c: str) -> None: + """ + Handle input of arguments. + The first number needs to be preceded by escape. + """ + + @handle(c, filter=has_arg) + @handle("escape", c) + def _(event: E) -> None: + event.append_to_arg_count(c) + + for c in "0123456789": + handle_digit(c) + + @handle("escape", "-", filter=~has_arg) + def _meta_dash(event: E) -> None: + """""" + if event._arg is None: + event.append_to_arg_count("-") + + @handle("-", filter=is_arg) + def _dash(event: E) -> None: + """ + When '-' is typed again, after exactly '-' has been given as an + argument, ignore this. + """ + event.app.key_processor.arg = "-" + + # Meta + Enter: always accept input. + handle("escape", "enter", filter=insert_mode & is_returnable)( + get_by_name("accept-line") + ) + + # Enter: accept input in single line mode. + handle("enter", filter=insert_mode & is_returnable & ~is_multiline)( + get_by_name("accept-line") + ) + + def character_search(buff: Buffer, char: str, count: int) -> None: + if count < 0: + match = buff.document.find_backwards( + char, in_current_line=True, count=-count + ) + else: + match = buff.document.find(char, in_current_line=True, count=count) + + if match is not None: + buff.cursor_position += match + + @handle("c-]", Keys.Any) + def _goto_char(event: E) -> None: + "When Ctl-] + a character is pressed. go to that character." + # Also named 'character-search' + character_search(event.current_buffer, event.data, event.arg) + + @handle("escape", "c-]", Keys.Any) + def _goto_char_backwards(event: E) -> None: + "Like Ctl-], but backwards." + # Also named 'character-search-backward' + character_search(event.current_buffer, event.data, -event.arg) + + @handle("escape", "a") + def _prev_sentence(event: E) -> None: + "Previous sentence." + # TODO: + + @handle("escape", "e") + def _end_of_sentence(event: E) -> None: + "Move to end of sentence." + # TODO: + + @handle("escape", "t", filter=insert_mode) + def _swap_characters(event: E) -> None: + """ + Swap the last two words before the cursor. + """ + # TODO + + @handle("escape", "*", filter=insert_mode) + def _insert_all_completions(event: E) -> None: + """ + `meta-*`: Insert all possible completions of the preceding text. + """ + buff = event.current_buffer + + # List all completions. + complete_event = CompleteEvent(text_inserted=False, completion_requested=True) + completions = list( + buff.completer.get_completions(buff.document, complete_event) + ) + + # Insert them. + text_to_insert = " ".join(c.text for c in completions) + buff.insert_text(text_to_insert) + + @handle("c-x", "c-x") + def _toggle_start_end(event: E) -> None: + """ + Move cursor back and forth between the start and end of the current + line. + """ + buffer = event.current_buffer + + if buffer.document.is_cursor_at_the_end_of_line: + buffer.cursor_position += buffer.document.get_start_of_line_position( + after_whitespace=False + ) + else: + buffer.cursor_position += buffer.document.get_end_of_line_position() + + @handle("c-@") # Control-space or Control-@ + def _start_selection(event: E) -> None: + """ + Start of the selection (if the current buffer is not empty). + """ + # Take the current cursor position as the start of this selection. + buff = event.current_buffer + if buff.text: + buff.start_selection(selection_type=SelectionType.CHARACTERS) + + @handle("c-g", filter=~has_selection) + def _cancel(event: E) -> None: + """ + Control + G: Cancel completion menu and validation state. + """ + event.current_buffer.complete_state = None + event.current_buffer.validation_error = None + + @handle("c-g", filter=has_selection) + def _cancel_selection(event: E) -> None: + """ + Cancel selection. + """ + event.current_buffer.exit_selection() + + @handle("c-w", filter=has_selection) + @handle("c-x", "r", "k", filter=has_selection) + def _cut(event: E) -> None: + """ + Cut selected text. + """ + data = event.current_buffer.cut_selection() + event.app.clipboard.set_data(data) + + @handle("escape", "w", filter=has_selection) + def _copy(event: E) -> None: + """ + Copy selected text. + """ + data = event.current_buffer.copy_selection() + event.app.clipboard.set_data(data) + + @handle("escape", "left") + def _start_of_word(event: E) -> None: + """ + Cursor to start of previous word. + """ + buffer = event.current_buffer + buffer.cursor_position += ( + buffer.document.find_previous_word_beginning(count=event.arg) or 0 + ) + + @handle("escape", "right") + def _start_next_word(event: E) -> None: + """ + Cursor to start of next word. + """ + buffer = event.current_buffer + buffer.cursor_position += ( + buffer.document.find_next_word_beginning(count=event.arg) + or buffer.document.get_end_of_document_position() + ) + + @handle("escape", "/", filter=insert_mode) + def _complete(event: E) -> None: + """ + M-/: Complete. + """ + b = event.current_buffer + if b.complete_state: + b.complete_next() + else: + b.start_completion(select_first=True) + + @handle("c-c", ">", filter=has_selection) + def _indent(event: E) -> None: + """ + Indent selected text. + """ + buffer = event.current_buffer + + buffer.cursor_position += buffer.document.get_start_of_line_position( + after_whitespace=True + ) + + from_, to = buffer.document.selection_range() + from_, _ = buffer.document.translate_index_to_position(from_) + to, _ = buffer.document.translate_index_to_position(to) + + indent(buffer, from_, to + 1, count=event.arg) + + @handle("c-c", "<", filter=has_selection) + def _unindent(event: E) -> None: + """ + Unindent selected text. + """ + buffer = event.current_buffer + + from_, to = buffer.document.selection_range() + from_, _ = buffer.document.translate_index_to_position(from_) + to, _ = buffer.document.translate_index_to_position(to) + + unindent(buffer, from_, to + 1, count=event.arg) + + return ConditionalKeyBindings(key_bindings, emacs_mode) + + +def load_emacs_search_bindings() -> KeyBindingsBase: + key_bindings = KeyBindings() + handle = key_bindings.add + from . import search + + # NOTE: We don't bind 'Escape' to 'abort_search'. The reason is that we + # want Alt+Enter to accept input directly in incremental search mode. + # Instead, we have double escape. + + handle("c-r")(search.start_reverse_incremental_search) + handle("c-s")(search.start_forward_incremental_search) + + handle("c-c")(search.abort_search) + handle("c-g")(search.abort_search) + handle("c-r")(search.reverse_incremental_search) + handle("c-s")(search.forward_incremental_search) + handle("up")(search.reverse_incremental_search) + handle("down")(search.forward_incremental_search) + handle("enter")(search.accept_search) + + # Handling of escape. + handle("escape", eager=True)(search.accept_search) + + # Like Readline, it's more natural to accept the search when escape has + # been pressed, however instead the following two bindings could be used + # instead. + # #handle('escape', 'escape', eager=True)(search.abort_search) + # #handle('escape', 'enter', eager=True)(search.accept_search_and_accept_input) + + # If Read-only: also include the following key bindings: + + # '/' and '?' key bindings for searching, just like Vi mode. + handle("?", filter=is_read_only & ~vi_search_direction_reversed)( + search.start_reverse_incremental_search + ) + handle("/", filter=is_read_only & ~vi_search_direction_reversed)( + search.start_forward_incremental_search + ) + handle("?", filter=is_read_only & vi_search_direction_reversed)( + search.start_forward_incremental_search + ) + handle("/", filter=is_read_only & vi_search_direction_reversed)( + search.start_reverse_incremental_search + ) + + @handle("n", filter=is_read_only) + def _jump_next(event: E) -> None: + "Jump to next match." + event.current_buffer.apply_search( + event.app.current_search_state, + include_current_position=False, + count=event.arg, + ) + + @handle("N", filter=is_read_only) + def _jump_prev(event: E) -> None: + "Jump to previous match." + event.current_buffer.apply_search( + ~event.app.current_search_state, + include_current_position=False, + count=event.arg, + ) + + return ConditionalKeyBindings(key_bindings, emacs_mode) + + +def load_emacs_shift_selection_bindings() -> KeyBindingsBase: + """ + Bindings to select text with shift + cursor movements + """ + + key_bindings = KeyBindings() + handle = key_bindings.add + + def unshift_move(event: E) -> None: + """ + Used for the shift selection mode. When called with + a shift + movement key press event, moves the cursor + as if shift is not pressed. + """ + key = event.key_sequence[0].key + + if key == Keys.ShiftUp: + event.current_buffer.auto_up(count=event.arg) + return + if key == Keys.ShiftDown: + event.current_buffer.auto_down(count=event.arg) + return + + # the other keys are handled through their readline command + key_to_command: dict[Keys | str, str] = { + Keys.ShiftLeft: "backward-char", + Keys.ShiftRight: "forward-char", + Keys.ShiftHome: "beginning-of-line", + Keys.ShiftEnd: "end-of-line", + Keys.ControlShiftLeft: "backward-word", + Keys.ControlShiftRight: "forward-word", + Keys.ControlShiftHome: "beginning-of-buffer", + Keys.ControlShiftEnd: "end-of-buffer", + } + + try: + # Both the dict lookup and `get_by_name` can raise KeyError. + binding = get_by_name(key_to_command[key]) + except KeyError: + pass + else: # (`else` is not really needed here.) + if isinstance(binding, Binding): + # (It should always be a binding here) + binding.call(event) + + @handle("s-left", filter=~has_selection) + @handle("s-right", filter=~has_selection) + @handle("s-up", filter=~has_selection) + @handle("s-down", filter=~has_selection) + @handle("s-home", filter=~has_selection) + @handle("s-end", filter=~has_selection) + @handle("c-s-left", filter=~has_selection) + @handle("c-s-right", filter=~has_selection) + @handle("c-s-home", filter=~has_selection) + @handle("c-s-end", filter=~has_selection) + def _start_selection(event: E) -> None: + """ + Start selection with shift + movement. + """ + # Take the current cursor position as the start of this selection. + buff = event.current_buffer + if buff.text: + buff.start_selection(selection_type=SelectionType.CHARACTERS) + + if buff.selection_state is not None: + # (`selection_state` should never be `None`, it is created by + # `start_selection`.) + buff.selection_state.enter_shift_mode() + + # Then move the cursor + original_position = buff.cursor_position + unshift_move(event) + if buff.cursor_position == original_position: + # Cursor didn't actually move - so cancel selection + # to avoid having an empty selection + buff.exit_selection() + + @handle("s-left", filter=shift_selection_mode) + @handle("s-right", filter=shift_selection_mode) + @handle("s-up", filter=shift_selection_mode) + @handle("s-down", filter=shift_selection_mode) + @handle("s-home", filter=shift_selection_mode) + @handle("s-end", filter=shift_selection_mode) + @handle("c-s-left", filter=shift_selection_mode) + @handle("c-s-right", filter=shift_selection_mode) + @handle("c-s-home", filter=shift_selection_mode) + @handle("c-s-end", filter=shift_selection_mode) + def _extend_selection(event: E) -> None: + """ + Extend the selection + """ + # Just move the cursor, like shift was not pressed + unshift_move(event) + buff = event.current_buffer + + if buff.selection_state is not None: + if buff.cursor_position == buff.selection_state.original_cursor_position: + # selection is now empty, so cancel selection + buff.exit_selection() + + @handle(Keys.Any, filter=shift_selection_mode) + def _replace_selection(event: E) -> None: + """ + Replace selection by what is typed + """ + event.current_buffer.cut_selection() + get_by_name("self-insert").call(event) + + @handle("enter", filter=shift_selection_mode & is_multiline) + def _newline(event: E) -> None: + """ + A newline replaces the selection + """ + event.current_buffer.cut_selection() + event.current_buffer.newline(copy_margin=not in_paste_mode()) + + @handle("backspace", filter=shift_selection_mode) + def _delete(event: E) -> None: + """ + Delete selection. + """ + event.current_buffer.cut_selection() + + @handle("c-y", filter=shift_selection_mode) + def _yank(event: E) -> None: + """ + In shift selection mode, yanking (pasting) replace the selection. + """ + buff = event.current_buffer + if buff.selection_state: + buff.cut_selection() + get_by_name("yank").call(event) + + # moving the cursor in shift selection mode cancels the selection + @handle("left", filter=shift_selection_mode) + @handle("right", filter=shift_selection_mode) + @handle("up", filter=shift_selection_mode) + @handle("down", filter=shift_selection_mode) + @handle("home", filter=shift_selection_mode) + @handle("end", filter=shift_selection_mode) + @handle("c-left", filter=shift_selection_mode) + @handle("c-right", filter=shift_selection_mode) + @handle("c-home", filter=shift_selection_mode) + @handle("c-end", filter=shift_selection_mode) + def _cancel(event: E) -> None: + """ + Cancel selection. + """ + event.current_buffer.exit_selection() + # we then process the cursor movement + key_press = event.key_sequence[0] + event.key_processor.feed(key_press, first=True) + + return ConditionalKeyBindings(key_bindings, emacs_mode) diff --git a/lib/prompt_toolkit/key_binding/bindings/focus.py b/lib/prompt_toolkit/key_binding/bindings/focus.py new file mode 100644 index 0000000..24aa3ce --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/focus.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from prompt_toolkit.key_binding.key_processor import KeyPressEvent + +__all__ = [ + "focus_next", + "focus_previous", +] + +E = KeyPressEvent + + +def focus_next(event: E) -> None: + """ + Focus the next visible Window. + (Often bound to the `Tab` key.) + """ + event.app.layout.focus_next() + + +def focus_previous(event: E) -> None: + """ + Focus the previous visible Window. + (Often bound to the `BackTab` key.) + """ + event.app.layout.focus_previous() diff --git a/lib/prompt_toolkit/key_binding/bindings/mouse.py b/lib/prompt_toolkit/key_binding/bindings/mouse.py new file mode 100644 index 0000000..cb426ce --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/mouse.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +from prompt_toolkit.data_structures import Point +from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.mouse_events import ( + MouseButton, + MouseEvent, + MouseEventType, + MouseModifier, +) + +from ..key_bindings import KeyBindings + +if TYPE_CHECKING: + from prompt_toolkit.key_binding.key_bindings import NotImplementedOrNone + +__all__ = [ + "load_mouse_bindings", +] + +E = KeyPressEvent + +# fmt: off +SCROLL_UP = MouseEventType.SCROLL_UP +SCROLL_DOWN = MouseEventType.SCROLL_DOWN +MOUSE_DOWN = MouseEventType.MOUSE_DOWN +MOUSE_MOVE = MouseEventType.MOUSE_MOVE +MOUSE_UP = MouseEventType.MOUSE_UP + +NO_MODIFIER : frozenset[MouseModifier] = frozenset() +SHIFT : frozenset[MouseModifier] = frozenset({MouseModifier.SHIFT}) +ALT : frozenset[MouseModifier] = frozenset({MouseModifier.ALT}) +SHIFT_ALT : frozenset[MouseModifier] = frozenset({MouseModifier.SHIFT, MouseModifier.ALT}) +CONTROL : frozenset[MouseModifier] = frozenset({MouseModifier.CONTROL}) +SHIFT_CONTROL : frozenset[MouseModifier] = frozenset({MouseModifier.SHIFT, MouseModifier.CONTROL}) +ALT_CONTROL : frozenset[MouseModifier] = frozenset({MouseModifier.ALT, MouseModifier.CONTROL}) +SHIFT_ALT_CONTROL: frozenset[MouseModifier] = frozenset({MouseModifier.SHIFT, MouseModifier.ALT, MouseModifier.CONTROL}) +UNKNOWN_MODIFIER : frozenset[MouseModifier] = frozenset() + +LEFT = MouseButton.LEFT +MIDDLE = MouseButton.MIDDLE +RIGHT = MouseButton.RIGHT +NO_BUTTON = MouseButton.NONE +UNKNOWN_BUTTON = MouseButton.UNKNOWN + +xterm_sgr_mouse_events = { + ( 0, "m") : (LEFT, MOUSE_UP, NO_MODIFIER), # left_up 0+ + + =0 + ( 4, "m") : (LEFT, MOUSE_UP, SHIFT), # left_up Shift 0+4+ + =4 + ( 8, "m") : (LEFT, MOUSE_UP, ALT), # left_up Alt 0+ +8+ =8 + (12, "m") : (LEFT, MOUSE_UP, SHIFT_ALT), # left_up Shift Alt 0+4+8+ =12 + (16, "m") : (LEFT, MOUSE_UP, CONTROL), # left_up Control 0+ + +16=16 + (20, "m") : (LEFT, MOUSE_UP, SHIFT_CONTROL), # left_up Shift Control 0+4+ +16=20 + (24, "m") : (LEFT, MOUSE_UP, ALT_CONTROL), # left_up Alt Control 0+ +8+16=24 + (28, "m") : (LEFT, MOUSE_UP, SHIFT_ALT_CONTROL), # left_up Shift Alt Control 0+4+8+16=28 + + ( 1, "m") : (MIDDLE, MOUSE_UP, NO_MODIFIER), # middle_up 1+ + + =1 + ( 5, "m") : (MIDDLE, MOUSE_UP, SHIFT), # middle_up Shift 1+4+ + =5 + ( 9, "m") : (MIDDLE, MOUSE_UP, ALT), # middle_up Alt 1+ +8+ =9 + (13, "m") : (MIDDLE, MOUSE_UP, SHIFT_ALT), # middle_up Shift Alt 1+4+8+ =13 + (17, "m") : (MIDDLE, MOUSE_UP, CONTROL), # middle_up Control 1+ + +16=17 + (21, "m") : (MIDDLE, MOUSE_UP, SHIFT_CONTROL), # middle_up Shift Control 1+4+ +16=21 + (25, "m") : (MIDDLE, MOUSE_UP, ALT_CONTROL), # middle_up Alt Control 1+ +8+16=25 + (29, "m") : (MIDDLE, MOUSE_UP, SHIFT_ALT_CONTROL), # middle_up Shift Alt Control 1+4+8+16=29 + + ( 2, "m") : (RIGHT, MOUSE_UP, NO_MODIFIER), # right_up 2+ + + =2 + ( 6, "m") : (RIGHT, MOUSE_UP, SHIFT), # right_up Shift 2+4+ + =6 + (10, "m") : (RIGHT, MOUSE_UP, ALT), # right_up Alt 2+ +8+ =10 + (14, "m") : (RIGHT, MOUSE_UP, SHIFT_ALT), # right_up Shift Alt 2+4+8+ =14 + (18, "m") : (RIGHT, MOUSE_UP, CONTROL), # right_up Control 2+ + +16=18 + (22, "m") : (RIGHT, MOUSE_UP, SHIFT_CONTROL), # right_up Shift Control 2+4+ +16=22 + (26, "m") : (RIGHT, MOUSE_UP, ALT_CONTROL), # right_up Alt Control 2+ +8+16=26 + (30, "m") : (RIGHT, MOUSE_UP, SHIFT_ALT_CONTROL), # right_up Shift Alt Control 2+4+8+16=30 + + ( 0, "M") : (LEFT, MOUSE_DOWN, NO_MODIFIER), # left_down 0+ + + =0 + ( 4, "M") : (LEFT, MOUSE_DOWN, SHIFT), # left_down Shift 0+4+ + =4 + ( 8, "M") : (LEFT, MOUSE_DOWN, ALT), # left_down Alt 0+ +8+ =8 + (12, "M") : (LEFT, MOUSE_DOWN, SHIFT_ALT), # left_down Shift Alt 0+4+8+ =12 + (16, "M") : (LEFT, MOUSE_DOWN, CONTROL), # left_down Control 0+ + +16=16 + (20, "M") : (LEFT, MOUSE_DOWN, SHIFT_CONTROL), # left_down Shift Control 0+4+ +16=20 + (24, "M") : (LEFT, MOUSE_DOWN, ALT_CONTROL), # left_down Alt Control 0+ +8+16=24 + (28, "M") : (LEFT, MOUSE_DOWN, SHIFT_ALT_CONTROL), # left_down Shift Alt Control 0+4+8+16=28 + + ( 1, "M") : (MIDDLE, MOUSE_DOWN, NO_MODIFIER), # middle_down 1+ + + =1 + ( 5, "M") : (MIDDLE, MOUSE_DOWN, SHIFT), # middle_down Shift 1+4+ + =5 + ( 9, "M") : (MIDDLE, MOUSE_DOWN, ALT), # middle_down Alt 1+ +8+ =9 + (13, "M") : (MIDDLE, MOUSE_DOWN, SHIFT_ALT), # middle_down Shift Alt 1+4+8+ =13 + (17, "M") : (MIDDLE, MOUSE_DOWN, CONTROL), # middle_down Control 1+ + +16=17 + (21, "M") : (MIDDLE, MOUSE_DOWN, SHIFT_CONTROL), # middle_down Shift Control 1+4+ +16=21 + (25, "M") : (MIDDLE, MOUSE_DOWN, ALT_CONTROL), # middle_down Alt Control 1+ +8+16=25 + (29, "M") : (MIDDLE, MOUSE_DOWN, SHIFT_ALT_CONTROL), # middle_down Shift Alt Control 1+4+8+16=29 + + ( 2, "M") : (RIGHT, MOUSE_DOWN, NO_MODIFIER), # right_down 2+ + + =2 + ( 6, "M") : (RIGHT, MOUSE_DOWN, SHIFT), # right_down Shift 2+4+ + =6 + (10, "M") : (RIGHT, MOUSE_DOWN, ALT), # right_down Alt 2+ +8+ =10 + (14, "M") : (RIGHT, MOUSE_DOWN, SHIFT_ALT), # right_down Shift Alt 2+4+8+ =14 + (18, "M") : (RIGHT, MOUSE_DOWN, CONTROL), # right_down Control 2+ + +16=18 + (22, "M") : (RIGHT, MOUSE_DOWN, SHIFT_CONTROL), # right_down Shift Control 2+4+ +16=22 + (26, "M") : (RIGHT, MOUSE_DOWN, ALT_CONTROL), # right_down Alt Control 2+ +8+16=26 + (30, "M") : (RIGHT, MOUSE_DOWN, SHIFT_ALT_CONTROL), # right_down Shift Alt Control 2+4+8+16=30 + + (32, "M") : (LEFT, MOUSE_MOVE, NO_MODIFIER), # left_drag 32+ + + =32 + (36, "M") : (LEFT, MOUSE_MOVE, SHIFT), # left_drag Shift 32+4+ + =36 + (40, "M") : (LEFT, MOUSE_MOVE, ALT), # left_drag Alt 32+ +8+ =40 + (44, "M") : (LEFT, MOUSE_MOVE, SHIFT_ALT), # left_drag Shift Alt 32+4+8+ =44 + (48, "M") : (LEFT, MOUSE_MOVE, CONTROL), # left_drag Control 32+ + +16=48 + (52, "M") : (LEFT, MOUSE_MOVE, SHIFT_CONTROL), # left_drag Shift Control 32+4+ +16=52 + (56, "M") : (LEFT, MOUSE_MOVE, ALT_CONTROL), # left_drag Alt Control 32+ +8+16=56 + (60, "M") : (LEFT, MOUSE_MOVE, SHIFT_ALT_CONTROL), # left_drag Shift Alt Control 32+4+8+16=60 + + (33, "M") : (MIDDLE, MOUSE_MOVE, NO_MODIFIER), # middle_drag 33+ + + =33 + (37, "M") : (MIDDLE, MOUSE_MOVE, SHIFT), # middle_drag Shift 33+4+ + =37 + (41, "M") : (MIDDLE, MOUSE_MOVE, ALT), # middle_drag Alt 33+ +8+ =41 + (45, "M") : (MIDDLE, MOUSE_MOVE, SHIFT_ALT), # middle_drag Shift Alt 33+4+8+ =45 + (49, "M") : (MIDDLE, MOUSE_MOVE, CONTROL), # middle_drag Control 33+ + +16=49 + (53, "M") : (MIDDLE, MOUSE_MOVE, SHIFT_CONTROL), # middle_drag Shift Control 33+4+ +16=53 + (57, "M") : (MIDDLE, MOUSE_MOVE, ALT_CONTROL), # middle_drag Alt Control 33+ +8+16=57 + (61, "M") : (MIDDLE, MOUSE_MOVE, SHIFT_ALT_CONTROL), # middle_drag Shift Alt Control 33+4+8+16=61 + + (34, "M") : (RIGHT, MOUSE_MOVE, NO_MODIFIER), # right_drag 34+ + + =34 + (38, "M") : (RIGHT, MOUSE_MOVE, SHIFT), # right_drag Shift 34+4+ + =38 + (42, "M") : (RIGHT, MOUSE_MOVE, ALT), # right_drag Alt 34+ +8+ =42 + (46, "M") : (RIGHT, MOUSE_MOVE, SHIFT_ALT), # right_drag Shift Alt 34+4+8+ =46 + (50, "M") : (RIGHT, MOUSE_MOVE, CONTROL), # right_drag Control 34+ + +16=50 + (54, "M") : (RIGHT, MOUSE_MOVE, SHIFT_CONTROL), # right_drag Shift Control 34+4+ +16=54 + (58, "M") : (RIGHT, MOUSE_MOVE, ALT_CONTROL), # right_drag Alt Control 34+ +8+16=58 + (62, "M") : (RIGHT, MOUSE_MOVE, SHIFT_ALT_CONTROL), # right_drag Shift Alt Control 34+4+8+16=62 + + (35, "M") : (NO_BUTTON, MOUSE_MOVE, NO_MODIFIER), # none_drag 35+ + + =35 + (39, "M") : (NO_BUTTON, MOUSE_MOVE, SHIFT), # none_drag Shift 35+4+ + =39 + (43, "M") : (NO_BUTTON, MOUSE_MOVE, ALT), # none_drag Alt 35+ +8+ =43 + (47, "M") : (NO_BUTTON, MOUSE_MOVE, SHIFT_ALT), # none_drag Shift Alt 35+4+8+ =47 + (51, "M") : (NO_BUTTON, MOUSE_MOVE, CONTROL), # none_drag Control 35+ + +16=51 + (55, "M") : (NO_BUTTON, MOUSE_MOVE, SHIFT_CONTROL), # none_drag Shift Control 35+4+ +16=55 + (59, "M") : (NO_BUTTON, MOUSE_MOVE, ALT_CONTROL), # none_drag Alt Control 35+ +8+16=59 + (63, "M") : (NO_BUTTON, MOUSE_MOVE, SHIFT_ALT_CONTROL), # none_drag Shift Alt Control 35+4+8+16=63 + + (64, "M") : (NO_BUTTON, SCROLL_UP, NO_MODIFIER), # scroll_up 64+ + + =64 + (68, "M") : (NO_BUTTON, SCROLL_UP, SHIFT), # scroll_up Shift 64+4+ + =68 + (72, "M") : (NO_BUTTON, SCROLL_UP, ALT), # scroll_up Alt 64+ +8+ =72 + (76, "M") : (NO_BUTTON, SCROLL_UP, SHIFT_ALT), # scroll_up Shift Alt 64+4+8+ =76 + (80, "M") : (NO_BUTTON, SCROLL_UP, CONTROL), # scroll_up Control 64+ + +16=80 + (84, "M") : (NO_BUTTON, SCROLL_UP, SHIFT_CONTROL), # scroll_up Shift Control 64+4+ +16=84 + (88, "M") : (NO_BUTTON, SCROLL_UP, ALT_CONTROL), # scroll_up Alt Control 64+ +8+16=88 + (92, "M") : (NO_BUTTON, SCROLL_UP, SHIFT_ALT_CONTROL), # scroll_up Shift Alt Control 64+4+8+16=92 + + (65, "M") : (NO_BUTTON, SCROLL_DOWN, NO_MODIFIER), # scroll_down 64+ + + =65 + (69, "M") : (NO_BUTTON, SCROLL_DOWN, SHIFT), # scroll_down Shift 64+4+ + =69 + (73, "M") : (NO_BUTTON, SCROLL_DOWN, ALT), # scroll_down Alt 64+ +8+ =73 + (77, "M") : (NO_BUTTON, SCROLL_DOWN, SHIFT_ALT), # scroll_down Shift Alt 64+4+8+ =77 + (81, "M") : (NO_BUTTON, SCROLL_DOWN, CONTROL), # scroll_down Control 64+ + +16=81 + (85, "M") : (NO_BUTTON, SCROLL_DOWN, SHIFT_CONTROL), # scroll_down Shift Control 64+4+ +16=85 + (89, "M") : (NO_BUTTON, SCROLL_DOWN, ALT_CONTROL), # scroll_down Alt Control 64+ +8+16=89 + (93, "M") : (NO_BUTTON, SCROLL_DOWN, SHIFT_ALT_CONTROL), # scroll_down Shift Alt Control 64+4+8+16=93 +} + +typical_mouse_events = { + 32: (LEFT , MOUSE_DOWN , UNKNOWN_MODIFIER), + 33: (MIDDLE , MOUSE_DOWN , UNKNOWN_MODIFIER), + 34: (RIGHT , MOUSE_DOWN , UNKNOWN_MODIFIER), + 35: (UNKNOWN_BUTTON , MOUSE_UP , UNKNOWN_MODIFIER), + + 64: (LEFT , MOUSE_MOVE , UNKNOWN_MODIFIER), + 65: (MIDDLE , MOUSE_MOVE , UNKNOWN_MODIFIER), + 66: (RIGHT , MOUSE_MOVE , UNKNOWN_MODIFIER), + 67: (NO_BUTTON , MOUSE_MOVE , UNKNOWN_MODIFIER), + + 96: (NO_BUTTON , SCROLL_UP , UNKNOWN_MODIFIER), + 97: (NO_BUTTON , SCROLL_DOWN, UNKNOWN_MODIFIER), +} + +urxvt_mouse_events={ + 32: (UNKNOWN_BUTTON, MOUSE_DOWN , UNKNOWN_MODIFIER), + 35: (UNKNOWN_BUTTON, MOUSE_UP , UNKNOWN_MODIFIER), + 96: (NO_BUTTON , SCROLL_UP , UNKNOWN_MODIFIER), + 97: (NO_BUTTON , SCROLL_DOWN, UNKNOWN_MODIFIER), +} +# fmt:on + + +def load_mouse_bindings() -> KeyBindings: + """ + Key bindings, required for mouse support. + (Mouse events enter through the key binding system.) + """ + key_bindings = KeyBindings() + + @key_bindings.add(Keys.Vt100MouseEvent) + def _(event: E) -> NotImplementedOrNone: + """ + Handling of incoming mouse event. + """ + # TypicaL: "eSC[MaB*" + # Urxvt: "Esc[96;14;13M" + # Xterm SGR: "Esc[<64;85;12M" + + # Parse incoming packet. + if event.data[2] == "M": + # Typical. + mouse_event, x, y = map(ord, event.data[3:]) + + # TODO: Is it possible to add modifiers here? + mouse_button, mouse_event_type, mouse_modifiers = typical_mouse_events[ + mouse_event + ] + + # Handle situations where `PosixStdinReader` used surrogateescapes. + if x >= 0xDC00: + x -= 0xDC00 + if y >= 0xDC00: + y -= 0xDC00 + + x -= 32 + y -= 32 + else: + # Urxvt and Xterm SGR. + # When the '<' is not present, we are not using the Xterm SGR mode, + # but Urxvt instead. + data = event.data[2:] + if data[:1] == "<": + sgr = True + data = data[1:] + else: + sgr = False + + # Extract coordinates. + mouse_event, x, y = map(int, data[:-1].split(";")) + m = data[-1] + + # Parse event type. + if sgr: + try: + ( + mouse_button, + mouse_event_type, + mouse_modifiers, + ) = xterm_sgr_mouse_events[mouse_event, m] + except KeyError: + return NotImplemented + + else: + # Some other terminals, like urxvt, Hyper terminal, ... + ( + mouse_button, + mouse_event_type, + mouse_modifiers, + ) = urxvt_mouse_events.get( + mouse_event, (UNKNOWN_BUTTON, MOUSE_MOVE, UNKNOWN_MODIFIER) + ) + + x -= 1 + y -= 1 + + # Only handle mouse events when we know the window height. + if event.app.renderer.height_is_known and mouse_event_type is not None: + # Take region above the layout into account. The reported + # coordinates are absolute to the visible part of the terminal. + from prompt_toolkit.renderer import HeightIsUnknownError + + try: + y -= event.app.renderer.rows_above_layout + except HeightIsUnknownError: + return NotImplemented + + # Call the mouse handler from the renderer. + + # Note: This can return `NotImplemented` if no mouse handler was + # found for this position, or if no repainting needs to + # happen. this way, we avoid excessive repaints during mouse + # movements. + handler = event.app.renderer.mouse_handlers.mouse_handlers[y][x] + return handler( + MouseEvent( + position=Point(x=x, y=y), + event_type=mouse_event_type, + button=mouse_button, + modifiers=mouse_modifiers, + ) + ) + + return NotImplemented + + @key_bindings.add(Keys.ScrollUp) + def _scroll_up(event: E) -> None: + """ + Scroll up event without cursor position. + """ + # We don't receive a cursor position, so we don't know which window to + # scroll. Just send an 'up' key press instead. + event.key_processor.feed(KeyPress(Keys.Up), first=True) + + @key_bindings.add(Keys.ScrollDown) + def _scroll_down(event: E) -> None: + """ + Scroll down event without cursor position. + """ + event.key_processor.feed(KeyPress(Keys.Down), first=True) + + @key_bindings.add(Keys.WindowsMouseEvent) + def _mouse(event: E) -> NotImplementedOrNone: + """ + Handling of mouse events for Windows. + """ + # This key binding should only exist for Windows. + if sys.platform == "win32": + # Parse data. + pieces = event.data.split(";") + + button = MouseButton(pieces[0]) + event_type = MouseEventType(pieces[1]) + x = int(pieces[2]) + y = int(pieces[3]) + + # Make coordinates absolute to the visible part of the terminal. + output = event.app.renderer.output + + from prompt_toolkit.output.win32 import Win32Output + from prompt_toolkit.output.windows10 import Windows10_Output + + if isinstance(output, (Win32Output, Windows10_Output)): + screen_buffer_info = output.get_win32_screen_buffer_info() + rows_above_cursor = ( + screen_buffer_info.dwCursorPosition.Y + - event.app.renderer._cursor_pos.y + ) + y -= rows_above_cursor + + # Call the mouse event handler. + # (Can return `NotImplemented`.) + handler = event.app.renderer.mouse_handlers.mouse_handlers[y][x] + + return handler( + MouseEvent( + position=Point(x=x, y=y), + event_type=event_type, + button=button, + modifiers=UNKNOWN_MODIFIER, + ) + ) + + # No mouse handler found. Return `NotImplemented` so that we don't + # invalidate the UI. + return NotImplemented + + return key_bindings diff --git a/lib/prompt_toolkit/key_binding/bindings/named_commands.py b/lib/prompt_toolkit/key_binding/bindings/named_commands.py new file mode 100644 index 0000000..8ea8dd0 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/named_commands.py @@ -0,0 +1,691 @@ +""" +Key bindings which are also known by GNU Readline by the given names. + +See: http://www.delorie.com/gnu/docs/readline/rlman_13.html +""" + +from __future__ import annotations + +from typing import Callable, TypeVar, Union, cast + +from prompt_toolkit.document import Document +from prompt_toolkit.enums import EditingMode +from prompt_toolkit.key_binding.key_bindings import Binding, key_binding +from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.layout.controls import BufferControl +from prompt_toolkit.search import SearchDirection +from prompt_toolkit.selection import PasteMode + +from .completion import display_completions_like_readline, generate_completions + +__all__ = [ + "get_by_name", +] + + +# Typing. +_Handler = Callable[[KeyPressEvent], None] +_HandlerOrBinding = Union[_Handler, Binding] +_T = TypeVar("_T", bound=_HandlerOrBinding) +E = KeyPressEvent + + +# Registry that maps the Readline command names to their handlers. +_readline_commands: dict[str, Binding] = {} + + +def register(name: str) -> Callable[[_T], _T]: + """ + Store handler in the `_readline_commands` dictionary. + """ + + def decorator(handler: _T) -> _T: + "`handler` is a callable or Binding." + if isinstance(handler, Binding): + _readline_commands[name] = handler + else: + _readline_commands[name] = key_binding()(cast(_Handler, handler)) + + return handler + + return decorator + + +def get_by_name(name: str) -> Binding: + """ + Return the handler for the (Readline) command with the given name. + """ + try: + return _readline_commands[name] + except KeyError as e: + raise KeyError(f"Unknown Readline command: {name!r}") from e + + +# +# Commands for moving +# See: http://www.delorie.com/gnu/docs/readline/rlman_14.html +# + + +@register("beginning-of-buffer") +def beginning_of_buffer(event: E) -> None: + """ + Move to the start of the buffer. + """ + buff = event.current_buffer + buff.cursor_position = 0 + + +@register("end-of-buffer") +def end_of_buffer(event: E) -> None: + """ + Move to the end of the buffer. + """ + buff = event.current_buffer + buff.cursor_position = len(buff.text) + + +@register("beginning-of-line") +def beginning_of_line(event: E) -> None: + """ + Move to the start of the current line. + """ + buff = event.current_buffer + buff.cursor_position += buff.document.get_start_of_line_position( + after_whitespace=False + ) + + +@register("end-of-line") +def end_of_line(event: E) -> None: + """ + Move to the end of the line. + """ + buff = event.current_buffer + buff.cursor_position += buff.document.get_end_of_line_position() + + +@register("forward-char") +def forward_char(event: E) -> None: + """ + Move forward a character. + """ + buff = event.current_buffer + buff.cursor_position += buff.document.get_cursor_right_position(count=event.arg) + + +@register("backward-char") +def backward_char(event: E) -> None: + "Move back a character." + buff = event.current_buffer + buff.cursor_position += buff.document.get_cursor_left_position(count=event.arg) + + +@register("forward-word") +def forward_word(event: E) -> None: + """ + Move forward to the end of the next word. Words are composed of letters and + digits. + """ + buff = event.current_buffer + pos = buff.document.find_next_word_ending(count=event.arg) + + if pos: + buff.cursor_position += pos + + +@register("backward-word") +def backward_word(event: E) -> None: + """ + Move back to the start of the current or previous word. Words are composed + of letters and digits. + """ + buff = event.current_buffer + pos = buff.document.find_previous_word_beginning(count=event.arg) + + if pos: + buff.cursor_position += pos + + +@register("clear-screen") +def clear_screen(event: E) -> None: + """ + Clear the screen and redraw everything at the top of the screen. + """ + event.app.renderer.clear() + + +@register("redraw-current-line") +def redraw_current_line(event: E) -> None: + """ + Refresh the current line. + (Readline defines this command, but prompt-toolkit doesn't have it.) + """ + pass + + +# +# Commands for manipulating the history. +# See: http://www.delorie.com/gnu/docs/readline/rlman_15.html +# + + +@register("accept-line") +def accept_line(event: E) -> None: + """ + Accept the line regardless of where the cursor is. + """ + event.current_buffer.validate_and_handle() + + +@register("previous-history") +def previous_history(event: E) -> None: + """ + Move `back` through the history list, fetching the previous command. + """ + event.current_buffer.history_backward(count=event.arg) + + +@register("next-history") +def next_history(event: E) -> None: + """ + Move `forward` through the history list, fetching the next command. + """ + event.current_buffer.history_forward(count=event.arg) + + +@register("beginning-of-history") +def beginning_of_history(event: E) -> None: + """ + Move to the first line in the history. + """ + event.current_buffer.go_to_history(0) + + +@register("end-of-history") +def end_of_history(event: E) -> None: + """ + Move to the end of the input history, i.e., the line currently being entered. + """ + event.current_buffer.history_forward(count=10**100) + buff = event.current_buffer + buff.go_to_history(len(buff._working_lines) - 1) + + +@register("reverse-search-history") +def reverse_search_history(event: E) -> None: + """ + Search backward starting at the current line and moving `up` through + the history as necessary. This is an incremental search. + """ + control = event.app.layout.current_control + + if isinstance(control, BufferControl) and control.search_buffer_control: + event.app.current_search_state.direction = SearchDirection.BACKWARD + event.app.layout.current_control = control.search_buffer_control + + +# +# Commands for changing text +# + + +@register("end-of-file") +def end_of_file(event: E) -> None: + """ + Exit. + """ + event.app.exit() + + +@register("delete-char") +def delete_char(event: E) -> None: + """ + Delete character before the cursor. + """ + deleted = event.current_buffer.delete(count=event.arg) + if not deleted: + event.app.output.bell() + + +@register("backward-delete-char") +def backward_delete_char(event: E) -> None: + """ + Delete the character behind the cursor. + """ + if event.arg < 0: + # When a negative argument has been given, this should delete in front + # of the cursor. + deleted = event.current_buffer.delete(count=-event.arg) + else: + deleted = event.current_buffer.delete_before_cursor(count=event.arg) + + if not deleted: + event.app.output.bell() + + +@register("self-insert") +def self_insert(event: E) -> None: + """ + Insert yourself. + """ + event.current_buffer.insert_text(event.data * event.arg) + + +@register("transpose-chars") +def transpose_chars(event: E) -> None: + """ + Emulate Emacs transpose-char behavior: at the beginning of the buffer, + do nothing. At the end of a line or buffer, swap the characters before + the cursor. Otherwise, move the cursor right, and then swap the + characters before the cursor. + """ + b = event.current_buffer + p = b.cursor_position + if p == 0: + return + elif p == len(b.text) or b.text[p] == "\n": + b.swap_characters_before_cursor() + else: + b.cursor_position += b.document.get_cursor_right_position() + b.swap_characters_before_cursor() + + +@register("uppercase-word") +def uppercase_word(event: E) -> None: + """ + Uppercase the current (or following) word. + """ + buff = event.current_buffer + + for i in range(event.arg): + pos = buff.document.find_next_word_ending() + words = buff.document.text_after_cursor[:pos] + buff.insert_text(words.upper(), overwrite=True) + + +@register("downcase-word") +def downcase_word(event: E) -> None: + """ + Lowercase the current (or following) word. + """ + buff = event.current_buffer + + for i in range(event.arg): # XXX: not DRY: see meta_c and meta_u!! + pos = buff.document.find_next_word_ending() + words = buff.document.text_after_cursor[:pos] + buff.insert_text(words.lower(), overwrite=True) + + +@register("capitalize-word") +def capitalize_word(event: E) -> None: + """ + Capitalize the current (or following) word. + """ + buff = event.current_buffer + + for i in range(event.arg): + pos = buff.document.find_next_word_ending() + words = buff.document.text_after_cursor[:pos] + buff.insert_text(words.title(), overwrite=True) + + +@register("quoted-insert") +def quoted_insert(event: E) -> None: + """ + Add the next character typed to the line verbatim. This is how to insert + key sequences like C-q, for example. + """ + event.app.quoted_insert = True + + +# +# Killing and yanking. +# + + +@register("kill-line") +def kill_line(event: E) -> None: + """ + Kill the text from the cursor to the end of the line. + + If we are at the end of the line, this should remove the newline. + (That way, it is possible to delete multiple lines by executing this + command multiple times.) + """ + buff = event.current_buffer + if event.arg < 0: + deleted = buff.delete_before_cursor( + count=-buff.document.get_start_of_line_position() + ) + else: + if buff.document.current_char == "\n": + deleted = buff.delete(1) + else: + deleted = buff.delete(count=buff.document.get_end_of_line_position()) + event.app.clipboard.set_text(deleted) + + +@register("kill-word") +def kill_word(event: E) -> None: + """ + Kill from point to the end of the current word, or if between words, to the + end of the next word. Word boundaries are the same as forward-word. + """ + buff = event.current_buffer + pos = buff.document.find_next_word_ending(count=event.arg) + + if pos: + deleted = buff.delete(count=pos) + + if event.is_repeat: + deleted = event.app.clipboard.get_data().text + deleted + + event.app.clipboard.set_text(deleted) + + +@register("unix-word-rubout") +def unix_word_rubout(event: E, WORD: bool = True) -> None: + """ + Kill the word behind point, using whitespace as a word boundary. + Usually bound to ControlW. + """ + buff = event.current_buffer + pos = buff.document.find_start_of_previous_word(count=event.arg, WORD=WORD) + + if pos is None: + # Nothing found? delete until the start of the document. (The + # input starts with whitespace and no words were found before the + # cursor.) + pos = -buff.cursor_position + + if pos: + deleted = buff.delete_before_cursor(count=-pos) + + # If the previous key press was also Control-W, concatenate deleted + # text. + if event.is_repeat: + deleted += event.app.clipboard.get_data().text + + event.app.clipboard.set_text(deleted) + else: + # Nothing to delete. Bell. + event.app.output.bell() + + +@register("backward-kill-word") +def backward_kill_word(event: E) -> None: + """ + Kills the word before point, using "not a letter nor a digit" as a word boundary. + Usually bound to M-Del or M-Backspace. + """ + unix_word_rubout(event, WORD=False) + + +@register("delete-horizontal-space") +def delete_horizontal_space(event: E) -> None: + """ + Delete all spaces and tabs around point. + """ + buff = event.current_buffer + text_before_cursor = buff.document.text_before_cursor + text_after_cursor = buff.document.text_after_cursor + + delete_before = len(text_before_cursor) - len(text_before_cursor.rstrip("\t ")) + delete_after = len(text_after_cursor) - len(text_after_cursor.lstrip("\t ")) + + buff.delete_before_cursor(count=delete_before) + buff.delete(count=delete_after) + + +@register("unix-line-discard") +def unix_line_discard(event: E) -> None: + """ + Kill backward from the cursor to the beginning of the current line. + """ + buff = event.current_buffer + + if buff.document.cursor_position_col == 0 and buff.document.cursor_position > 0: + buff.delete_before_cursor(count=1) + else: + deleted = buff.delete_before_cursor( + count=-buff.document.get_start_of_line_position() + ) + event.app.clipboard.set_text(deleted) + + +@register("yank") +def yank(event: E) -> None: + """ + Paste before cursor. + """ + event.current_buffer.paste_clipboard_data( + event.app.clipboard.get_data(), count=event.arg, paste_mode=PasteMode.EMACS + ) + + +@register("yank-nth-arg") +def yank_nth_arg(event: E) -> None: + """ + Insert the first argument of the previous command. With an argument, insert + the nth word from the previous command (start counting at 0). + """ + n = event.arg if event.arg_present else None + event.current_buffer.yank_nth_arg(n) + + +@register("yank-last-arg") +def yank_last_arg(event: E) -> None: + """ + Like `yank_nth_arg`, but if no argument has been given, yank the last word + of each line. + """ + n = event.arg if event.arg_present else None + event.current_buffer.yank_last_arg(n) + + +@register("yank-pop") +def yank_pop(event: E) -> None: + """ + Rotate the kill ring, and yank the new top. Only works following yank or + yank-pop. + """ + buff = event.current_buffer + doc_before_paste = buff.document_before_paste + clipboard = event.app.clipboard + + if doc_before_paste is not None: + buff.document = doc_before_paste + clipboard.rotate() + buff.paste_clipboard_data(clipboard.get_data(), paste_mode=PasteMode.EMACS) + + +# +# Completion. +# + + +@register("complete") +def complete(event: E) -> None: + """ + Attempt to perform completion. + """ + display_completions_like_readline(event) + + +@register("menu-complete") +def menu_complete(event: E) -> None: + """ + Generate completions, or go to the next completion. (This is the default + way of completing input in prompt_toolkit.) + """ + generate_completions(event) + + +@register("menu-complete-backward") +def menu_complete_backward(event: E) -> None: + """ + Move backward through the list of possible completions. + """ + event.current_buffer.complete_previous() + + +# +# Keyboard macros. +# + + +@register("start-kbd-macro") +def start_kbd_macro(event: E) -> None: + """ + Begin saving the characters typed into the current keyboard macro. + """ + event.app.emacs_state.start_macro() + + +@register("end-kbd-macro") +def end_kbd_macro(event: E) -> None: + """ + Stop saving the characters typed into the current keyboard macro and save + the definition. + """ + event.app.emacs_state.end_macro() + + +@register("call-last-kbd-macro") +@key_binding(record_in_macro=False) +def call_last_kbd_macro(event: E) -> None: + """ + Re-execute the last keyboard macro defined, by making the characters in the + macro appear as if typed at the keyboard. + + Notice that we pass `record_in_macro=False`. This ensures that the 'c-x e' + key sequence doesn't appear in the recording itself. This function inserts + the body of the called macro back into the KeyProcessor, so these keys will + be added later on to the macro of their handlers have `record_in_macro=True`. + """ + # Insert the macro. + macro = event.app.emacs_state.macro + + if macro: + event.app.key_processor.feed_multiple(macro, first=True) + + +@register("print-last-kbd-macro") +def print_last_kbd_macro(event: E) -> None: + """ + Print the last keyboard macro. + """ + + # TODO: Make the format suitable for the inputrc file. + def print_macro() -> None: + macro = event.app.emacs_state.macro + if macro: + for k in macro: + print(k) + + from prompt_toolkit.application.run_in_terminal import run_in_terminal + + run_in_terminal(print_macro) + + +# +# Miscellaneous Commands. +# + + +@register("undo") +def undo(event: E) -> None: + """ + Incremental undo. + """ + event.current_buffer.undo() + + +@register("insert-comment") +def insert_comment(event: E) -> None: + """ + Without numeric argument, comment all lines. + With numeric argument, uncomment all lines. + In any case accept the input. + """ + buff = event.current_buffer + + # Transform all lines. + if event.arg != 1: + + def change(line: str) -> str: + return line[1:] if line.startswith("#") else line + + else: + + def change(line: str) -> str: + return "#" + line + + buff.document = Document( + text="\n".join(map(change, buff.text.splitlines())), cursor_position=0 + ) + + # Accept input. + buff.validate_and_handle() + + +@register("vi-editing-mode") +def vi_editing_mode(event: E) -> None: + """ + Switch to Vi editing mode. + """ + event.app.editing_mode = EditingMode.VI + + +@register("emacs-editing-mode") +def emacs_editing_mode(event: E) -> None: + """ + Switch to Emacs editing mode. + """ + event.app.editing_mode = EditingMode.EMACS + + +@register("prefix-meta") +def prefix_meta(event: E) -> None: + """ + Metafy the next character typed. This is for keyboards without a meta key. + + Sometimes people also want to bind other keys to Meta, e.g. 'jj':: + + key_bindings.add_key_binding('j', 'j', filter=ViInsertMode())(prefix_meta) + """ + # ('first' should be true, because we want to insert it at the current + # position in the queue.) + event.app.key_processor.feed(KeyPress(Keys.Escape), first=True) + + +@register("operate-and-get-next") +def operate_and_get_next(event: E) -> None: + """ + Accept the current line for execution and fetch the next line relative to + the current line from the history for editing. + """ + buff = event.current_buffer + new_index = buff.working_index + 1 + + # Accept the current input. (This will also redraw the interface in the + # 'done' state.) + buff.validate_and_handle() + + # Set the new index at the start of the next run. + def set_working_index() -> None: + if new_index < len(buff._working_lines): + buff.working_index = new_index + + event.app.pre_run_callables.append(set_working_index) + + +@register("edit-and-execute-command") +def edit_and_execute(event: E) -> None: + """ + Invoke an editor on the current command line, and accept the result. + """ + buff = event.current_buffer + buff.open_in_editor(validate_and_handle=True) diff --git a/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py b/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py new file mode 100644 index 0000000..26b8685 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py @@ -0,0 +1,52 @@ +""" +Open in editor key bindings. +""" + +from __future__ import annotations + +from prompt_toolkit.filters import emacs_mode, has_selection, vi_navigation_mode + +from ..key_bindings import KeyBindings, KeyBindingsBase, merge_key_bindings +from .named_commands import get_by_name + +__all__ = [ + "load_open_in_editor_bindings", + "load_emacs_open_in_editor_bindings", + "load_vi_open_in_editor_bindings", +] + + +def load_open_in_editor_bindings() -> KeyBindingsBase: + """ + Load both the Vi and emacs key bindings for handling edit-and-execute-command. + """ + return merge_key_bindings( + [ + load_emacs_open_in_editor_bindings(), + load_vi_open_in_editor_bindings(), + ] + ) + + +def load_emacs_open_in_editor_bindings() -> KeyBindings: + """ + Pressing C-X C-E will open the buffer in an external editor. + """ + key_bindings = KeyBindings() + + key_bindings.add("c-x", "c-e", filter=emacs_mode & ~has_selection)( + get_by_name("edit-and-execute-command") + ) + + return key_bindings + + +def load_vi_open_in_editor_bindings() -> KeyBindings: + """ + Pressing 'v' in navigation mode will open the buffer in an external editor. + """ + key_bindings = KeyBindings() + key_bindings.add("v", filter=vi_navigation_mode)( + get_by_name("edit-and-execute-command") + ) + return key_bindings diff --git a/lib/prompt_toolkit/key_binding/bindings/page_navigation.py b/lib/prompt_toolkit/key_binding/bindings/page_navigation.py new file mode 100644 index 0000000..c490425 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/page_navigation.py @@ -0,0 +1,85 @@ +""" +Key bindings for extra page navigation: bindings for up/down scrolling through +long pages, like in Emacs or Vi. +""" + +from __future__ import annotations + +from prompt_toolkit.filters import buffer_has_focus, emacs_mode, vi_mode +from prompt_toolkit.key_binding.key_bindings import ( + ConditionalKeyBindings, + KeyBindings, + KeyBindingsBase, + merge_key_bindings, +) + +from .scroll import ( + scroll_backward, + scroll_forward, + scroll_half_page_down, + scroll_half_page_up, + scroll_one_line_down, + scroll_one_line_up, + scroll_page_down, + scroll_page_up, +) + +__all__ = [ + "load_page_navigation_bindings", + "load_emacs_page_navigation_bindings", + "load_vi_page_navigation_bindings", +] + + +def load_page_navigation_bindings() -> KeyBindingsBase: + """ + Load both the Vi and Emacs bindings for page navigation. + """ + # Only enable when a `Buffer` is focused, otherwise, we would catch keys + # when another widget is focused (like for instance `c-d` in a + # ptterm.Terminal). + return ConditionalKeyBindings( + merge_key_bindings( + [ + load_emacs_page_navigation_bindings(), + load_vi_page_navigation_bindings(), + ] + ), + buffer_has_focus, + ) + + +def load_emacs_page_navigation_bindings() -> KeyBindingsBase: + """ + Key bindings, for scrolling up and down through pages. + This are separate bindings, because GNU readline doesn't have them. + """ + key_bindings = KeyBindings() + handle = key_bindings.add + + handle("c-v")(scroll_page_down) + handle("pagedown")(scroll_page_down) + handle("escape", "v")(scroll_page_up) + handle("pageup")(scroll_page_up) + + return ConditionalKeyBindings(key_bindings, emacs_mode) + + +def load_vi_page_navigation_bindings() -> KeyBindingsBase: + """ + Key bindings, for scrolling up and down through pages. + This are separate bindings, because GNU readline doesn't have them. + """ + key_bindings = KeyBindings() + handle = key_bindings.add + + handle("c-f")(scroll_forward) + handle("c-b")(scroll_backward) + handle("c-d")(scroll_half_page_down) + handle("c-u")(scroll_half_page_up) + handle("c-e")(scroll_one_line_down) + handle("c-y")(scroll_one_line_up) + handle("pagedown")(scroll_page_down) + handle("pageup")(scroll_page_up) + + return ConditionalKeyBindings(key_bindings, vi_mode) diff --git a/lib/prompt_toolkit/key_binding/bindings/scroll.py b/lib/prompt_toolkit/key_binding/bindings/scroll.py new file mode 100644 index 0000000..13e44ed --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/scroll.py @@ -0,0 +1,190 @@ +""" +Key bindings, for scrolling up and down through pages. + +This are separate bindings, because GNU readline doesn't have them, but +they are very useful for navigating through long multiline buffers, like in +Vi, Emacs, etc... +""" + +from __future__ import annotations + +from prompt_toolkit.key_binding.key_processor import KeyPressEvent + +__all__ = [ + "scroll_forward", + "scroll_backward", + "scroll_half_page_up", + "scroll_half_page_down", + "scroll_one_line_up", + "scroll_one_line_down", +] + +E = KeyPressEvent + + +def scroll_forward(event: E, half: bool = False) -> None: + """ + Scroll window down. + """ + w = event.app.layout.current_window + b = event.app.current_buffer + + if w and w.render_info: + info = w.render_info + ui_content = info.ui_content + + # Height to scroll. + scroll_height = info.window_height + if half: + scroll_height //= 2 + + # Calculate how many lines is equivalent to that vertical space. + y = b.document.cursor_position_row + 1 + height = 0 + while y < ui_content.line_count: + line_height = info.get_height_for_line(y) + + if height + line_height < scroll_height: + height += line_height + y += 1 + else: + break + + b.cursor_position = b.document.translate_row_col_to_index(y, 0) + + +def scroll_backward(event: E, half: bool = False) -> None: + """ + Scroll window up. + """ + w = event.app.layout.current_window + b = event.app.current_buffer + + if w and w.render_info: + info = w.render_info + + # Height to scroll. + scroll_height = info.window_height + if half: + scroll_height //= 2 + + # Calculate how many lines is equivalent to that vertical space. + y = max(0, b.document.cursor_position_row - 1) + height = 0 + while y > 0: + line_height = info.get_height_for_line(y) + + if height + line_height < scroll_height: + height += line_height + y -= 1 + else: + break + + b.cursor_position = b.document.translate_row_col_to_index(y, 0) + + +def scroll_half_page_down(event: E) -> None: + """ + Same as ControlF, but only scroll half a page. + """ + scroll_forward(event, half=True) + + +def scroll_half_page_up(event: E) -> None: + """ + Same as ControlB, but only scroll half a page. + """ + scroll_backward(event, half=True) + + +def scroll_one_line_down(event: E) -> None: + """ + scroll_offset += 1 + """ + w = event.app.layout.current_window + b = event.app.current_buffer + + if w: + # When the cursor is at the top, move to the next line. (Otherwise, only scroll.) + if w.render_info: + info = w.render_info + + if w.vertical_scroll < info.content_height - info.window_height: + if info.cursor_position.y <= info.configured_scroll_offsets.top: + b.cursor_position += b.document.get_cursor_down_position() + + w.vertical_scroll += 1 + + +def scroll_one_line_up(event: E) -> None: + """ + scroll_offset -= 1 + """ + w = event.app.layout.current_window + b = event.app.current_buffer + + if w: + # When the cursor is at the bottom, move to the previous line. (Otherwise, only scroll.) + if w.render_info: + info = w.render_info + + if w.vertical_scroll > 0: + first_line_height = info.get_height_for_line(info.first_visible_line()) + + cursor_up = info.cursor_position.y - ( + info.window_height + - 1 + - first_line_height + - info.configured_scroll_offsets.bottom + ) + + # Move cursor up, as many steps as the height of the first line. + # TODO: not entirely correct yet, in case of line wrapping and many long lines. + for _ in range(max(0, cursor_up)): + b.cursor_position += b.document.get_cursor_up_position() + + # Scroll window + w.vertical_scroll -= 1 + + +def scroll_page_down(event: E) -> None: + """ + Scroll page down. (Prefer the cursor at the top of the page, after scrolling.) + """ + w = event.app.layout.current_window + b = event.app.current_buffer + + if w and w.render_info: + # Scroll down one page. + line_index = max(w.render_info.last_visible_line(), w.vertical_scroll + 1) + w.vertical_scroll = line_index + + b.cursor_position = b.document.translate_row_col_to_index(line_index, 0) + b.cursor_position += b.document.get_start_of_line_position( + after_whitespace=True + ) + + +def scroll_page_up(event: E) -> None: + """ + Scroll page up. (Prefer the cursor at the bottom of the page, after scrolling.) + """ + w = event.app.layout.current_window + b = event.app.current_buffer + + if w and w.render_info: + # Put cursor at the first visible line. (But make sure that the cursor + # moves at least one line up.) + line_index = max( + 0, + min(w.render_info.first_visible_line(), b.document.cursor_position_row - 1), + ) + + b.cursor_position = b.document.translate_row_col_to_index(line_index, 0) + b.cursor_position += b.document.get_start_of_line_position( + after_whitespace=True + ) + + # Set the scroll offset. We can safely set it to zero; the Window will + # make sure that it scrolls at least until the cursor becomes visible. + w.vertical_scroll = 0 diff --git a/lib/prompt_toolkit/key_binding/bindings/search.py b/lib/prompt_toolkit/key_binding/bindings/search.py new file mode 100644 index 0000000..a57c52e --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/search.py @@ -0,0 +1,96 @@ +""" +Search related key bindings. +""" + +from __future__ import annotations + +from prompt_toolkit import search +from prompt_toolkit.application.current import get_app +from prompt_toolkit.filters import Condition, control_is_searchable, is_searching +from prompt_toolkit.key_binding.key_processor import KeyPressEvent + +from ..key_bindings import key_binding + +__all__ = [ + "abort_search", + "accept_search", + "start_reverse_incremental_search", + "start_forward_incremental_search", + "reverse_incremental_search", + "forward_incremental_search", + "accept_search_and_accept_input", +] + +E = KeyPressEvent + + +@key_binding(filter=is_searching) +def abort_search(event: E) -> None: + """ + Abort an incremental search and restore the original + line. + (Usually bound to ControlG/ControlC.) + """ + search.stop_search() + + +@key_binding(filter=is_searching) +def accept_search(event: E) -> None: + """ + When enter pressed in isearch, quit isearch mode. (Multiline + isearch would be too complicated.) + (Usually bound to Enter.) + """ + search.accept_search() + + +@key_binding(filter=control_is_searchable) +def start_reverse_incremental_search(event: E) -> None: + """ + Enter reverse incremental search. + (Usually ControlR.) + """ + search.start_search(direction=search.SearchDirection.BACKWARD) + + +@key_binding(filter=control_is_searchable) +def start_forward_incremental_search(event: E) -> None: + """ + Enter forward incremental search. + (Usually ControlS.) + """ + search.start_search(direction=search.SearchDirection.FORWARD) + + +@key_binding(filter=is_searching) +def reverse_incremental_search(event: E) -> None: + """ + Apply reverse incremental search, but keep search buffer focused. + """ + search.do_incremental_search(search.SearchDirection.BACKWARD, count=event.arg) + + +@key_binding(filter=is_searching) +def forward_incremental_search(event: E) -> None: + """ + Apply forward incremental search, but keep search buffer focused. + """ + search.do_incremental_search(search.SearchDirection.FORWARD, count=event.arg) + + +@Condition +def _previous_buffer_is_returnable() -> bool: + """ + True if the previously focused buffer has a return handler. + """ + prev_control = get_app().layout.search_target_buffer_control + return bool(prev_control and prev_control.buffer.is_returnable) + + +@key_binding(filter=is_searching & _previous_buffer_is_returnable) +def accept_search_and_accept_input(event: E) -> None: + """ + Accept the search operation first, then accept the input. + """ + search.accept_search() + event.current_buffer.validate_and_handle() diff --git a/lib/prompt_toolkit/key_binding/bindings/vi.py b/lib/prompt_toolkit/key_binding/bindings/vi.py new file mode 100644 index 0000000..d68a31f --- /dev/null +++ b/lib/prompt_toolkit/key_binding/bindings/vi.py @@ -0,0 +1,2233 @@ +# pylint: disable=function-redefined +from __future__ import annotations + +import codecs +import string +from enum import Enum +from itertools import accumulate +from typing import Callable, Iterable, Tuple, TypeVar + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.buffer import Buffer, indent, reshape_text, unindent +from prompt_toolkit.clipboard import ClipboardData +from prompt_toolkit.document import Document +from prompt_toolkit.filters import ( + Always, + Condition, + Filter, + has_arg, + is_read_only, + is_searching, +) +from prompt_toolkit.filters.app import ( + in_paste_mode, + is_multiline, + vi_digraph_mode, + vi_insert_mode, + vi_insert_multiple_mode, + vi_mode, + vi_navigation_mode, + vi_recording_macro, + vi_replace_mode, + vi_replace_single_mode, + vi_search_direction_reversed, + vi_selection_mode, + vi_waiting_for_text_object_mode, +) +from prompt_toolkit.input.vt100_parser import Vt100Parser +from prompt_toolkit.key_binding.digraphs import DIGRAPHS +from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent +from prompt_toolkit.key_binding.vi_state import CharacterFind, InputMode +from prompt_toolkit.keys import Keys +from prompt_toolkit.search import SearchDirection +from prompt_toolkit.selection import PasteMode, SelectionState, SelectionType + +from ..key_bindings import ConditionalKeyBindings, KeyBindings, KeyBindingsBase +from .named_commands import get_by_name + +__all__ = [ + "load_vi_bindings", + "load_vi_search_bindings", +] + +E = KeyPressEvent + +ascii_lowercase = string.ascii_lowercase + +vi_register_names = ascii_lowercase + "0123456789" + + +class TextObjectType(Enum): + EXCLUSIVE = "EXCLUSIVE" + INCLUSIVE = "INCLUSIVE" + LINEWISE = "LINEWISE" + BLOCK = "BLOCK" + + +class TextObject: + """ + Return struct for functions wrapped in ``text_object``. + Both `start` and `end` are relative to the current cursor position. + """ + + def __init__( + self, start: int, end: int = 0, type: TextObjectType = TextObjectType.EXCLUSIVE + ): + self.start = start + self.end = end + self.type = type + + @property + def selection_type(self) -> SelectionType: + if self.type == TextObjectType.LINEWISE: + return SelectionType.LINES + if self.type == TextObjectType.BLOCK: + return SelectionType.BLOCK + else: + return SelectionType.CHARACTERS + + def sorted(self) -> tuple[int, int]: + """ + Return a (start, end) tuple where start <= end. + """ + if self.start < self.end: + return self.start, self.end + else: + return self.end, self.start + + def operator_range(self, document: Document) -> tuple[int, int]: + """ + Return a (start, end) tuple with start <= end that indicates the range + operators should operate on. + `buffer` is used to get start and end of line positions. + + This should return something that can be used in a slice, so the `end` + position is *not* included. + """ + start, end = self.sorted() + doc = document + + if ( + self.type == TextObjectType.EXCLUSIVE + and doc.translate_index_to_position(end + doc.cursor_position)[1] == 0 + ): + # If the motion is exclusive and the end of motion is on the first + # column, the end position becomes end of previous line. + end -= 1 + if self.type == TextObjectType.INCLUSIVE: + end += 1 + if self.type == TextObjectType.LINEWISE: + # Select whole lines + row, col = doc.translate_index_to_position(start + doc.cursor_position) + start = doc.translate_row_col_to_index(row, 0) - doc.cursor_position + row, col = doc.translate_index_to_position(end + doc.cursor_position) + end = ( + doc.translate_row_col_to_index(row, len(doc.lines[row])) + - doc.cursor_position + ) + return start, end + + def get_line_numbers(self, buffer: Buffer) -> tuple[int, int]: + """ + Return a (start_line, end_line) pair. + """ + # Get absolute cursor positions from the text object. + from_, to = self.operator_range(buffer.document) + from_ += buffer.cursor_position + to += buffer.cursor_position + + # Take the start of the lines. + from_, _ = buffer.document.translate_index_to_position(from_) + to, _ = buffer.document.translate_index_to_position(to) + + return from_, to + + def cut(self, buffer: Buffer) -> tuple[Document, ClipboardData]: + """ + Turn text object into `ClipboardData` instance. + """ + from_, to = self.operator_range(buffer.document) + + from_ += buffer.cursor_position + to += buffer.cursor_position + + # For Vi mode, the SelectionState does include the upper position, + # while `self.operator_range` does not. So, go one to the left, unless + # we're in the line mode, then we don't want to risk going to the + # previous line, and missing one line in the selection. + if self.type != TextObjectType.LINEWISE: + to -= 1 + + document = Document( + buffer.text, + to, + SelectionState(original_cursor_position=from_, type=self.selection_type), + ) + + new_document, clipboard_data = document.cut_selection() + return new_document, clipboard_data + + +# Typevar for any text object function: +TextObjectFunction = Callable[[E], TextObject] +_TOF = TypeVar("_TOF", bound=TextObjectFunction) + + +def create_text_object_decorator( + key_bindings: KeyBindings, +) -> Callable[..., Callable[[_TOF], _TOF]]: + """ + Create a decorator that can be used to register Vi text object implementations. + """ + + def text_object_decorator( + *keys: Keys | str, + filter: Filter = Always(), + no_move_handler: bool = False, + no_selection_handler: bool = False, + eager: bool = False, + ) -> Callable[[_TOF], _TOF]: + """ + Register a text object function. + + Usage:: + + @text_object('w', filter=..., no_move_handler=False) + def handler(event): + # Return a text object for this key. + return TextObject(...) + + :param no_move_handler: Disable the move handler in navigation mode. + (It's still active in selection mode.) + """ + + def decorator(text_object_func: _TOF) -> _TOF: + @key_bindings.add( + *keys, filter=vi_waiting_for_text_object_mode & filter, eager=eager + ) + def _apply_operator_to_text_object(event: E) -> None: + # Arguments are multiplied. + vi_state = event.app.vi_state + event._arg = str((vi_state.operator_arg or 1) * (event.arg or 1)) + + # Call the text object handler. + text_obj = text_object_func(event) + + # Get the operator function. + # (Should never be None here, given the + # `vi_waiting_for_text_object_mode` filter state.) + operator_func = vi_state.operator_func + + if text_obj is not None and operator_func is not None: + # Call the operator function with the text object. + operator_func(event, text_obj) + + # Clear operator. + event.app.vi_state.operator_func = None + event.app.vi_state.operator_arg = None + + # Register a move operation. (Doesn't need an operator.) + if not no_move_handler: + + @key_bindings.add( + *keys, + filter=~vi_waiting_for_text_object_mode + & filter + & vi_navigation_mode, + eager=eager, + ) + def _move_in_navigation_mode(event: E) -> None: + """ + Move handler for navigation mode. + """ + text_object = text_object_func(event) + event.current_buffer.cursor_position += text_object.start + + # Register a move selection operation. + if not no_selection_handler: + + @key_bindings.add( + *keys, + filter=~vi_waiting_for_text_object_mode + & filter + & vi_selection_mode, + eager=eager, + ) + def _move_in_selection_mode(event: E) -> None: + """ + Move handler for selection mode. + """ + text_object = text_object_func(event) + buff = event.current_buffer + selection_state = buff.selection_state + + if selection_state is None: + return # Should not happen, because of the `vi_selection_mode` filter. + + # When the text object has both a start and end position, like 'i(' or 'iw', + # Turn this into a selection, otherwise the cursor. + if text_object.end: + # Take selection positions from text object. + start, end = text_object.operator_range(buff.document) + start += buff.cursor_position + end += buff.cursor_position + + selection_state.original_cursor_position = start + buff.cursor_position = end + + # Take selection type from text object. + if text_object.type == TextObjectType.LINEWISE: + selection_state.type = SelectionType.LINES + else: + selection_state.type = SelectionType.CHARACTERS + else: + event.current_buffer.cursor_position += text_object.start + + # Make it possible to chain @text_object decorators. + return text_object_func + + return decorator + + return text_object_decorator + + +# Typevar for any operator function: +OperatorFunction = Callable[[E, TextObject], None] +_OF = TypeVar("_OF", bound=OperatorFunction) + + +def create_operator_decorator( + key_bindings: KeyBindings, +) -> Callable[..., Callable[[_OF], _OF]]: + """ + Create a decorator that can be used for registering Vi operators. + """ + + def operator_decorator( + *keys: Keys | str, filter: Filter = Always(), eager: bool = False + ) -> Callable[[_OF], _OF]: + """ + Register a Vi operator. + + Usage:: + + @operator('d', filter=...) + def handler(event, text_object): + # Do something with the text object here. + """ + + def decorator(operator_func: _OF) -> _OF: + @key_bindings.add( + *keys, + filter=~vi_waiting_for_text_object_mode & filter & vi_navigation_mode, + eager=eager, + ) + def _operator_in_navigation(event: E) -> None: + """ + Handle operator in navigation mode. + """ + # When this key binding is matched, only set the operator + # function in the ViState. We should execute it after a text + # object has been received. + event.app.vi_state.operator_func = operator_func + event.app.vi_state.operator_arg = event.arg + + @key_bindings.add( + *keys, + filter=~vi_waiting_for_text_object_mode & filter & vi_selection_mode, + eager=eager, + ) + def _operator_in_selection(event: E) -> None: + """ + Handle operator in selection mode. + """ + buff = event.current_buffer + selection_state = buff.selection_state + + if selection_state is not None: + # Create text object from selection. + if selection_state.type == SelectionType.LINES: + text_obj_type = TextObjectType.LINEWISE + elif selection_state.type == SelectionType.BLOCK: + text_obj_type = TextObjectType.BLOCK + else: + text_obj_type = TextObjectType.INCLUSIVE + + text_object = TextObject( + selection_state.original_cursor_position - buff.cursor_position, + type=text_obj_type, + ) + + # Execute operator. + operator_func(event, text_object) + + # Quit selection mode. + buff.selection_state = None + + return operator_func + + return decorator + + return operator_decorator + + +@Condition +def is_returnable() -> bool: + return get_app().current_buffer.is_returnable + + +@Condition +def in_block_selection() -> bool: + buff = get_app().current_buffer + return bool( + buff.selection_state and buff.selection_state.type == SelectionType.BLOCK + ) + + +@Condition +def digraph_symbol_1_given() -> bool: + return get_app().vi_state.digraph_symbol1 is not None + + +@Condition +def search_buffer_is_empty() -> bool: + "Returns True when the search buffer is empty." + return get_app().current_buffer.text == "" + + +@Condition +def tilde_operator() -> bool: + return get_app().vi_state.tilde_operator + + +def load_vi_bindings() -> KeyBindingsBase: + """ + Vi extensions. + + # Overview of Readline Vi commands: + # http://www.catonmat.net/download/bash-vi-editing-mode-cheat-sheet.pdf + """ + # Note: Some key bindings have the "~IsReadOnly()" filter added. This + # prevents the handler to be executed when the focus is on a + # read-only buffer. + # This is however only required for those that change the ViState to + # INSERT mode. The `Buffer` class itself throws the + # `EditReadOnlyBuffer` exception for any text operations which is + # handled correctly. There is no need to add "~IsReadOnly" to all key + # bindings that do text manipulation. + + key_bindings = KeyBindings() + handle = key_bindings.add + + # (Note: Always take the navigation bindings in read-only mode, even when + # ViState says different.) + + TransformFunction = Tuple[Tuple[str, ...], Filter, Callable[[str], str]] + + vi_transform_functions: list[TransformFunction] = [ + # Rot 13 transformation + ( + ("g", "?"), + Always(), + lambda string: codecs.encode(string, "rot_13"), + ), + # To lowercase + (("g", "u"), Always(), lambda string: string.lower()), + # To uppercase. + (("g", "U"), Always(), lambda string: string.upper()), + # Swap case. + (("g", "~"), Always(), lambda string: string.swapcase()), + ( + ("~",), + tilde_operator, + lambda string: string.swapcase(), + ), + ] + + # Insert a character literally (quoted insert). + handle("c-v", filter=vi_insert_mode)(get_by_name("quoted-insert")) + + @handle("escape") + def _back_to_navigation(event: E) -> None: + """ + Escape goes to vi navigation mode. + """ + buffer = event.current_buffer + vi_state = event.app.vi_state + + if vi_state.input_mode in (InputMode.INSERT, InputMode.REPLACE): + buffer.cursor_position += buffer.document.get_cursor_left_position() + + vi_state.input_mode = InputMode.NAVIGATION + + if bool(buffer.selection_state): + buffer.exit_selection() + + @handle("k", filter=vi_selection_mode) + def _up_in_selection(event: E) -> None: + """ + Arrow up in selection mode. + """ + event.current_buffer.cursor_up(count=event.arg) + + @handle("j", filter=vi_selection_mode) + def _down_in_selection(event: E) -> None: + """ + Arrow down in selection mode. + """ + event.current_buffer.cursor_down(count=event.arg) + + @handle("up", filter=vi_navigation_mode) + @handle("c-p", filter=vi_navigation_mode) + def _up_in_navigation(event: E) -> None: + """ + Arrow up and ControlP in navigation mode go up. + """ + event.current_buffer.auto_up(count=event.arg) + + @handle("k", filter=vi_navigation_mode) + def _go_up(event: E) -> None: + """ + Go up, but if we enter a new history entry, move to the start of the + line. + """ + event.current_buffer.auto_up( + count=event.arg, go_to_start_of_line_if_history_changes=True + ) + + @handle("down", filter=vi_navigation_mode) + @handle("c-n", filter=vi_navigation_mode) + def _go_down(event: E) -> None: + """ + Arrow down and Control-N in navigation mode. + """ + event.current_buffer.auto_down(count=event.arg) + + @handle("j", filter=vi_navigation_mode) + def _go_down2(event: E) -> None: + """ + Go down, but if we enter a new history entry, go to the start of the line. + """ + event.current_buffer.auto_down( + count=event.arg, go_to_start_of_line_if_history_changes=True + ) + + @handle("backspace", filter=vi_navigation_mode) + def _go_left(event: E) -> None: + """ + In navigation-mode, move cursor. + """ + event.current_buffer.cursor_position += ( + event.current_buffer.document.get_cursor_left_position(count=event.arg) + ) + + @handle("c-n", filter=vi_insert_mode) + def _complete_next(event: E) -> None: + b = event.current_buffer + + if b.complete_state: + b.complete_next() + else: + b.start_completion(select_first=True) + + @handle("c-p", filter=vi_insert_mode) + def _complete_prev(event: E) -> None: + """ + Control-P: To previous completion. + """ + b = event.current_buffer + + if b.complete_state: + b.complete_previous() + else: + b.start_completion(select_last=True) + + @handle("c-g", filter=vi_insert_mode) + @handle("c-y", filter=vi_insert_mode) + def _accept_completion(event: E) -> None: + """ + Accept current completion. + """ + event.current_buffer.complete_state = None + + @handle("c-e", filter=vi_insert_mode) + def _cancel_completion(event: E) -> None: + """ + Cancel completion. Go back to originally typed text. + """ + event.current_buffer.cancel_completion() + + # In navigation mode, pressing enter will always return the input. + handle("enter", filter=vi_navigation_mode & is_returnable)( + get_by_name("accept-line") + ) + + # In insert mode, also accept input when enter is pressed, and the buffer + # has been marked as single line. + handle("enter", filter=is_returnable & ~is_multiline)(get_by_name("accept-line")) + + @handle("enter", filter=~is_returnable & vi_navigation_mode) + def _start_of_next_line(event: E) -> None: + """ + Go to the beginning of next line. + """ + b = event.current_buffer + b.cursor_down(count=event.arg) + b.cursor_position += b.document.get_start_of_line_position( + after_whitespace=True + ) + + # ** In navigation mode ** + + # List of navigation commands: http://hea-www.harvard.edu/~fine/Tech/vi.html + + @handle("insert", filter=vi_navigation_mode) + def _insert_mode(event: E) -> None: + """ + Pressing the Insert key. + """ + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("insert", filter=vi_insert_mode) + def _navigation_mode(event: E) -> None: + """ + Pressing the Insert key. + """ + event.app.vi_state.input_mode = InputMode.NAVIGATION + + @handle("a", filter=vi_navigation_mode & ~is_read_only) + # ~IsReadOnly, because we want to stay in navigation mode for + # read-only buffers. + def _a(event: E) -> None: + event.current_buffer.cursor_position += ( + event.current_buffer.document.get_cursor_right_position() + ) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("A", filter=vi_navigation_mode & ~is_read_only) + def _A(event: E) -> None: + event.current_buffer.cursor_position += ( + event.current_buffer.document.get_end_of_line_position() + ) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("C", filter=vi_navigation_mode & ~is_read_only) + def _change_until_end_of_line(event: E) -> None: + """ + Change to end of line. + Same as 'c$' (which is implemented elsewhere.) + """ + buffer = event.current_buffer + + deleted = buffer.delete(count=buffer.document.get_end_of_line_position()) + event.app.clipboard.set_text(deleted) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("c", "c", filter=vi_navigation_mode & ~is_read_only) + @handle("S", filter=vi_navigation_mode & ~is_read_only) + def _change_current_line(event: E) -> None: # TODO: implement 'arg' + """ + Change current line + """ + buffer = event.current_buffer + + # We copy the whole line. + data = ClipboardData(buffer.document.current_line, SelectionType.LINES) + event.app.clipboard.set_data(data) + + # But we delete after the whitespace + buffer.cursor_position += buffer.document.get_start_of_line_position( + after_whitespace=True + ) + buffer.delete(count=buffer.document.get_end_of_line_position()) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("D", filter=vi_navigation_mode) + def _delete_until_end_of_line(event: E) -> None: + """ + Delete from cursor position until the end of the line. + """ + buffer = event.current_buffer + deleted = buffer.delete(count=buffer.document.get_end_of_line_position()) + event.app.clipboard.set_text(deleted) + + @handle("d", "d", filter=vi_navigation_mode) + def _delete_line(event: E) -> None: + """ + Delete line. (Or the following 'n' lines.) + """ + buffer = event.current_buffer + + # Split string in before/deleted/after text. + lines = buffer.document.lines + + before = "\n".join(lines[: buffer.document.cursor_position_row]) + deleted = "\n".join( + lines[ + buffer.document.cursor_position_row : buffer.document.cursor_position_row + + event.arg + ] + ) + after = "\n".join(lines[buffer.document.cursor_position_row + event.arg :]) + + # Set new text. + if before and after: + before = before + "\n" + + # Set text and cursor position. + buffer.document = Document( + text=before + after, + # Cursor At the start of the first 'after' line, after the leading whitespace. + cursor_position=len(before) + len(after) - len(after.lstrip(" ")), + ) + + # Set clipboard data + event.app.clipboard.set_data(ClipboardData(deleted, SelectionType.LINES)) + + @handle("x", filter=vi_selection_mode) + def _cut(event: E) -> None: + """ + Cut selection. + ('x' is not an operator.) + """ + clipboard_data = event.current_buffer.cut_selection() + event.app.clipboard.set_data(clipboard_data) + + @handle("i", filter=vi_navigation_mode & ~is_read_only) + def _i(event: E) -> None: + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("I", filter=vi_navigation_mode & ~is_read_only) + def _I(event: E) -> None: + event.app.vi_state.input_mode = InputMode.INSERT + event.current_buffer.cursor_position += ( + event.current_buffer.document.get_start_of_line_position( + after_whitespace=True + ) + ) + + @handle("I", filter=in_block_selection & ~is_read_only) + def insert_in_block_selection(event: E, after: bool = False) -> None: + """ + Insert in block selection mode. + """ + buff = event.current_buffer + + # Store all cursor positions. + positions = [] + + if after: + + def get_pos(from_to: tuple[int, int]) -> int: + return from_to[1] + + else: + + def get_pos(from_to: tuple[int, int]) -> int: + return from_to[0] + + for i, from_to in enumerate(buff.document.selection_ranges()): + positions.append(get_pos(from_to)) + if i == 0: + buff.cursor_position = get_pos(from_to) + + buff.multiple_cursor_positions = positions + + # Go to 'INSERT_MULTIPLE' mode. + event.app.vi_state.input_mode = InputMode.INSERT_MULTIPLE + buff.exit_selection() + + @handle("A", filter=in_block_selection & ~is_read_only) + def _append_after_block(event: E) -> None: + insert_in_block_selection(event, after=True) + + @handle("J", filter=vi_navigation_mode & ~is_read_only) + def _join(event: E) -> None: + """ + Join lines. + """ + for i in range(event.arg): + event.current_buffer.join_next_line() + + @handle("g", "J", filter=vi_navigation_mode & ~is_read_only) + def _join_nospace(event: E) -> None: + """ + Join lines without space. + """ + for i in range(event.arg): + event.current_buffer.join_next_line(separator="") + + @handle("J", filter=vi_selection_mode & ~is_read_only) + def _join_selection(event: E) -> None: + """ + Join selected lines. + """ + event.current_buffer.join_selected_lines() + + @handle("g", "J", filter=vi_selection_mode & ~is_read_only) + def _join_selection_nospace(event: E) -> None: + """ + Join selected lines without space. + """ + event.current_buffer.join_selected_lines(separator="") + + @handle("p", filter=vi_navigation_mode) + def _paste(event: E) -> None: + """ + Paste after + """ + event.current_buffer.paste_clipboard_data( + event.app.clipboard.get_data(), + count=event.arg, + paste_mode=PasteMode.VI_AFTER, + ) + + @handle("P", filter=vi_navigation_mode) + def _paste_before(event: E) -> None: + """ + Paste before + """ + event.current_buffer.paste_clipboard_data( + event.app.clipboard.get_data(), + count=event.arg, + paste_mode=PasteMode.VI_BEFORE, + ) + + @handle('"', Keys.Any, "p", filter=vi_navigation_mode) + def _paste_register(event: E) -> None: + """ + Paste from named register. + """ + c = event.key_sequence[1].data + if c in vi_register_names: + data = event.app.vi_state.named_registers.get(c) + if data: + event.current_buffer.paste_clipboard_data( + data, count=event.arg, paste_mode=PasteMode.VI_AFTER + ) + + @handle('"', Keys.Any, "P", filter=vi_navigation_mode) + def _paste_register_before(event: E) -> None: + """ + Paste (before) from named register. + """ + c = event.key_sequence[1].data + if c in vi_register_names: + data = event.app.vi_state.named_registers.get(c) + if data: + event.current_buffer.paste_clipboard_data( + data, count=event.arg, paste_mode=PasteMode.VI_BEFORE + ) + + @handle("r", filter=vi_navigation_mode) + def _replace(event: E) -> None: + """ + Go to 'replace-single'-mode. + """ + event.app.vi_state.input_mode = InputMode.REPLACE_SINGLE + + @handle("R", filter=vi_navigation_mode) + def _replace_mode(event: E) -> None: + """ + Go to 'replace'-mode. + """ + event.app.vi_state.input_mode = InputMode.REPLACE + + @handle("s", filter=vi_navigation_mode & ~is_read_only) + def _substitute(event: E) -> None: + """ + Substitute with new text + (Delete character(s) and go to insert mode.) + """ + text = event.current_buffer.delete(count=event.arg) + event.app.clipboard.set_text(text) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("u", filter=vi_navigation_mode, save_before=(lambda e: False)) + def _undo(event: E) -> None: + for i in range(event.arg): + event.current_buffer.undo() + + @handle("V", filter=vi_navigation_mode) + def _visual_line(event: E) -> None: + """ + Start lines selection. + """ + event.current_buffer.start_selection(selection_type=SelectionType.LINES) + + @handle("c-v", filter=vi_navigation_mode) + def _visual_block(event: E) -> None: + """ + Enter block selection mode. + """ + event.current_buffer.start_selection(selection_type=SelectionType.BLOCK) + + @handle("V", filter=vi_selection_mode) + def _visual_line2(event: E) -> None: + """ + Exit line selection mode, or go from non line selection mode to line + selection mode. + """ + selection_state = event.current_buffer.selection_state + + if selection_state is not None: + if selection_state.type != SelectionType.LINES: + selection_state.type = SelectionType.LINES + else: + event.current_buffer.exit_selection() + + @handle("v", filter=vi_navigation_mode) + def _visual(event: E) -> None: + """ + Enter character selection mode. + """ + event.current_buffer.start_selection(selection_type=SelectionType.CHARACTERS) + + @handle("v", filter=vi_selection_mode) + def _visual2(event: E) -> None: + """ + Exit character selection mode, or go from non-character-selection mode + to character selection mode. + """ + selection_state = event.current_buffer.selection_state + + if selection_state is not None: + if selection_state.type != SelectionType.CHARACTERS: + selection_state.type = SelectionType.CHARACTERS + else: + event.current_buffer.exit_selection() + + @handle("c-v", filter=vi_selection_mode) + def _visual_block2(event: E) -> None: + """ + Exit block selection mode, or go from non block selection mode to block + selection mode. + """ + selection_state = event.current_buffer.selection_state + + if selection_state is not None: + if selection_state.type != SelectionType.BLOCK: + selection_state.type = SelectionType.BLOCK + else: + event.current_buffer.exit_selection() + + @handle("a", "w", filter=vi_selection_mode) + @handle("a", "W", filter=vi_selection_mode) + def _visual_auto_word(event: E) -> None: + """ + Switch from visual linewise mode to visual characterwise mode. + """ + buffer = event.current_buffer + + if ( + buffer.selection_state + and buffer.selection_state.type == SelectionType.LINES + ): + buffer.selection_state.type = SelectionType.CHARACTERS + + @handle("x", filter=vi_navigation_mode) + def _delete(event: E) -> None: + """ + Delete character. + """ + buff = event.current_buffer + count = min(event.arg, len(buff.document.current_line_after_cursor)) + if count: + text = event.current_buffer.delete(count=count) + event.app.clipboard.set_text(text) + + @handle("X", filter=vi_navigation_mode) + def _delete_before_cursor(event: E) -> None: + buff = event.current_buffer + count = min(event.arg, len(buff.document.current_line_before_cursor)) + if count: + text = event.current_buffer.delete_before_cursor(count=count) + event.app.clipboard.set_text(text) + + @handle("y", "y", filter=vi_navigation_mode) + @handle("Y", filter=vi_navigation_mode) + def _yank_line(event: E) -> None: + """ + Yank the whole line. + """ + text = "\n".join(event.current_buffer.document.lines_from_current[: event.arg]) + event.app.clipboard.set_data(ClipboardData(text, SelectionType.LINES)) + + @handle("+", filter=vi_navigation_mode) + def _next_line(event: E) -> None: + """ + Move to first non whitespace of next line + """ + buffer = event.current_buffer + buffer.cursor_position += buffer.document.get_cursor_down_position( + count=event.arg + ) + buffer.cursor_position += buffer.document.get_start_of_line_position( + after_whitespace=True + ) + + @handle("-", filter=vi_navigation_mode) + def _prev_line(event: E) -> None: + """ + Move to first non whitespace of previous line + """ + buffer = event.current_buffer + buffer.cursor_position += buffer.document.get_cursor_up_position( + count=event.arg + ) + buffer.cursor_position += buffer.document.get_start_of_line_position( + after_whitespace=True + ) + + @handle(">", ">", filter=vi_navigation_mode) + @handle("c-t", filter=vi_insert_mode) + def _indent(event: E) -> None: + """ + Indent lines. + """ + buffer = event.current_buffer + current_row = buffer.document.cursor_position_row + indent(buffer, current_row, current_row + event.arg) + + @handle("<", "<", filter=vi_navigation_mode) + @handle("c-d", filter=vi_insert_mode) + def _unindent(event: E) -> None: + """ + Unindent lines. + """ + current_row = event.current_buffer.document.cursor_position_row + unindent(event.current_buffer, current_row, current_row + event.arg) + + @handle("O", filter=vi_navigation_mode & ~is_read_only) + def _open_above(event: E) -> None: + """ + Open line above and enter insertion mode + """ + event.current_buffer.insert_line_above(copy_margin=not in_paste_mode()) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("o", filter=vi_navigation_mode & ~is_read_only) + def _open_below(event: E) -> None: + """ + Open line below and enter insertion mode + """ + event.current_buffer.insert_line_below(copy_margin=not in_paste_mode()) + event.app.vi_state.input_mode = InputMode.INSERT + + @handle("~", filter=vi_navigation_mode) + def _reverse_case(event: E) -> None: + """ + Reverse case of current character and move cursor forward. + """ + buffer = event.current_buffer + c = buffer.document.current_char + + if c is not None and c != "\n": + buffer.insert_text(c.swapcase(), overwrite=True) + + @handle("g", "u", "u", filter=vi_navigation_mode & ~is_read_only) + def _lowercase_line(event: E) -> None: + """ + Lowercase current line. + """ + buff = event.current_buffer + buff.transform_current_line(lambda s: s.lower()) + + @handle("g", "U", "U", filter=vi_navigation_mode & ~is_read_only) + def _uppercase_line(event: E) -> None: + """ + Uppercase current line. + """ + buff = event.current_buffer + buff.transform_current_line(lambda s: s.upper()) + + @handle("g", "~", "~", filter=vi_navigation_mode & ~is_read_only) + def _swapcase_line(event: E) -> None: + """ + Swap case of the current line. + """ + buff = event.current_buffer + buff.transform_current_line(lambda s: s.swapcase()) + + @handle("#", filter=vi_navigation_mode) + def _prev_occurrence(event: E) -> None: + """ + Go to previous occurrence of this word. + """ + b = event.current_buffer + search_state = event.app.current_search_state + + search_state.text = b.document.get_word_under_cursor() + search_state.direction = SearchDirection.BACKWARD + + b.apply_search(search_state, count=event.arg, include_current_position=False) + + @handle("*", filter=vi_navigation_mode) + def _next_occurrence(event: E) -> None: + """ + Go to next occurrence of this word. + """ + b = event.current_buffer + search_state = event.app.current_search_state + + search_state.text = b.document.get_word_under_cursor() + search_state.direction = SearchDirection.FORWARD + + b.apply_search(search_state, count=event.arg, include_current_position=False) + + @handle("(", filter=vi_navigation_mode) + def _begin_of_sentence(event: E) -> None: + # TODO: go to begin of sentence. + # XXX: should become text_object. + pass + + @handle(")", filter=vi_navigation_mode) + def _end_of_sentence(event: E) -> None: + # TODO: go to end of sentence. + # XXX: should become text_object. + pass + + operator = create_operator_decorator(key_bindings) + text_object = create_text_object_decorator(key_bindings) + + @handle(Keys.Any, filter=vi_waiting_for_text_object_mode) + def _unknown_text_object(event: E) -> None: + """ + Unknown key binding while waiting for a text object. + """ + event.app.output.bell() + + # + # *** Operators *** + # + + def create_delete_and_change_operators( + delete_only: bool, with_register: bool = False + ) -> None: + """ + Delete and change operators. + + :param delete_only: Create an operator that deletes, but doesn't go to insert mode. + :param with_register: Copy the deleted text to this named register instead of the clipboard. + """ + handler_keys: Iterable[str] + if with_register: + handler_keys = ('"', Keys.Any, "cd"[delete_only]) + else: + handler_keys = "cd"[delete_only] + + @operator(*handler_keys, filter=~is_read_only) + def delete_or_change_operator(event: E, text_object: TextObject) -> None: + clipboard_data = None + buff = event.current_buffer + + if text_object: + new_document, clipboard_data = text_object.cut(buff) + buff.document = new_document + + # Set deleted/changed text to clipboard or named register. + if clipboard_data and clipboard_data.text: + if with_register: + reg_name = event.key_sequence[1].data + if reg_name in vi_register_names: + event.app.vi_state.named_registers[reg_name] = clipboard_data + else: + event.app.clipboard.set_data(clipboard_data) + + # Only go back to insert mode in case of 'change'. + if not delete_only: + event.app.vi_state.input_mode = InputMode.INSERT + + create_delete_and_change_operators(False, False) + create_delete_and_change_operators(False, True) + create_delete_and_change_operators(True, False) + create_delete_and_change_operators(True, True) + + def create_transform_handler( + filter: Filter, transform_func: Callable[[str], str], *a: str + ) -> None: + @operator(*a, filter=filter & ~is_read_only) + def _(event: E, text_object: TextObject) -> None: + """ + Apply transformation (uppercase, lowercase, rot13, swap case). + """ + buff = event.current_buffer + start, end = text_object.operator_range(buff.document) + + if start < end: + # Transform. + buff.transform_region( + buff.cursor_position + start, + buff.cursor_position + end, + transform_func, + ) + + # Move cursor + buff.cursor_position += text_object.end or text_object.start + + for k, f, func in vi_transform_functions: + create_transform_handler(f, func, *k) + + @operator("y") + def _yank(event: E, text_object: TextObject) -> None: + """ + Yank operator. (Copy text.) + """ + _, clipboard_data = text_object.cut(event.current_buffer) + if clipboard_data.text: + event.app.clipboard.set_data(clipboard_data) + + @operator('"', Keys.Any, "y") + def _yank_to_register(event: E, text_object: TextObject) -> None: + """ + Yank selection to named register. + """ + c = event.key_sequence[1].data + if c in vi_register_names: + _, clipboard_data = text_object.cut(event.current_buffer) + event.app.vi_state.named_registers[c] = clipboard_data + + @operator(">") + def _indent_text_object(event: E, text_object: TextObject) -> None: + """ + Indent. + """ + buff = event.current_buffer + from_, to = text_object.get_line_numbers(buff) + indent(buff, from_, to + 1, count=event.arg) + + @operator("<") + def _unindent_text_object(event: E, text_object: TextObject) -> None: + """ + Unindent. + """ + buff = event.current_buffer + from_, to = text_object.get_line_numbers(buff) + unindent(buff, from_, to + 1, count=event.arg) + + @operator("g", "q") + def _reshape(event: E, text_object: TextObject) -> None: + """ + Reshape text. + """ + buff = event.current_buffer + from_, to = text_object.get_line_numbers(buff) + reshape_text(buff, from_, to) + + # + # *** Text objects *** + # + + @text_object("b") + def _b(event: E) -> TextObject: + """ + Move one word or token left. + """ + return TextObject( + event.current_buffer.document.find_start_of_previous_word(count=event.arg) + or 0 + ) + + @text_object("B") + def _B(event: E) -> TextObject: + """ + Move one non-blank word left + """ + return TextObject( + event.current_buffer.document.find_start_of_previous_word( + count=event.arg, WORD=True + ) + or 0 + ) + + @text_object("$") + def _dollar(event: E) -> TextObject: + """ + 'c$', 'd$' and '$': Delete/change/move until end of line. + """ + return TextObject(event.current_buffer.document.get_end_of_line_position()) + + @text_object("w") + def _word_forward(event: E) -> TextObject: + """ + 'word' forward. 'cw', 'dw', 'w': Delete/change/move one word. + """ + return TextObject( + event.current_buffer.document.find_next_word_beginning(count=event.arg) + or event.current_buffer.document.get_end_of_document_position() + ) + + @text_object("W") + def _WORD_forward(event: E) -> TextObject: + """ + 'WORD' forward. 'cW', 'dW', 'W': Delete/change/move one WORD. + """ + return TextObject( + event.current_buffer.document.find_next_word_beginning( + count=event.arg, WORD=True + ) + or event.current_buffer.document.get_end_of_document_position() + ) + + @text_object("e") + def _end_of_word(event: E) -> TextObject: + """ + End of 'word': 'ce', 'de', 'e' + """ + end = event.current_buffer.document.find_next_word_ending(count=event.arg) + return TextObject(end - 1 if end else 0, type=TextObjectType.INCLUSIVE) + + @text_object("E") + def _end_of_WORD(event: E) -> TextObject: + """ + End of 'WORD': 'cE', 'dE', 'E' + """ + end = event.current_buffer.document.find_next_word_ending( + count=event.arg, WORD=True + ) + return TextObject(end - 1 if end else 0, type=TextObjectType.INCLUSIVE) + + @text_object("i", "w", no_move_handler=True) + def _inner_word(event: E) -> TextObject: + """ + Inner 'word': ciw and diw + """ + start, end = event.current_buffer.document.find_boundaries_of_current_word() + return TextObject(start, end) + + @text_object("a", "w", no_move_handler=True) + def _a_word(event: E) -> TextObject: + """ + A 'word': caw and daw + """ + start, end = event.current_buffer.document.find_boundaries_of_current_word( + include_trailing_whitespace=True + ) + return TextObject(start, end) + + @text_object("i", "W", no_move_handler=True) + def _inner_WORD(event: E) -> TextObject: + """ + Inner 'WORD': ciW and diW + """ + start, end = event.current_buffer.document.find_boundaries_of_current_word( + WORD=True + ) + return TextObject(start, end) + + @text_object("a", "W", no_move_handler=True) + def _a_WORD(event: E) -> TextObject: + """ + A 'WORD': caw and daw + """ + start, end = event.current_buffer.document.find_boundaries_of_current_word( + WORD=True, include_trailing_whitespace=True + ) + return TextObject(start, end) + + @text_object("a", "p", no_move_handler=True) + def _paragraph(event: E) -> TextObject: + """ + Auto paragraph. + """ + start = event.current_buffer.document.start_of_paragraph() + end = event.current_buffer.document.end_of_paragraph(count=event.arg) + return TextObject(start, end) + + @text_object("^") + def _start_of_line(event: E) -> TextObject: + """'c^', 'd^' and '^': Soft start of line, after whitespace.""" + return TextObject( + event.current_buffer.document.get_start_of_line_position( + after_whitespace=True + ) + ) + + @text_object("0") + def _hard_start_of_line(event: E) -> TextObject: + """ + 'c0', 'd0': Hard start of line, before whitespace. + (The move '0' key is implemented elsewhere, because a '0' could also change the `arg`.) + """ + return TextObject( + event.current_buffer.document.get_start_of_line_position( + after_whitespace=False + ) + ) + + def create_ci_ca_handles( + ci_start: str, ci_end: str, inner: bool, key: str | None = None + ) -> None: + # TODO: 'dat', 'dit', (tags (like xml) + """ + Delete/Change string between this start and stop character. But keep these characters. + This implements all the ci", ci<, ci{, ci(, di", di<, ca", ca<, ... combinations. + """ + + def handler(event: E) -> TextObject: + if ci_start == ci_end: + # Quotes + start = event.current_buffer.document.find_backwards( + ci_start, in_current_line=False + ) + end = event.current_buffer.document.find(ci_end, in_current_line=False) + else: + # Brackets + start = event.current_buffer.document.find_enclosing_bracket_left( + ci_start, ci_end + ) + end = event.current_buffer.document.find_enclosing_bracket_right( + ci_start, ci_end + ) + + if start is not None and end is not None: + offset = 0 if inner else 1 + return TextObject(start + 1 - offset, end + offset) + else: + # Nothing found. + return TextObject(0) + + if key is None: + text_object("ai"[inner], ci_start, no_move_handler=True)(handler) + text_object("ai"[inner], ci_end, no_move_handler=True)(handler) + else: + text_object("ai"[inner], key, no_move_handler=True)(handler) + + for inner in (False, True): + for ci_start, ci_end in [ + ('"', '"'), + ("'", "'"), + ("`", "`"), + ("[", "]"), + ("<", ">"), + ("{", "}"), + ("(", ")"), + ]: + create_ci_ca_handles(ci_start, ci_end, inner) + + create_ci_ca_handles("(", ")", inner, "b") # 'dab', 'dib' + create_ci_ca_handles("{", "}", inner, "B") # 'daB', 'diB' + + @text_object("{") + def _previous_section(event: E) -> TextObject: + """ + Move to previous blank-line separated section. + Implements '{', 'c{', 'd{', 'y{' + """ + index = event.current_buffer.document.start_of_paragraph( + count=event.arg, before=True + ) + return TextObject(index) + + @text_object("}") + def _next_section(event: E) -> TextObject: + """ + Move to next blank-line separated section. + Implements '}', 'c}', 'd}', 'y}' + """ + index = event.current_buffer.document.end_of_paragraph( + count=event.arg, after=True + ) + return TextObject(index) + + @text_object("f", Keys.Any) + def _find_next_occurrence(event: E) -> TextObject: + """ + Go to next occurrence of character. Typing 'fx' will move the + cursor to the next occurrence of character. 'x'. + """ + event.app.vi_state.last_character_find = CharacterFind(event.data, False) + match = event.current_buffer.document.find( + event.data, in_current_line=True, count=event.arg + ) + if match: + return TextObject(match, type=TextObjectType.INCLUSIVE) + else: + return TextObject(0) + + @text_object("F", Keys.Any) + def _find_previous_occurrence(event: E) -> TextObject: + """ + Go to previous occurrence of character. Typing 'Fx' will move the + cursor to the previous occurrence of character. 'x'. + """ + event.app.vi_state.last_character_find = CharacterFind(event.data, True) + return TextObject( + event.current_buffer.document.find_backwards( + event.data, in_current_line=True, count=event.arg + ) + or 0 + ) + + @text_object("t", Keys.Any) + def _t(event: E) -> TextObject: + """ + Move right to the next occurrence of c, then one char backward. + """ + event.app.vi_state.last_character_find = CharacterFind(event.data, False) + match = event.current_buffer.document.find( + event.data, in_current_line=True, count=event.arg + ) + if match: + return TextObject(match - 1, type=TextObjectType.INCLUSIVE) + else: + return TextObject(0) + + @text_object("T", Keys.Any) + def _T(event: E) -> TextObject: + """ + Move left to the previous occurrence of c, then one char forward. + """ + event.app.vi_state.last_character_find = CharacterFind(event.data, True) + match = event.current_buffer.document.find_backwards( + event.data, in_current_line=True, count=event.arg + ) + return TextObject(match + 1 if match else 0) + + def repeat(reverse: bool) -> None: + """ + Create ',' and ';' commands. + """ + + @text_object("," if reverse else ";") + def _(event: E) -> TextObject: + """ + Repeat the last 'f'/'F'/'t'/'T' command. + """ + pos: int | None = 0 + vi_state = event.app.vi_state + + type = TextObjectType.EXCLUSIVE + + if vi_state.last_character_find: + char = vi_state.last_character_find.character + backwards = vi_state.last_character_find.backwards + + if reverse: + backwards = not backwards + + if backwards: + pos = event.current_buffer.document.find_backwards( + char, in_current_line=True, count=event.arg + ) + else: + pos = event.current_buffer.document.find( + char, in_current_line=True, count=event.arg + ) + type = TextObjectType.INCLUSIVE + if pos: + return TextObject(pos, type=type) + else: + return TextObject(0) + + repeat(True) + repeat(False) + + @text_object("h") + @text_object("left") + def _left(event: E) -> TextObject: + """ + Implements 'ch', 'dh', 'h': Cursor left. + """ + return TextObject( + event.current_buffer.document.get_cursor_left_position(count=event.arg) + ) + + @text_object("j", no_move_handler=True, no_selection_handler=True) + # Note: We also need `no_selection_handler`, because we in + # selection mode, we prefer the other 'j' binding that keeps + # `buffer.preferred_column`. + def _down(event: E) -> TextObject: + """ + Implements 'cj', 'dj', 'j', ... Cursor up. + """ + return TextObject( + event.current_buffer.document.get_cursor_down_position(count=event.arg), + type=TextObjectType.LINEWISE, + ) + + @text_object("k", no_move_handler=True, no_selection_handler=True) + def _up(event: E) -> TextObject: + """ + Implements 'ck', 'dk', 'k', ... Cursor up. + """ + return TextObject( + event.current_buffer.document.get_cursor_up_position(count=event.arg), + type=TextObjectType.LINEWISE, + ) + + @text_object("l") + @text_object(" ") + @text_object("right") + def _right(event: E) -> TextObject: + """ + Implements 'cl', 'dl', 'l', 'c ', 'd ', ' '. Cursor right. + """ + return TextObject( + event.current_buffer.document.get_cursor_right_position(count=event.arg) + ) + + @text_object("H") + def _top_of_screen(event: E) -> TextObject: + """ + Moves to the start of the visible region. (Below the scroll offset.) + Implements 'cH', 'dH', 'H'. + """ + w = event.app.layout.current_window + b = event.current_buffer + + if w and w.render_info: + # When we find a Window that has BufferControl showing this window, + # move to the start of the visible area. + pos = ( + b.document.translate_row_col_to_index( + w.render_info.first_visible_line(after_scroll_offset=True), 0 + ) + - b.cursor_position + ) + + else: + # Otherwise, move to the start of the input. + pos = -len(b.document.text_before_cursor) + return TextObject(pos, type=TextObjectType.LINEWISE) + + @text_object("M") + def _middle_of_screen(event: E) -> TextObject: + """ + Moves cursor to the vertical center of the visible region. + Implements 'cM', 'dM', 'M'. + """ + w = event.app.layout.current_window + b = event.current_buffer + + if w and w.render_info: + # When we find a Window that has BufferControl showing this window, + # move to the center of the visible area. + pos = ( + b.document.translate_row_col_to_index( + w.render_info.center_visible_line(), 0 + ) + - b.cursor_position + ) + + else: + # Otherwise, move to the start of the input. + pos = -len(b.document.text_before_cursor) + return TextObject(pos, type=TextObjectType.LINEWISE) + + @text_object("L") + def _end_of_screen(event: E) -> TextObject: + """ + Moves to the end of the visible region. (Above the scroll offset.) + """ + w = event.app.layout.current_window + b = event.current_buffer + + if w and w.render_info: + # When we find a Window that has BufferControl showing this window, + # move to the end of the visible area. + pos = ( + b.document.translate_row_col_to_index( + w.render_info.last_visible_line(before_scroll_offset=True), 0 + ) + - b.cursor_position + ) + + else: + # Otherwise, move to the end of the input. + pos = len(b.document.text_after_cursor) + return TextObject(pos, type=TextObjectType.LINEWISE) + + @text_object("n", no_move_handler=True) + def _search_next(event: E) -> TextObject: + """ + Search next. + """ + buff = event.current_buffer + search_state = event.app.current_search_state + + cursor_position = buff.get_search_position( + search_state, include_current_position=False, count=event.arg + ) + return TextObject(cursor_position - buff.cursor_position) + + @handle("n", filter=vi_navigation_mode) + def _search_next2(event: E) -> None: + """ + Search next in navigation mode. (This goes through the history.) + """ + search_state = event.app.current_search_state + + event.current_buffer.apply_search( + search_state, include_current_position=False, count=event.arg + ) + + @text_object("N", no_move_handler=True) + def _search_previous(event: E) -> TextObject: + """ + Search previous. + """ + buff = event.current_buffer + search_state = event.app.current_search_state + + cursor_position = buff.get_search_position( + ~search_state, include_current_position=False, count=event.arg + ) + return TextObject(cursor_position - buff.cursor_position) + + @handle("N", filter=vi_navigation_mode) + def _search_previous2(event: E) -> None: + """ + Search previous in navigation mode. (This goes through the history.) + """ + search_state = event.app.current_search_state + + event.current_buffer.apply_search( + ~search_state, include_current_position=False, count=event.arg + ) + + @handle("z", "+", filter=vi_navigation_mode | vi_selection_mode) + @handle("z", "t", filter=vi_navigation_mode | vi_selection_mode) + @handle("z", "enter", filter=vi_navigation_mode | vi_selection_mode) + def _scroll_top(event: E) -> None: + """ + Scrolls the window to makes the current line the first line in the visible region. + """ + b = event.current_buffer + event.app.layout.current_window.vertical_scroll = b.document.cursor_position_row + + @handle("z", "-", filter=vi_navigation_mode | vi_selection_mode) + @handle("z", "b", filter=vi_navigation_mode | vi_selection_mode) + def _scroll_bottom(event: E) -> None: + """ + Scrolls the window to makes the current line the last line in the visible region. + """ + # We can safely set the scroll offset to zero; the Window will make + # sure that it scrolls at least enough to make the cursor visible + # again. + event.app.layout.current_window.vertical_scroll = 0 + + @handle("z", "z", filter=vi_navigation_mode | vi_selection_mode) + def _scroll_center(event: E) -> None: + """ + Center Window vertically around cursor. + """ + w = event.app.layout.current_window + b = event.current_buffer + + if w and w.render_info: + info = w.render_info + + # Calculate the offset that we need in order to position the row + # containing the cursor in the center. + scroll_height = info.window_height // 2 + + y = max(0, b.document.cursor_position_row - 1) + height = 0 + while y > 0: + line_height = info.get_height_for_line(y) + + if height + line_height < scroll_height: + height += line_height + y -= 1 + else: + break + + w.vertical_scroll = y + + @text_object("%") + def _goto_corresponding_bracket(event: E) -> TextObject: + """ + Implements 'c%', 'd%', '%, 'y%' (Move to corresponding bracket.) + If an 'arg' has been given, go this this % position in the file. + """ + buffer = event.current_buffer + + if event._arg: + # If 'arg' has been given, the meaning of % is to go to the 'x%' + # row in the file. + if 0 < event.arg <= 100: + absolute_index = buffer.document.translate_row_col_to_index( + int((event.arg * buffer.document.line_count - 1) / 100), 0 + ) + return TextObject( + absolute_index - buffer.document.cursor_position, + type=TextObjectType.LINEWISE, + ) + else: + return TextObject(0) # Do nothing. + + else: + # Move to the corresponding opening/closing bracket (()'s, []'s and {}'s). + match = buffer.document.find_matching_bracket_position() + if match: + return TextObject(match, type=TextObjectType.INCLUSIVE) + else: + return TextObject(0) + + @text_object("|") + def _to_column(event: E) -> TextObject: + """ + Move to the n-th column (you may specify the argument n by typing it on + number keys, for example, 20|). + """ + return TextObject( + event.current_buffer.document.get_column_cursor_position(event.arg - 1) + ) + + @text_object("g", "g") + def _goto_first_line(event: E) -> TextObject: + """ + Go to the start of the very first line. + Implements 'gg', 'cgg', 'ygg' + """ + d = event.current_buffer.document + + if event._arg: + # Move to the given line. + return TextObject( + d.translate_row_col_to_index(event.arg - 1, 0) - d.cursor_position, + type=TextObjectType.LINEWISE, + ) + else: + # Move to the top of the input. + return TextObject( + d.get_start_of_document_position(), type=TextObjectType.LINEWISE + ) + + @text_object("g", "_") + def _goto_last_line(event: E) -> TextObject: + """ + Go to last non-blank of line. + 'g_', 'cg_', 'yg_', etc.. + """ + return TextObject( + event.current_buffer.document.last_non_blank_of_current_line_position(), + type=TextObjectType.INCLUSIVE, + ) + + @text_object("g", "e") + def _ge(event: E) -> TextObject: + """ + Go to last character of previous word. + 'ge', 'cge', 'yge', etc.. + """ + prev_end = event.current_buffer.document.find_previous_word_ending( + count=event.arg + ) + return TextObject( + prev_end - 1 if prev_end is not None else 0, type=TextObjectType.INCLUSIVE + ) + + @text_object("g", "E") + def _gE(event: E) -> TextObject: + """ + Go to last character of previous WORD. + 'gE', 'cgE', 'ygE', etc.. + """ + prev_end = event.current_buffer.document.find_previous_word_ending( + count=event.arg, WORD=True + ) + return TextObject( + prev_end - 1 if prev_end is not None else 0, type=TextObjectType.INCLUSIVE + ) + + @text_object("g", "m") + def _gm(event: E) -> TextObject: + """ + Like g0, but half a screenwidth to the right. (Or as much as possible.) + """ + w = event.app.layout.current_window + buff = event.current_buffer + + if w and w.render_info: + width = w.render_info.window_width + start = buff.document.get_start_of_line_position(after_whitespace=False) + start += int(min(width / 2, len(buff.document.current_line))) + + return TextObject(start, type=TextObjectType.INCLUSIVE) + return TextObject(0) + + @text_object("G") + def _last_line(event: E) -> TextObject: + """ + Go to the end of the document. (If no arg has been given.) + """ + buf = event.current_buffer + return TextObject( + buf.document.translate_row_col_to_index(buf.document.line_count - 1, 0) + - buf.cursor_position, + type=TextObjectType.LINEWISE, + ) + + # + # *** Other *** + # + + @handle("G", filter=has_arg) + def _to_nth_history_line(event: E) -> None: + """ + If an argument is given, move to this line in the history. (for + example, 15G) + """ + event.current_buffer.go_to_history(event.arg - 1) + + for n in "123456789": + + @handle( + n, + filter=vi_navigation_mode + | vi_selection_mode + | vi_waiting_for_text_object_mode, + ) + def _arg(event: E) -> None: + """ + Always handle numerics in navigation mode as arg. + """ + event.append_to_arg_count(event.data) + + @handle( + "0", + filter=( + vi_navigation_mode | vi_selection_mode | vi_waiting_for_text_object_mode + ) + & has_arg, + ) + def _0_arg(event: E) -> None: + """ + Zero when an argument was already give. + """ + event.append_to_arg_count(event.data) + + @handle(Keys.Any, filter=vi_replace_mode) + def _insert_text(event: E) -> None: + """ + Insert data at cursor position. + """ + event.current_buffer.insert_text(event.data, overwrite=True) + + @handle(Keys.Any, filter=vi_replace_single_mode) + def _replace_single(event: E) -> None: + """ + Replace single character at cursor position. + """ + event.current_buffer.insert_text(event.data, overwrite=True) + event.current_buffer.cursor_position -= 1 + event.app.vi_state.input_mode = InputMode.NAVIGATION + + @handle( + Keys.Any, + filter=vi_insert_multiple_mode, + save_before=(lambda e: not e.is_repeat), + ) + def _insert_text_multiple_cursors(event: E) -> None: + """ + Insert data at multiple cursor positions at once. + (Usually a result of pressing 'I' or 'A' in block-selection mode.) + """ + buff = event.current_buffer + original_text = buff.text + + # Construct new text. + text = [] + p = 0 + + for p2 in buff.multiple_cursor_positions: + text.append(original_text[p:p2]) + text.append(event.data) + p = p2 + + text.append(original_text[p:]) + + # Shift all cursor positions. + new_cursor_positions = [ + pos + i + 1 for i, pos in enumerate(buff.multiple_cursor_positions) + ] + + # Set result. + buff.text = "".join(text) + buff.multiple_cursor_positions = new_cursor_positions + buff.cursor_position += 1 + + @handle("backspace", filter=vi_insert_multiple_mode) + def _delete_before_multiple_cursors(event: E) -> None: + """ + Backspace, using multiple cursors. + """ + buff = event.current_buffer + original_text = buff.text + + # Construct new text. + deleted_something = False + text = [] + p = 0 + + for p2 in buff.multiple_cursor_positions: + if p2 > 0 and original_text[p2 - 1] != "\n": # Don't delete across lines. + text.append(original_text[p : p2 - 1]) + deleted_something = True + else: + text.append(original_text[p:p2]) + p = p2 + + text.append(original_text[p:]) + + if deleted_something: + # Shift all cursor positions. + lengths = [len(part) for part in text[:-1]] + new_cursor_positions = list(accumulate(lengths)) + + # Set result. + buff.text = "".join(text) + buff.multiple_cursor_positions = new_cursor_positions + buff.cursor_position -= 1 + else: + event.app.output.bell() + + @handle("delete", filter=vi_insert_multiple_mode) + def _delete_after_multiple_cursors(event: E) -> None: + """ + Delete, using multiple cursors. + """ + buff = event.current_buffer + original_text = buff.text + + # Construct new text. + deleted_something = False + text = [] + new_cursor_positions = [] + p = 0 + + for p2 in buff.multiple_cursor_positions: + text.append(original_text[p:p2]) + if p2 >= len(original_text) or original_text[p2] == "\n": + # Don't delete across lines. + p = p2 + else: + p = p2 + 1 + deleted_something = True + + text.append(original_text[p:]) + + if deleted_something: + # Shift all cursor positions. + lengths = [len(part) for part in text[:-1]] + new_cursor_positions = list(accumulate(lengths)) + + # Set result. + buff.text = "".join(text) + buff.multiple_cursor_positions = new_cursor_positions + else: + event.app.output.bell() + + @handle("left", filter=vi_insert_multiple_mode) + def _left_multiple(event: E) -> None: + """ + Move all cursors to the left. + (But keep all cursors on the same line.) + """ + buff = event.current_buffer + new_positions = [] + + for p in buff.multiple_cursor_positions: + if buff.document.translate_index_to_position(p)[1] > 0: + p -= 1 + new_positions.append(p) + + buff.multiple_cursor_positions = new_positions + + if buff.document.cursor_position_col > 0: + buff.cursor_position -= 1 + + @handle("right", filter=vi_insert_multiple_mode) + def _right_multiple(event: E) -> None: + """ + Move all cursors to the right. + (But keep all cursors on the same line.) + """ + buff = event.current_buffer + new_positions = [] + + for p in buff.multiple_cursor_positions: + row, column = buff.document.translate_index_to_position(p) + if column < len(buff.document.lines[row]): + p += 1 + new_positions.append(p) + + buff.multiple_cursor_positions = new_positions + + if not buff.document.is_cursor_at_the_end_of_line: + buff.cursor_position += 1 + + @handle("up", filter=vi_insert_multiple_mode) + @handle("down", filter=vi_insert_multiple_mode) + def _updown_multiple(event: E) -> None: + """ + Ignore all up/down key presses when in multiple cursor mode. + """ + + @handle("c-x", "c-l", filter=vi_insert_mode) + def _complete_line(event: E) -> None: + """ + Pressing the ControlX - ControlL sequence in Vi mode does line + completion based on the other lines in the document and the history. + """ + event.current_buffer.start_history_lines_completion() + + @handle("c-x", "c-f", filter=vi_insert_mode) + def _complete_filename(event: E) -> None: + """ + Complete file names. + """ + # TODO + pass + + @handle("c-k", filter=vi_insert_mode | vi_replace_mode) + def _digraph(event: E) -> None: + """ + Go into digraph mode. + """ + event.app.vi_state.waiting_for_digraph = True + + @handle(Keys.Any, filter=vi_digraph_mode & ~digraph_symbol_1_given) + def _digraph1(event: E) -> None: + """ + First digraph symbol. + """ + event.app.vi_state.digraph_symbol1 = event.data + + @handle(Keys.Any, filter=vi_digraph_mode & digraph_symbol_1_given) + def _create_digraph(event: E) -> None: + """ + Insert digraph. + """ + try: + # Lookup. + code: tuple[str, str] = ( + event.app.vi_state.digraph_symbol1 or "", + event.data, + ) + if code not in DIGRAPHS: + code = code[::-1] # Try reversing. + symbol = DIGRAPHS[code] + except KeyError: + # Unknown digraph. + event.app.output.bell() + else: + # Insert digraph. + overwrite = event.app.vi_state.input_mode == InputMode.REPLACE + event.current_buffer.insert_text(chr(symbol), overwrite=overwrite) + event.app.vi_state.waiting_for_digraph = False + finally: + event.app.vi_state.waiting_for_digraph = False + event.app.vi_state.digraph_symbol1 = None + + @handle("c-o", filter=vi_insert_mode | vi_replace_mode) + def _quick_normal_mode(event: E) -> None: + """ + Go into normal mode for one single action. + """ + event.app.vi_state.temporary_navigation_mode = True + + @handle("q", Keys.Any, filter=vi_navigation_mode & ~vi_recording_macro) + def _start_macro(event: E) -> None: + """ + Start recording macro. + """ + c = event.key_sequence[1].data + if c in vi_register_names: + vi_state = event.app.vi_state + + vi_state.recording_register = c + vi_state.current_recording = "" + + @handle("q", filter=vi_navigation_mode & vi_recording_macro) + def _stop_macro(event: E) -> None: + """ + Stop recording macro. + """ + vi_state = event.app.vi_state + + # Store and stop recording. + if vi_state.recording_register: + vi_state.named_registers[vi_state.recording_register] = ClipboardData( + vi_state.current_recording + ) + vi_state.recording_register = None + vi_state.current_recording = "" + + @handle("@", Keys.Any, filter=vi_navigation_mode, record_in_macro=False) + def _execute_macro(event: E) -> None: + """ + Execute macro. + + Notice that we pass `record_in_macro=False`. This ensures that the `@x` + keys don't appear in the recording itself. This function inserts the + body of the called macro back into the KeyProcessor, so these keys will + be added later on to the macro of their handlers have + `record_in_macro=True`. + """ + # Retrieve macro. + c = event.key_sequence[1].data + try: + macro = event.app.vi_state.named_registers[c] + except KeyError: + return + + # Expand macro (which is a string in the register), in individual keys. + # Use vt100 parser for this. + keys: list[KeyPress] = [] + + parser = Vt100Parser(keys.append) + parser.feed(macro.text) + parser.flush() + + # Now feed keys back to the input processor. + for _ in range(event.arg): + event.app.key_processor.feed_multiple(keys, first=True) + + return ConditionalKeyBindings(key_bindings, vi_mode) + + +def load_vi_search_bindings() -> KeyBindingsBase: + key_bindings = KeyBindings() + handle = key_bindings.add + from . import search + + # Vi-style forward search. + handle( + "/", + filter=(vi_navigation_mode | vi_selection_mode) & ~vi_search_direction_reversed, + )(search.start_forward_incremental_search) + handle( + "?", + filter=(vi_navigation_mode | vi_selection_mode) & vi_search_direction_reversed, + )(search.start_forward_incremental_search) + handle("c-s")(search.start_forward_incremental_search) + + # Vi-style backward search. + handle( + "?", + filter=(vi_navigation_mode | vi_selection_mode) & ~vi_search_direction_reversed, + )(search.start_reverse_incremental_search) + handle( + "/", + filter=(vi_navigation_mode | vi_selection_mode) & vi_search_direction_reversed, + )(search.start_reverse_incremental_search) + handle("c-r")(search.start_reverse_incremental_search) + + # Apply the search. (At the / or ? prompt.) + handle("enter", filter=is_searching)(search.accept_search) + + handle("c-r", filter=is_searching)(search.reverse_incremental_search) + handle("c-s", filter=is_searching)(search.forward_incremental_search) + + handle("c-c")(search.abort_search) + handle("c-g")(search.abort_search) + handle("backspace", filter=search_buffer_is_empty)(search.abort_search) + + # Handle escape. This should accept the search, just like readline. + # `abort_search` would be a meaningful alternative. + handle("escape")(search.accept_search) + + return ConditionalKeyBindings(key_bindings, vi_mode) diff --git a/lib/prompt_toolkit/key_binding/defaults.py b/lib/prompt_toolkit/key_binding/defaults.py new file mode 100644 index 0000000..6c26571 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/defaults.py @@ -0,0 +1,63 @@ +""" +Default key bindings.:: + + key_bindings = load_key_bindings() + app = Application(key_bindings=key_bindings) +""" + +from __future__ import annotations + +from prompt_toolkit.filters import buffer_has_focus +from prompt_toolkit.key_binding.bindings.basic import load_basic_bindings +from prompt_toolkit.key_binding.bindings.cpr import load_cpr_bindings +from prompt_toolkit.key_binding.bindings.emacs import ( + load_emacs_bindings, + load_emacs_search_bindings, + load_emacs_shift_selection_bindings, +) +from prompt_toolkit.key_binding.bindings.mouse import load_mouse_bindings +from prompt_toolkit.key_binding.bindings.vi import ( + load_vi_bindings, + load_vi_search_bindings, +) +from prompt_toolkit.key_binding.key_bindings import ( + ConditionalKeyBindings, + KeyBindingsBase, + merge_key_bindings, +) + +__all__ = [ + "load_key_bindings", +] + + +def load_key_bindings() -> KeyBindingsBase: + """ + Create a KeyBindings object that contains the default key bindings. + """ + all_bindings = merge_key_bindings( + [ + # Load basic bindings. + load_basic_bindings(), + # Load emacs bindings. + load_emacs_bindings(), + load_emacs_search_bindings(), + load_emacs_shift_selection_bindings(), + # Load Vi bindings. + load_vi_bindings(), + load_vi_search_bindings(), + ] + ) + + return merge_key_bindings( + [ + # Make sure that the above key bindings are only active if the + # currently focused control is a `BufferControl`. For other controls, we + # don't want these key bindings to intervene. (This would break "ptterm" + # for instance, which handles 'Keys.Any' in the user control itself.) + ConditionalKeyBindings(all_bindings, buffer_has_focus), + # Active, even when no buffer has been focused. + load_mouse_bindings(), + load_cpr_bindings(), + ] + ) diff --git a/lib/prompt_toolkit/key_binding/digraphs.py b/lib/prompt_toolkit/key_binding/digraphs.py new file mode 100644 index 0000000..f0152dc --- /dev/null +++ b/lib/prompt_toolkit/key_binding/digraphs.py @@ -0,0 +1,1378 @@ +""" +Vi Digraphs. +This is a list of special characters that can be inserted in Vi insert mode by +pressing Control-K followed by to normal characters. + +Taken from Neovim and translated to Python: +https://raw.githubusercontent.com/neovim/neovim/master/src/nvim/digraph.c +""" + +from __future__ import annotations + +__all__ = [ + "DIGRAPHS", +] + +# digraphs for Unicode from RFC1345 +# (also work for ISO-8859-1 aka latin1) +DIGRAPHS: dict[tuple[str, str], int] = { + ("N", "U"): 0x00, + ("S", "H"): 0x01, + ("S", "X"): 0x02, + ("E", "X"): 0x03, + ("E", "T"): 0x04, + ("E", "Q"): 0x05, + ("A", "K"): 0x06, + ("B", "L"): 0x07, + ("B", "S"): 0x08, + ("H", "T"): 0x09, + ("L", "F"): 0x0A, + ("V", "T"): 0x0B, + ("F", "F"): 0x0C, + ("C", "R"): 0x0D, + ("S", "O"): 0x0E, + ("S", "I"): 0x0F, + ("D", "L"): 0x10, + ("D", "1"): 0x11, + ("D", "2"): 0x12, + ("D", "3"): 0x13, + ("D", "4"): 0x14, + ("N", "K"): 0x15, + ("S", "Y"): 0x16, + ("E", "B"): 0x17, + ("C", "N"): 0x18, + ("E", "M"): 0x19, + ("S", "B"): 0x1A, + ("E", "C"): 0x1B, + ("F", "S"): 0x1C, + ("G", "S"): 0x1D, + ("R", "S"): 0x1E, + ("U", "S"): 0x1F, + ("S", "P"): 0x20, + ("N", "b"): 0x23, + ("D", "O"): 0x24, + ("A", "t"): 0x40, + ("<", "("): 0x5B, + ("/", "/"): 0x5C, + (")", ">"): 0x5D, + ("'", ">"): 0x5E, + ("'", "!"): 0x60, + ("(", "!"): 0x7B, + ("!", "!"): 0x7C, + ("!", ")"): 0x7D, + ("'", "?"): 0x7E, + ("D", "T"): 0x7F, + ("P", "A"): 0x80, + ("H", "O"): 0x81, + ("B", "H"): 0x82, + ("N", "H"): 0x83, + ("I", "N"): 0x84, + ("N", "L"): 0x85, + ("S", "A"): 0x86, + ("E", "S"): 0x87, + ("H", "S"): 0x88, + ("H", "J"): 0x89, + ("V", "S"): 0x8A, + ("P", "D"): 0x8B, + ("P", "U"): 0x8C, + ("R", "I"): 0x8D, + ("S", "2"): 0x8E, + ("S", "3"): 0x8F, + ("D", "C"): 0x90, + ("P", "1"): 0x91, + ("P", "2"): 0x92, + ("T", "S"): 0x93, + ("C", "C"): 0x94, + ("M", "W"): 0x95, + ("S", "G"): 0x96, + ("E", "G"): 0x97, + ("S", "S"): 0x98, + ("G", "C"): 0x99, + ("S", "C"): 0x9A, + ("C", "I"): 0x9B, + ("S", "T"): 0x9C, + ("O", "C"): 0x9D, + ("P", "M"): 0x9E, + ("A", "C"): 0x9F, + ("N", "S"): 0xA0, + ("!", "I"): 0xA1, + ("C", "t"): 0xA2, + ("P", "d"): 0xA3, + ("C", "u"): 0xA4, + ("Y", "e"): 0xA5, + ("B", "B"): 0xA6, + ("S", "E"): 0xA7, + ("'", ":"): 0xA8, + ("C", "o"): 0xA9, + ("-", "a"): 0xAA, + ("<", "<"): 0xAB, + ("N", "O"): 0xAC, + ("-", "-"): 0xAD, + ("R", "g"): 0xAE, + ("'", "m"): 0xAF, + ("D", "G"): 0xB0, + ("+", "-"): 0xB1, + ("2", "S"): 0xB2, + ("3", "S"): 0xB3, + ("'", "'"): 0xB4, + ("M", "y"): 0xB5, + ("P", "I"): 0xB6, + (".", "M"): 0xB7, + ("'", ","): 0xB8, + ("1", "S"): 0xB9, + ("-", "o"): 0xBA, + (">", ">"): 0xBB, + ("1", "4"): 0xBC, + ("1", "2"): 0xBD, + ("3", "4"): 0xBE, + ("?", "I"): 0xBF, + ("A", "!"): 0xC0, + ("A", "'"): 0xC1, + ("A", ">"): 0xC2, + ("A", "?"): 0xC3, + ("A", ":"): 0xC4, + ("A", "A"): 0xC5, + ("A", "E"): 0xC6, + ("C", ","): 0xC7, + ("E", "!"): 0xC8, + ("E", "'"): 0xC9, + ("E", ">"): 0xCA, + ("E", ":"): 0xCB, + ("I", "!"): 0xCC, + ("I", "'"): 0xCD, + ("I", ">"): 0xCE, + ("I", ":"): 0xCF, + ("D", "-"): 0xD0, + ("N", "?"): 0xD1, + ("O", "!"): 0xD2, + ("O", "'"): 0xD3, + ("O", ">"): 0xD4, + ("O", "?"): 0xD5, + ("O", ":"): 0xD6, + ("*", "X"): 0xD7, + ("O", "/"): 0xD8, + ("U", "!"): 0xD9, + ("U", "'"): 0xDA, + ("U", ">"): 0xDB, + ("U", ":"): 0xDC, + ("Y", "'"): 0xDD, + ("T", "H"): 0xDE, + ("s", "s"): 0xDF, + ("a", "!"): 0xE0, + ("a", "'"): 0xE1, + ("a", ">"): 0xE2, + ("a", "?"): 0xE3, + ("a", ":"): 0xE4, + ("a", "a"): 0xE5, + ("a", "e"): 0xE6, + ("c", ","): 0xE7, + ("e", "!"): 0xE8, + ("e", "'"): 0xE9, + ("e", ">"): 0xEA, + ("e", ":"): 0xEB, + ("i", "!"): 0xEC, + ("i", "'"): 0xED, + ("i", ">"): 0xEE, + ("i", ":"): 0xEF, + ("d", "-"): 0xF0, + ("n", "?"): 0xF1, + ("o", "!"): 0xF2, + ("o", "'"): 0xF3, + ("o", ">"): 0xF4, + ("o", "?"): 0xF5, + ("o", ":"): 0xF6, + ("-", ":"): 0xF7, + ("o", "/"): 0xF8, + ("u", "!"): 0xF9, + ("u", "'"): 0xFA, + ("u", ">"): 0xFB, + ("u", ":"): 0xFC, + ("y", "'"): 0xFD, + ("t", "h"): 0xFE, + ("y", ":"): 0xFF, + ("A", "-"): 0x0100, + ("a", "-"): 0x0101, + ("A", "("): 0x0102, + ("a", "("): 0x0103, + ("A", ";"): 0x0104, + ("a", ";"): 0x0105, + ("C", "'"): 0x0106, + ("c", "'"): 0x0107, + ("C", ">"): 0x0108, + ("c", ">"): 0x0109, + ("C", "."): 0x010A, + ("c", "."): 0x010B, + ("C", "<"): 0x010C, + ("c", "<"): 0x010D, + ("D", "<"): 0x010E, + ("d", "<"): 0x010F, + ("D", "/"): 0x0110, + ("d", "/"): 0x0111, + ("E", "-"): 0x0112, + ("e", "-"): 0x0113, + ("E", "("): 0x0114, + ("e", "("): 0x0115, + ("E", "."): 0x0116, + ("e", "."): 0x0117, + ("E", ";"): 0x0118, + ("e", ";"): 0x0119, + ("E", "<"): 0x011A, + ("e", "<"): 0x011B, + ("G", ">"): 0x011C, + ("g", ">"): 0x011D, + ("G", "("): 0x011E, + ("g", "("): 0x011F, + ("G", "."): 0x0120, + ("g", "."): 0x0121, + ("G", ","): 0x0122, + ("g", ","): 0x0123, + ("H", ">"): 0x0124, + ("h", ">"): 0x0125, + ("H", "/"): 0x0126, + ("h", "/"): 0x0127, + ("I", "?"): 0x0128, + ("i", "?"): 0x0129, + ("I", "-"): 0x012A, + ("i", "-"): 0x012B, + ("I", "("): 0x012C, + ("i", "("): 0x012D, + ("I", ";"): 0x012E, + ("i", ";"): 0x012F, + ("I", "."): 0x0130, + ("i", "."): 0x0131, + ("I", "J"): 0x0132, + ("i", "j"): 0x0133, + ("J", ">"): 0x0134, + ("j", ">"): 0x0135, + ("K", ","): 0x0136, + ("k", ","): 0x0137, + ("k", "k"): 0x0138, + ("L", "'"): 0x0139, + ("l", "'"): 0x013A, + ("L", ","): 0x013B, + ("l", ","): 0x013C, + ("L", "<"): 0x013D, + ("l", "<"): 0x013E, + ("L", "."): 0x013F, + ("l", "."): 0x0140, + ("L", "/"): 0x0141, + ("l", "/"): 0x0142, + ("N", "'"): 0x0143, + ("n", "'"): 0x0144, + ("N", ","): 0x0145, + ("n", ","): 0x0146, + ("N", "<"): 0x0147, + ("n", "<"): 0x0148, + ("'", "n"): 0x0149, + ("N", "G"): 0x014A, + ("n", "g"): 0x014B, + ("O", "-"): 0x014C, + ("o", "-"): 0x014D, + ("O", "("): 0x014E, + ("o", "("): 0x014F, + ("O", '"'): 0x0150, + ("o", '"'): 0x0151, + ("O", "E"): 0x0152, + ("o", "e"): 0x0153, + ("R", "'"): 0x0154, + ("r", "'"): 0x0155, + ("R", ","): 0x0156, + ("r", ","): 0x0157, + ("R", "<"): 0x0158, + ("r", "<"): 0x0159, + ("S", "'"): 0x015A, + ("s", "'"): 0x015B, + ("S", ">"): 0x015C, + ("s", ">"): 0x015D, + ("S", ","): 0x015E, + ("s", ","): 0x015F, + ("S", "<"): 0x0160, + ("s", "<"): 0x0161, + ("T", ","): 0x0162, + ("t", ","): 0x0163, + ("T", "<"): 0x0164, + ("t", "<"): 0x0165, + ("T", "/"): 0x0166, + ("t", "/"): 0x0167, + ("U", "?"): 0x0168, + ("u", "?"): 0x0169, + ("U", "-"): 0x016A, + ("u", "-"): 0x016B, + ("U", "("): 0x016C, + ("u", "("): 0x016D, + ("U", "0"): 0x016E, + ("u", "0"): 0x016F, + ("U", '"'): 0x0170, + ("u", '"'): 0x0171, + ("U", ";"): 0x0172, + ("u", ";"): 0x0173, + ("W", ">"): 0x0174, + ("w", ">"): 0x0175, + ("Y", ">"): 0x0176, + ("y", ">"): 0x0177, + ("Y", ":"): 0x0178, + ("Z", "'"): 0x0179, + ("z", "'"): 0x017A, + ("Z", "."): 0x017B, + ("z", "."): 0x017C, + ("Z", "<"): 0x017D, + ("z", "<"): 0x017E, + ("O", "9"): 0x01A0, + ("o", "9"): 0x01A1, + ("O", "I"): 0x01A2, + ("o", "i"): 0x01A3, + ("y", "r"): 0x01A6, + ("U", "9"): 0x01AF, + ("u", "9"): 0x01B0, + ("Z", "/"): 0x01B5, + ("z", "/"): 0x01B6, + ("E", "D"): 0x01B7, + ("A", "<"): 0x01CD, + ("a", "<"): 0x01CE, + ("I", "<"): 0x01CF, + ("i", "<"): 0x01D0, + ("O", "<"): 0x01D1, + ("o", "<"): 0x01D2, + ("U", "<"): 0x01D3, + ("u", "<"): 0x01D4, + ("A", "1"): 0x01DE, + ("a", "1"): 0x01DF, + ("A", "7"): 0x01E0, + ("a", "7"): 0x01E1, + ("A", "3"): 0x01E2, + ("a", "3"): 0x01E3, + ("G", "/"): 0x01E4, + ("g", "/"): 0x01E5, + ("G", "<"): 0x01E6, + ("g", "<"): 0x01E7, + ("K", "<"): 0x01E8, + ("k", "<"): 0x01E9, + ("O", ";"): 0x01EA, + ("o", ";"): 0x01EB, + ("O", "1"): 0x01EC, + ("o", "1"): 0x01ED, + ("E", "Z"): 0x01EE, + ("e", "z"): 0x01EF, + ("j", "<"): 0x01F0, + ("G", "'"): 0x01F4, + ("g", "'"): 0x01F5, + (";", "S"): 0x02BF, + ("'", "<"): 0x02C7, + ("'", "("): 0x02D8, + ("'", "."): 0x02D9, + ("'", "0"): 0x02DA, + ("'", ";"): 0x02DB, + ("'", '"'): 0x02DD, + ("A", "%"): 0x0386, + ("E", "%"): 0x0388, + ("Y", "%"): 0x0389, + ("I", "%"): 0x038A, + ("O", "%"): 0x038C, + ("U", "%"): 0x038E, + ("W", "%"): 0x038F, + ("i", "3"): 0x0390, + ("A", "*"): 0x0391, + ("B", "*"): 0x0392, + ("G", "*"): 0x0393, + ("D", "*"): 0x0394, + ("E", "*"): 0x0395, + ("Z", "*"): 0x0396, + ("Y", "*"): 0x0397, + ("H", "*"): 0x0398, + ("I", "*"): 0x0399, + ("K", "*"): 0x039A, + ("L", "*"): 0x039B, + ("M", "*"): 0x039C, + ("N", "*"): 0x039D, + ("C", "*"): 0x039E, + ("O", "*"): 0x039F, + ("P", "*"): 0x03A0, + ("R", "*"): 0x03A1, + ("S", "*"): 0x03A3, + ("T", "*"): 0x03A4, + ("U", "*"): 0x03A5, + ("F", "*"): 0x03A6, + ("X", "*"): 0x03A7, + ("Q", "*"): 0x03A8, + ("W", "*"): 0x03A9, + ("J", "*"): 0x03AA, + ("V", "*"): 0x03AB, + ("a", "%"): 0x03AC, + ("e", "%"): 0x03AD, + ("y", "%"): 0x03AE, + ("i", "%"): 0x03AF, + ("u", "3"): 0x03B0, + ("a", "*"): 0x03B1, + ("b", "*"): 0x03B2, + ("g", "*"): 0x03B3, + ("d", "*"): 0x03B4, + ("e", "*"): 0x03B5, + ("z", "*"): 0x03B6, + ("y", "*"): 0x03B7, + ("h", "*"): 0x03B8, + ("i", "*"): 0x03B9, + ("k", "*"): 0x03BA, + ("l", "*"): 0x03BB, + ("m", "*"): 0x03BC, + ("n", "*"): 0x03BD, + ("c", "*"): 0x03BE, + ("o", "*"): 0x03BF, + ("p", "*"): 0x03C0, + ("r", "*"): 0x03C1, + ("*", "s"): 0x03C2, + ("s", "*"): 0x03C3, + ("t", "*"): 0x03C4, + ("u", "*"): 0x03C5, + ("f", "*"): 0x03C6, + ("x", "*"): 0x03C7, + ("q", "*"): 0x03C8, + ("w", "*"): 0x03C9, + ("j", "*"): 0x03CA, + ("v", "*"): 0x03CB, + ("o", "%"): 0x03CC, + ("u", "%"): 0x03CD, + ("w", "%"): 0x03CE, + ("'", "G"): 0x03D8, + (",", "G"): 0x03D9, + ("T", "3"): 0x03DA, + ("t", "3"): 0x03DB, + ("M", "3"): 0x03DC, + ("m", "3"): 0x03DD, + ("K", "3"): 0x03DE, + ("k", "3"): 0x03DF, + ("P", "3"): 0x03E0, + ("p", "3"): 0x03E1, + ("'", "%"): 0x03F4, + ("j", "3"): 0x03F5, + ("I", "O"): 0x0401, + ("D", "%"): 0x0402, + ("G", "%"): 0x0403, + ("I", "E"): 0x0404, + ("D", "S"): 0x0405, + ("I", "I"): 0x0406, + ("Y", "I"): 0x0407, + ("J", "%"): 0x0408, + ("L", "J"): 0x0409, + ("N", "J"): 0x040A, + ("T", "s"): 0x040B, + ("K", "J"): 0x040C, + ("V", "%"): 0x040E, + ("D", "Z"): 0x040F, + ("A", "="): 0x0410, + ("B", "="): 0x0411, + ("V", "="): 0x0412, + ("G", "="): 0x0413, + ("D", "="): 0x0414, + ("E", "="): 0x0415, + ("Z", "%"): 0x0416, + ("Z", "="): 0x0417, + ("I", "="): 0x0418, + ("J", "="): 0x0419, + ("K", "="): 0x041A, + ("L", "="): 0x041B, + ("M", "="): 0x041C, + ("N", "="): 0x041D, + ("O", "="): 0x041E, + ("P", "="): 0x041F, + ("R", "="): 0x0420, + ("S", "="): 0x0421, + ("T", "="): 0x0422, + ("U", "="): 0x0423, + ("F", "="): 0x0424, + ("H", "="): 0x0425, + ("C", "="): 0x0426, + ("C", "%"): 0x0427, + ("S", "%"): 0x0428, + ("S", "c"): 0x0429, + ("=", '"'): 0x042A, + ("Y", "="): 0x042B, + ("%", '"'): 0x042C, + ("J", "E"): 0x042D, + ("J", "U"): 0x042E, + ("J", "A"): 0x042F, + ("a", "="): 0x0430, + ("b", "="): 0x0431, + ("v", "="): 0x0432, + ("g", "="): 0x0433, + ("d", "="): 0x0434, + ("e", "="): 0x0435, + ("z", "%"): 0x0436, + ("z", "="): 0x0437, + ("i", "="): 0x0438, + ("j", "="): 0x0439, + ("k", "="): 0x043A, + ("l", "="): 0x043B, + ("m", "="): 0x043C, + ("n", "="): 0x043D, + ("o", "="): 0x043E, + ("p", "="): 0x043F, + ("r", "="): 0x0440, + ("s", "="): 0x0441, + ("t", "="): 0x0442, + ("u", "="): 0x0443, + ("f", "="): 0x0444, + ("h", "="): 0x0445, + ("c", "="): 0x0446, + ("c", "%"): 0x0447, + ("s", "%"): 0x0448, + ("s", "c"): 0x0449, + ("=", "'"): 0x044A, + ("y", "="): 0x044B, + ("%", "'"): 0x044C, + ("j", "e"): 0x044D, + ("j", "u"): 0x044E, + ("j", "a"): 0x044F, + ("i", "o"): 0x0451, + ("d", "%"): 0x0452, + ("g", "%"): 0x0453, + ("i", "e"): 0x0454, + ("d", "s"): 0x0455, + ("i", "i"): 0x0456, + ("y", "i"): 0x0457, + ("j", "%"): 0x0458, + ("l", "j"): 0x0459, + ("n", "j"): 0x045A, + ("t", "s"): 0x045B, + ("k", "j"): 0x045C, + ("v", "%"): 0x045E, + ("d", "z"): 0x045F, + ("Y", "3"): 0x0462, + ("y", "3"): 0x0463, + ("O", "3"): 0x046A, + ("o", "3"): 0x046B, + ("F", "3"): 0x0472, + ("f", "3"): 0x0473, + ("V", "3"): 0x0474, + ("v", "3"): 0x0475, + ("C", "3"): 0x0480, + ("c", "3"): 0x0481, + ("G", "3"): 0x0490, + ("g", "3"): 0x0491, + ("A", "+"): 0x05D0, + ("B", "+"): 0x05D1, + ("G", "+"): 0x05D2, + ("D", "+"): 0x05D3, + ("H", "+"): 0x05D4, + ("W", "+"): 0x05D5, + ("Z", "+"): 0x05D6, + ("X", "+"): 0x05D7, + ("T", "j"): 0x05D8, + ("J", "+"): 0x05D9, + ("K", "%"): 0x05DA, + ("K", "+"): 0x05DB, + ("L", "+"): 0x05DC, + ("M", "%"): 0x05DD, + ("M", "+"): 0x05DE, + ("N", "%"): 0x05DF, + ("N", "+"): 0x05E0, + ("S", "+"): 0x05E1, + ("E", "+"): 0x05E2, + ("P", "%"): 0x05E3, + ("P", "+"): 0x05E4, + ("Z", "j"): 0x05E5, + ("Z", "J"): 0x05E6, + ("Q", "+"): 0x05E7, + ("R", "+"): 0x05E8, + ("S", "h"): 0x05E9, + ("T", "+"): 0x05EA, + (",", "+"): 0x060C, + (";", "+"): 0x061B, + ("?", "+"): 0x061F, + ("H", "'"): 0x0621, + ("a", "M"): 0x0622, + ("a", "H"): 0x0623, + ("w", "H"): 0x0624, + ("a", "h"): 0x0625, + ("y", "H"): 0x0626, + ("a", "+"): 0x0627, + ("b", "+"): 0x0628, + ("t", "m"): 0x0629, + ("t", "+"): 0x062A, + ("t", "k"): 0x062B, + ("g", "+"): 0x062C, + ("h", "k"): 0x062D, + ("x", "+"): 0x062E, + ("d", "+"): 0x062F, + ("d", "k"): 0x0630, + ("r", "+"): 0x0631, + ("z", "+"): 0x0632, + ("s", "+"): 0x0633, + ("s", "n"): 0x0634, + ("c", "+"): 0x0635, + ("d", "d"): 0x0636, + ("t", "j"): 0x0637, + ("z", "H"): 0x0638, + ("e", "+"): 0x0639, + ("i", "+"): 0x063A, + ("+", "+"): 0x0640, + ("f", "+"): 0x0641, + ("q", "+"): 0x0642, + ("k", "+"): 0x0643, + ("l", "+"): 0x0644, + ("m", "+"): 0x0645, + ("n", "+"): 0x0646, + ("h", "+"): 0x0647, + ("w", "+"): 0x0648, + ("j", "+"): 0x0649, + ("y", "+"): 0x064A, + (":", "+"): 0x064B, + ('"', "+"): 0x064C, + ("=", "+"): 0x064D, + ("/", "+"): 0x064E, + ("'", "+"): 0x064F, + ("1", "+"): 0x0650, + ("3", "+"): 0x0651, + ("0", "+"): 0x0652, + ("a", "S"): 0x0670, + ("p", "+"): 0x067E, + ("v", "+"): 0x06A4, + ("g", "f"): 0x06AF, + ("0", "a"): 0x06F0, + ("1", "a"): 0x06F1, + ("2", "a"): 0x06F2, + ("3", "a"): 0x06F3, + ("4", "a"): 0x06F4, + ("5", "a"): 0x06F5, + ("6", "a"): 0x06F6, + ("7", "a"): 0x06F7, + ("8", "a"): 0x06F8, + ("9", "a"): 0x06F9, + ("B", "."): 0x1E02, + ("b", "."): 0x1E03, + ("B", "_"): 0x1E06, + ("b", "_"): 0x1E07, + ("D", "."): 0x1E0A, + ("d", "."): 0x1E0B, + ("D", "_"): 0x1E0E, + ("d", "_"): 0x1E0F, + ("D", ","): 0x1E10, + ("d", ","): 0x1E11, + ("F", "."): 0x1E1E, + ("f", "."): 0x1E1F, + ("G", "-"): 0x1E20, + ("g", "-"): 0x1E21, + ("H", "."): 0x1E22, + ("h", "."): 0x1E23, + ("H", ":"): 0x1E26, + ("h", ":"): 0x1E27, + ("H", ","): 0x1E28, + ("h", ","): 0x1E29, + ("K", "'"): 0x1E30, + ("k", "'"): 0x1E31, + ("K", "_"): 0x1E34, + ("k", "_"): 0x1E35, + ("L", "_"): 0x1E3A, + ("l", "_"): 0x1E3B, + ("M", "'"): 0x1E3E, + ("m", "'"): 0x1E3F, + ("M", "."): 0x1E40, + ("m", "."): 0x1E41, + ("N", "."): 0x1E44, + ("n", "."): 0x1E45, + ("N", "_"): 0x1E48, + ("n", "_"): 0x1E49, + ("P", "'"): 0x1E54, + ("p", "'"): 0x1E55, + ("P", "."): 0x1E56, + ("p", "."): 0x1E57, + ("R", "."): 0x1E58, + ("r", "."): 0x1E59, + ("R", "_"): 0x1E5E, + ("r", "_"): 0x1E5F, + ("S", "."): 0x1E60, + ("s", "."): 0x1E61, + ("T", "."): 0x1E6A, + ("t", "."): 0x1E6B, + ("T", "_"): 0x1E6E, + ("t", "_"): 0x1E6F, + ("V", "?"): 0x1E7C, + ("v", "?"): 0x1E7D, + ("W", "!"): 0x1E80, + ("w", "!"): 0x1E81, + ("W", "'"): 0x1E82, + ("w", "'"): 0x1E83, + ("W", ":"): 0x1E84, + ("w", ":"): 0x1E85, + ("W", "."): 0x1E86, + ("w", "."): 0x1E87, + ("X", "."): 0x1E8A, + ("x", "."): 0x1E8B, + ("X", ":"): 0x1E8C, + ("x", ":"): 0x1E8D, + ("Y", "."): 0x1E8E, + ("y", "."): 0x1E8F, + ("Z", ">"): 0x1E90, + ("z", ">"): 0x1E91, + ("Z", "_"): 0x1E94, + ("z", "_"): 0x1E95, + ("h", "_"): 0x1E96, + ("t", ":"): 0x1E97, + ("w", "0"): 0x1E98, + ("y", "0"): 0x1E99, + ("A", "2"): 0x1EA2, + ("a", "2"): 0x1EA3, + ("E", "2"): 0x1EBA, + ("e", "2"): 0x1EBB, + ("E", "?"): 0x1EBC, + ("e", "?"): 0x1EBD, + ("I", "2"): 0x1EC8, + ("i", "2"): 0x1EC9, + ("O", "2"): 0x1ECE, + ("o", "2"): 0x1ECF, + ("U", "2"): 0x1EE6, + ("u", "2"): 0x1EE7, + ("Y", "!"): 0x1EF2, + ("y", "!"): 0x1EF3, + ("Y", "2"): 0x1EF6, + ("y", "2"): 0x1EF7, + ("Y", "?"): 0x1EF8, + ("y", "?"): 0x1EF9, + (";", "'"): 0x1F00, + (",", "'"): 0x1F01, + (";", "!"): 0x1F02, + (",", "!"): 0x1F03, + ("?", ";"): 0x1F04, + ("?", ","): 0x1F05, + ("!", ":"): 0x1F06, + ("?", ":"): 0x1F07, + ("1", "N"): 0x2002, + ("1", "M"): 0x2003, + ("3", "M"): 0x2004, + ("4", "M"): 0x2005, + ("6", "M"): 0x2006, + ("1", "T"): 0x2009, + ("1", "H"): 0x200A, + ("-", "1"): 0x2010, + ("-", "N"): 0x2013, + ("-", "M"): 0x2014, + ("-", "3"): 0x2015, + ("!", "2"): 0x2016, + ("=", "2"): 0x2017, + ("'", "6"): 0x2018, + ("'", "9"): 0x2019, + (".", "9"): 0x201A, + ("9", "'"): 0x201B, + ('"', "6"): 0x201C, + ('"', "9"): 0x201D, + (":", "9"): 0x201E, + ("9", '"'): 0x201F, + ("/", "-"): 0x2020, + ("/", "="): 0x2021, + (".", "."): 0x2025, + ("%", "0"): 0x2030, + ("1", "'"): 0x2032, + ("2", "'"): 0x2033, + ("3", "'"): 0x2034, + ("1", '"'): 0x2035, + ("2", '"'): 0x2036, + ("3", '"'): 0x2037, + ("C", "a"): 0x2038, + ("<", "1"): 0x2039, + (">", "1"): 0x203A, + (":", "X"): 0x203B, + ("'", "-"): 0x203E, + ("/", "f"): 0x2044, + ("0", "S"): 0x2070, + ("4", "S"): 0x2074, + ("5", "S"): 0x2075, + ("6", "S"): 0x2076, + ("7", "S"): 0x2077, + ("8", "S"): 0x2078, + ("9", "S"): 0x2079, + ("+", "S"): 0x207A, + ("-", "S"): 0x207B, + ("=", "S"): 0x207C, + ("(", "S"): 0x207D, + (")", "S"): 0x207E, + ("n", "S"): 0x207F, + ("0", "s"): 0x2080, + ("1", "s"): 0x2081, + ("2", "s"): 0x2082, + ("3", "s"): 0x2083, + ("4", "s"): 0x2084, + ("5", "s"): 0x2085, + ("6", "s"): 0x2086, + ("7", "s"): 0x2087, + ("8", "s"): 0x2088, + ("9", "s"): 0x2089, + ("+", "s"): 0x208A, + ("-", "s"): 0x208B, + ("=", "s"): 0x208C, + ("(", "s"): 0x208D, + (")", "s"): 0x208E, + ("L", "i"): 0x20A4, + ("P", "t"): 0x20A7, + ("W", "="): 0x20A9, + ("=", "e"): 0x20AC, # euro + ("E", "u"): 0x20AC, # euro + ("=", "R"): 0x20BD, # rouble + ("=", "P"): 0x20BD, # rouble + ("o", "C"): 0x2103, + ("c", "o"): 0x2105, + ("o", "F"): 0x2109, + ("N", "0"): 0x2116, + ("P", "O"): 0x2117, + ("R", "x"): 0x211E, + ("S", "M"): 0x2120, + ("T", "M"): 0x2122, + ("O", "m"): 0x2126, + ("A", "O"): 0x212B, + ("1", "3"): 0x2153, + ("2", "3"): 0x2154, + ("1", "5"): 0x2155, + ("2", "5"): 0x2156, + ("3", "5"): 0x2157, + ("4", "5"): 0x2158, + ("1", "6"): 0x2159, + ("5", "6"): 0x215A, + ("1", "8"): 0x215B, + ("3", "8"): 0x215C, + ("5", "8"): 0x215D, + ("7", "8"): 0x215E, + ("1", "R"): 0x2160, + ("2", "R"): 0x2161, + ("3", "R"): 0x2162, + ("4", "R"): 0x2163, + ("5", "R"): 0x2164, + ("6", "R"): 0x2165, + ("7", "R"): 0x2166, + ("8", "R"): 0x2167, + ("9", "R"): 0x2168, + ("a", "R"): 0x2169, + ("b", "R"): 0x216A, + ("c", "R"): 0x216B, + ("1", "r"): 0x2170, + ("2", "r"): 0x2171, + ("3", "r"): 0x2172, + ("4", "r"): 0x2173, + ("5", "r"): 0x2174, + ("6", "r"): 0x2175, + ("7", "r"): 0x2176, + ("8", "r"): 0x2177, + ("9", "r"): 0x2178, + ("a", "r"): 0x2179, + ("b", "r"): 0x217A, + ("c", "r"): 0x217B, + ("<", "-"): 0x2190, + ("-", "!"): 0x2191, + ("-", ">"): 0x2192, + ("-", "v"): 0x2193, + ("<", ">"): 0x2194, + ("U", "D"): 0x2195, + ("<", "="): 0x21D0, + ("=", ">"): 0x21D2, + ("=", "="): 0x21D4, + ("F", "A"): 0x2200, + ("d", "P"): 0x2202, + ("T", "E"): 0x2203, + ("/", "0"): 0x2205, + ("D", "E"): 0x2206, + ("N", "B"): 0x2207, + ("(", "-"): 0x2208, + ("-", ")"): 0x220B, + ("*", "P"): 0x220F, + ("+", "Z"): 0x2211, + ("-", "2"): 0x2212, + ("-", "+"): 0x2213, + ("*", "-"): 0x2217, + ("O", "b"): 0x2218, + ("S", "b"): 0x2219, + ("R", "T"): 0x221A, + ("0", "("): 0x221D, + ("0", "0"): 0x221E, + ("-", "L"): 0x221F, + ("-", "V"): 0x2220, + ("P", "P"): 0x2225, + ("A", "N"): 0x2227, + ("O", "R"): 0x2228, + ("(", "U"): 0x2229, + (")", "U"): 0x222A, + ("I", "n"): 0x222B, + ("D", "I"): 0x222C, + ("I", "o"): 0x222E, + (".", ":"): 0x2234, + (":", "."): 0x2235, + (":", "R"): 0x2236, + (":", ":"): 0x2237, + ("?", "1"): 0x223C, + ("C", "G"): 0x223E, + ("?", "-"): 0x2243, + ("?", "="): 0x2245, + ("?", "2"): 0x2248, + ("=", "?"): 0x224C, + ("H", "I"): 0x2253, + ("!", "="): 0x2260, + ("=", "3"): 0x2261, + ("=", "<"): 0x2264, + (">", "="): 0x2265, + ("<", "*"): 0x226A, + ("*", ">"): 0x226B, + ("!", "<"): 0x226E, + ("!", ">"): 0x226F, + ("(", "C"): 0x2282, + (")", "C"): 0x2283, + ("(", "_"): 0x2286, + (")", "_"): 0x2287, + ("0", "."): 0x2299, + ("0", "2"): 0x229A, + ("-", "T"): 0x22A5, + (".", "P"): 0x22C5, + (":", "3"): 0x22EE, + (".", "3"): 0x22EF, + ("E", "h"): 0x2302, + ("<", "7"): 0x2308, + (">", "7"): 0x2309, + ("7", "<"): 0x230A, + ("7", ">"): 0x230B, + ("N", "I"): 0x2310, + ("(", "A"): 0x2312, + ("T", "R"): 0x2315, + ("I", "u"): 0x2320, + ("I", "l"): 0x2321, + ("<", "/"): 0x2329, + ("/", ">"): 0x232A, + ("V", "s"): 0x2423, + ("1", "h"): 0x2440, + ("3", "h"): 0x2441, + ("2", "h"): 0x2442, + ("4", "h"): 0x2443, + ("1", "j"): 0x2446, + ("2", "j"): 0x2447, + ("3", "j"): 0x2448, + ("4", "j"): 0x2449, + ("1", "."): 0x2488, + ("2", "."): 0x2489, + ("3", "."): 0x248A, + ("4", "."): 0x248B, + ("5", "."): 0x248C, + ("6", "."): 0x248D, + ("7", "."): 0x248E, + ("8", "."): 0x248F, + ("9", "."): 0x2490, + ("h", "h"): 0x2500, + ("H", "H"): 0x2501, + ("v", "v"): 0x2502, + ("V", "V"): 0x2503, + ("3", "-"): 0x2504, + ("3", "_"): 0x2505, + ("3", "!"): 0x2506, + ("3", "/"): 0x2507, + ("4", "-"): 0x2508, + ("4", "_"): 0x2509, + ("4", "!"): 0x250A, + ("4", "/"): 0x250B, + ("d", "r"): 0x250C, + ("d", "R"): 0x250D, + ("D", "r"): 0x250E, + ("D", "R"): 0x250F, + ("d", "l"): 0x2510, + ("d", "L"): 0x2511, + ("D", "l"): 0x2512, + ("L", "D"): 0x2513, + ("u", "r"): 0x2514, + ("u", "R"): 0x2515, + ("U", "r"): 0x2516, + ("U", "R"): 0x2517, + ("u", "l"): 0x2518, + ("u", "L"): 0x2519, + ("U", "l"): 0x251A, + ("U", "L"): 0x251B, + ("v", "r"): 0x251C, + ("v", "R"): 0x251D, + ("V", "r"): 0x2520, + ("V", "R"): 0x2523, + ("v", "l"): 0x2524, + ("v", "L"): 0x2525, + ("V", "l"): 0x2528, + ("V", "L"): 0x252B, + ("d", "h"): 0x252C, + ("d", "H"): 0x252F, + ("D", "h"): 0x2530, + ("D", "H"): 0x2533, + ("u", "h"): 0x2534, + ("u", "H"): 0x2537, + ("U", "h"): 0x2538, + ("U", "H"): 0x253B, + ("v", "h"): 0x253C, + ("v", "H"): 0x253F, + ("V", "h"): 0x2542, + ("V", "H"): 0x254B, + ("F", "D"): 0x2571, + ("B", "D"): 0x2572, + ("T", "B"): 0x2580, + ("L", "B"): 0x2584, + ("F", "B"): 0x2588, + ("l", "B"): 0x258C, + ("R", "B"): 0x2590, + (".", "S"): 0x2591, + (":", "S"): 0x2592, + ("?", "S"): 0x2593, + ("f", "S"): 0x25A0, + ("O", "S"): 0x25A1, + ("R", "O"): 0x25A2, + ("R", "r"): 0x25A3, + ("R", "F"): 0x25A4, + ("R", "Y"): 0x25A5, + ("R", "H"): 0x25A6, + ("R", "Z"): 0x25A7, + ("R", "K"): 0x25A8, + ("R", "X"): 0x25A9, + ("s", "B"): 0x25AA, + ("S", "R"): 0x25AC, + ("O", "r"): 0x25AD, + ("U", "T"): 0x25B2, + ("u", "T"): 0x25B3, + ("P", "R"): 0x25B6, + ("T", "r"): 0x25B7, + ("D", "t"): 0x25BC, + ("d", "T"): 0x25BD, + ("P", "L"): 0x25C0, + ("T", "l"): 0x25C1, + ("D", "b"): 0x25C6, + ("D", "w"): 0x25C7, + ("L", "Z"): 0x25CA, + ("0", "m"): 0x25CB, + ("0", "o"): 0x25CE, + ("0", "M"): 0x25CF, + ("0", "L"): 0x25D0, + ("0", "R"): 0x25D1, + ("S", "n"): 0x25D8, + ("I", "c"): 0x25D9, + ("F", "d"): 0x25E2, + ("B", "d"): 0x25E3, + ("*", "2"): 0x2605, + ("*", "1"): 0x2606, + ("<", "H"): 0x261C, + (">", "H"): 0x261E, + ("0", "u"): 0x263A, + ("0", "U"): 0x263B, + ("S", "U"): 0x263C, + ("F", "m"): 0x2640, + ("M", "l"): 0x2642, + ("c", "S"): 0x2660, + ("c", "H"): 0x2661, + ("c", "D"): 0x2662, + ("c", "C"): 0x2663, + ("M", "d"): 0x2669, + ("M", "8"): 0x266A, + ("M", "2"): 0x266B, + ("M", "b"): 0x266D, + ("M", "x"): 0x266E, + ("M", "X"): 0x266F, + ("O", "K"): 0x2713, + ("X", "X"): 0x2717, + ("-", "X"): 0x2720, + ("I", "S"): 0x3000, + (",", "_"): 0x3001, + (".", "_"): 0x3002, + ("+", '"'): 0x3003, + ("+", "_"): 0x3004, + ("*", "_"): 0x3005, + (";", "_"): 0x3006, + ("0", "_"): 0x3007, + ("<", "+"): 0x300A, + (">", "+"): 0x300B, + ("<", "'"): 0x300C, + (">", "'"): 0x300D, + ("<", '"'): 0x300E, + (">", '"'): 0x300F, + ("(", '"'): 0x3010, + (")", '"'): 0x3011, + ("=", "T"): 0x3012, + ("=", "_"): 0x3013, + ("(", "'"): 0x3014, + (")", "'"): 0x3015, + ("(", "I"): 0x3016, + (")", "I"): 0x3017, + ("-", "?"): 0x301C, + ("A", "5"): 0x3041, + ("a", "5"): 0x3042, + ("I", "5"): 0x3043, + ("i", "5"): 0x3044, + ("U", "5"): 0x3045, + ("u", "5"): 0x3046, + ("E", "5"): 0x3047, + ("e", "5"): 0x3048, + ("O", "5"): 0x3049, + ("o", "5"): 0x304A, + ("k", "a"): 0x304B, + ("g", "a"): 0x304C, + ("k", "i"): 0x304D, + ("g", "i"): 0x304E, + ("k", "u"): 0x304F, + ("g", "u"): 0x3050, + ("k", "e"): 0x3051, + ("g", "e"): 0x3052, + ("k", "o"): 0x3053, + ("g", "o"): 0x3054, + ("s", "a"): 0x3055, + ("z", "a"): 0x3056, + ("s", "i"): 0x3057, + ("z", "i"): 0x3058, + ("s", "u"): 0x3059, + ("z", "u"): 0x305A, + ("s", "e"): 0x305B, + ("z", "e"): 0x305C, + ("s", "o"): 0x305D, + ("z", "o"): 0x305E, + ("t", "a"): 0x305F, + ("d", "a"): 0x3060, + ("t", "i"): 0x3061, + ("d", "i"): 0x3062, + ("t", "U"): 0x3063, + ("t", "u"): 0x3064, + ("d", "u"): 0x3065, + ("t", "e"): 0x3066, + ("d", "e"): 0x3067, + ("t", "o"): 0x3068, + ("d", "o"): 0x3069, + ("n", "a"): 0x306A, + ("n", "i"): 0x306B, + ("n", "u"): 0x306C, + ("n", "e"): 0x306D, + ("n", "o"): 0x306E, + ("h", "a"): 0x306F, + ("b", "a"): 0x3070, + ("p", "a"): 0x3071, + ("h", "i"): 0x3072, + ("b", "i"): 0x3073, + ("p", "i"): 0x3074, + ("h", "u"): 0x3075, + ("b", "u"): 0x3076, + ("p", "u"): 0x3077, + ("h", "e"): 0x3078, + ("b", "e"): 0x3079, + ("p", "e"): 0x307A, + ("h", "o"): 0x307B, + ("b", "o"): 0x307C, + ("p", "o"): 0x307D, + ("m", "a"): 0x307E, + ("m", "i"): 0x307F, + ("m", "u"): 0x3080, + ("m", "e"): 0x3081, + ("m", "o"): 0x3082, + ("y", "A"): 0x3083, + ("y", "a"): 0x3084, + ("y", "U"): 0x3085, + ("y", "u"): 0x3086, + ("y", "O"): 0x3087, + ("y", "o"): 0x3088, + ("r", "a"): 0x3089, + ("r", "i"): 0x308A, + ("r", "u"): 0x308B, + ("r", "e"): 0x308C, + ("r", "o"): 0x308D, + ("w", "A"): 0x308E, + ("w", "a"): 0x308F, + ("w", "i"): 0x3090, + ("w", "e"): 0x3091, + ("w", "o"): 0x3092, + ("n", "5"): 0x3093, + ("v", "u"): 0x3094, + ('"', "5"): 0x309B, + ("0", "5"): 0x309C, + ("*", "5"): 0x309D, + ("+", "5"): 0x309E, + ("a", "6"): 0x30A1, + ("A", "6"): 0x30A2, + ("i", "6"): 0x30A3, + ("I", "6"): 0x30A4, + ("u", "6"): 0x30A5, + ("U", "6"): 0x30A6, + ("e", "6"): 0x30A7, + ("E", "6"): 0x30A8, + ("o", "6"): 0x30A9, + ("O", "6"): 0x30AA, + ("K", "a"): 0x30AB, + ("G", "a"): 0x30AC, + ("K", "i"): 0x30AD, + ("G", "i"): 0x30AE, + ("K", "u"): 0x30AF, + ("G", "u"): 0x30B0, + ("K", "e"): 0x30B1, + ("G", "e"): 0x30B2, + ("K", "o"): 0x30B3, + ("G", "o"): 0x30B4, + ("S", "a"): 0x30B5, + ("Z", "a"): 0x30B6, + ("S", "i"): 0x30B7, + ("Z", "i"): 0x30B8, + ("S", "u"): 0x30B9, + ("Z", "u"): 0x30BA, + ("S", "e"): 0x30BB, + ("Z", "e"): 0x30BC, + ("S", "o"): 0x30BD, + ("Z", "o"): 0x30BE, + ("T", "a"): 0x30BF, + ("D", "a"): 0x30C0, + ("T", "i"): 0x30C1, + ("D", "i"): 0x30C2, + ("T", "U"): 0x30C3, + ("T", "u"): 0x30C4, + ("D", "u"): 0x30C5, + ("T", "e"): 0x30C6, + ("D", "e"): 0x30C7, + ("T", "o"): 0x30C8, + ("D", "o"): 0x30C9, + ("N", "a"): 0x30CA, + ("N", "i"): 0x30CB, + ("N", "u"): 0x30CC, + ("N", "e"): 0x30CD, + ("N", "o"): 0x30CE, + ("H", "a"): 0x30CF, + ("B", "a"): 0x30D0, + ("P", "a"): 0x30D1, + ("H", "i"): 0x30D2, + ("B", "i"): 0x30D3, + ("P", "i"): 0x30D4, + ("H", "u"): 0x30D5, + ("B", "u"): 0x30D6, + ("P", "u"): 0x30D7, + ("H", "e"): 0x30D8, + ("B", "e"): 0x30D9, + ("P", "e"): 0x30DA, + ("H", "o"): 0x30DB, + ("B", "o"): 0x30DC, + ("P", "o"): 0x30DD, + ("M", "a"): 0x30DE, + ("M", "i"): 0x30DF, + ("M", "u"): 0x30E0, + ("M", "e"): 0x30E1, + ("M", "o"): 0x30E2, + ("Y", "A"): 0x30E3, + ("Y", "a"): 0x30E4, + ("Y", "U"): 0x30E5, + ("Y", "u"): 0x30E6, + ("Y", "O"): 0x30E7, + ("Y", "o"): 0x30E8, + ("R", "a"): 0x30E9, + ("R", "i"): 0x30EA, + ("R", "u"): 0x30EB, + ("R", "e"): 0x30EC, + ("R", "o"): 0x30ED, + ("W", "A"): 0x30EE, + ("W", "a"): 0x30EF, + ("W", "i"): 0x30F0, + ("W", "e"): 0x30F1, + ("W", "o"): 0x30F2, + ("N", "6"): 0x30F3, + ("V", "u"): 0x30F4, + ("K", "A"): 0x30F5, + ("K", "E"): 0x30F6, + ("V", "a"): 0x30F7, + ("V", "i"): 0x30F8, + ("V", "e"): 0x30F9, + ("V", "o"): 0x30FA, + (".", "6"): 0x30FB, + ("-", "6"): 0x30FC, + ("*", "6"): 0x30FD, + ("+", "6"): 0x30FE, + ("b", "4"): 0x3105, + ("p", "4"): 0x3106, + ("m", "4"): 0x3107, + ("f", "4"): 0x3108, + ("d", "4"): 0x3109, + ("t", "4"): 0x310A, + ("n", "4"): 0x310B, + ("l", "4"): 0x310C, + ("g", "4"): 0x310D, + ("k", "4"): 0x310E, + ("h", "4"): 0x310F, + ("j", "4"): 0x3110, + ("q", "4"): 0x3111, + ("x", "4"): 0x3112, + ("z", "h"): 0x3113, + ("c", "h"): 0x3114, + ("s", "h"): 0x3115, + ("r", "4"): 0x3116, + ("z", "4"): 0x3117, + ("c", "4"): 0x3118, + ("s", "4"): 0x3119, + ("a", "4"): 0x311A, + ("o", "4"): 0x311B, + ("e", "4"): 0x311C, + ("a", "i"): 0x311E, + ("e", "i"): 0x311F, + ("a", "u"): 0x3120, + ("o", "u"): 0x3121, + ("a", "n"): 0x3122, + ("e", "n"): 0x3123, + ("a", "N"): 0x3124, + ("e", "N"): 0x3125, + ("e", "r"): 0x3126, + ("i", "4"): 0x3127, + ("u", "4"): 0x3128, + ("i", "u"): 0x3129, + ("v", "4"): 0x312A, + ("n", "G"): 0x312B, + ("g", "n"): 0x312C, + ("1", "c"): 0x3220, + ("2", "c"): 0x3221, + ("3", "c"): 0x3222, + ("4", "c"): 0x3223, + ("5", "c"): 0x3224, + ("6", "c"): 0x3225, + ("7", "c"): 0x3226, + ("8", "c"): 0x3227, + ("9", "c"): 0x3228, + # code points 0xe000 - 0xefff excluded, they have no assigned + # characters, only used in proposals. + ("f", "f"): 0xFB00, + ("f", "i"): 0xFB01, + ("f", "l"): 0xFB02, + ("f", "t"): 0xFB05, + ("s", "t"): 0xFB06, + # Vim 5.x compatible digraphs that don't conflict with the above + ("~", "!"): 161, + ("c", "|"): 162, + ("$", "$"): 163, + ("o", "x"): 164, # currency symbol in ISO 8859-1 + ("Y", "-"): 165, + ("|", "|"): 166, + ("c", "O"): 169, + ("-", ","): 172, + ("-", "="): 175, + ("~", "o"): 176, + ("2", "2"): 178, + ("3", "3"): 179, + ("p", "p"): 182, + ("~", "."): 183, + ("1", "1"): 185, + ("~", "?"): 191, + ("A", "`"): 192, + ("A", "^"): 194, + ("A", "~"): 195, + ("A", '"'): 196, + ("A", "@"): 197, + ("E", "`"): 200, + ("E", "^"): 202, + ("E", '"'): 203, + ("I", "`"): 204, + ("I", "^"): 206, + ("I", '"'): 207, + ("N", "~"): 209, + ("O", "`"): 210, + ("O", "^"): 212, + ("O", "~"): 213, + ("/", "\\"): 215, # multiplication symbol in ISO 8859-1 + ("U", "`"): 217, + ("U", "^"): 219, + ("I", "p"): 222, + ("a", "`"): 224, + ("a", "^"): 226, + ("a", "~"): 227, + ("a", '"'): 228, + ("a", "@"): 229, + ("e", "`"): 232, + ("e", "^"): 234, + ("e", '"'): 235, + ("i", "`"): 236, + ("i", "^"): 238, + ("n", "~"): 241, + ("o", "`"): 242, + ("o", "^"): 244, + ("o", "~"): 245, + ("u", "`"): 249, + ("u", "^"): 251, + ("y", '"'): 255, +} diff --git a/lib/prompt_toolkit/key_binding/emacs_state.py b/lib/prompt_toolkit/key_binding/emacs_state.py new file mode 100644 index 0000000..6a2ebf4 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/emacs_state.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from .key_processor import KeyPress + +__all__ = [ + "EmacsState", +] + + +class EmacsState: + """ + Mutable class to hold Emacs specific state. + """ + + def __init__(self) -> None: + # Simple macro recording. (Like Readline does.) + # (For Emacs mode.) + self.macro: list[KeyPress] | None = [] + self.current_recording: list[KeyPress] | None = None + + def reset(self) -> None: + self.current_recording = None + + @property + def is_recording(self) -> bool: + "Tell whether we are recording a macro." + return self.current_recording is not None + + def start_macro(self) -> None: + "Start recording macro." + self.current_recording = [] + + def end_macro(self) -> None: + "End recording macro." + self.macro = self.current_recording + self.current_recording = None diff --git a/lib/prompt_toolkit/key_binding/key_bindings.py b/lib/prompt_toolkit/key_binding/key_bindings.py new file mode 100644 index 0000000..cf37c6d --- /dev/null +++ b/lib/prompt_toolkit/key_binding/key_bindings.py @@ -0,0 +1,672 @@ +""" +Key bindings registry. + +A `KeyBindings` object is a container that holds a list of key bindings. It has a +very efficient internal data structure for checking which key bindings apply +for a pressed key. + +Typical usage:: + + kb = KeyBindings() + + @kb.add(Keys.ControlX, Keys.ControlC, filter=INSERT) + def handler(event): + # Handle ControlX-ControlC key sequence. + pass + +It is also possible to combine multiple KeyBindings objects. We do this in the +default key bindings. There are some KeyBindings objects that contain the Emacs +bindings, while others contain the Vi bindings. They are merged together using +`merge_key_bindings`. + +We also have a `ConditionalKeyBindings` object that can enable/disable a group of +key bindings at once. + + +It is also possible to add a filter to a function, before a key binding has +been assigned, through the `key_binding` decorator.:: + + # First define a key handler with the `filter`. + @key_binding(filter=condition) + def my_key_binding(event): + ... + + # Later, add it to the key bindings. + kb.add(Keys.A, my_key_binding) +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from inspect import isawaitable +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Coroutine, + Hashable, + Sequence, + Tuple, + TypeVar, + Union, + cast, +) + +from prompt_toolkit.cache import SimpleCache +from prompt_toolkit.filters import FilterOrBool, Never, to_filter +from prompt_toolkit.keys import KEY_ALIASES, Keys + +if TYPE_CHECKING: + # Avoid circular imports. + from .key_processor import KeyPressEvent + + # The only two return values for a mouse handler (and key bindings) are + # `None` and `NotImplemented`. For the type checker it's best to annotate + # this as `object`. (The consumer never expects a more specific instance: + # checking for NotImplemented can be done using `is NotImplemented`.) + NotImplementedOrNone = object + # Other non-working options are: + # * Optional[Literal[NotImplemented]] + # --> Doesn't work, Literal can't take an Any. + # * None + # --> Doesn't work. We can't assign the result of a function that + # returns `None` to a variable. + # * Any + # --> Works, but too broad. + + +__all__ = [ + "NotImplementedOrNone", + "Binding", + "KeyBindingsBase", + "KeyBindings", + "ConditionalKeyBindings", + "merge_key_bindings", + "DynamicKeyBindings", + "GlobalOnlyKeyBindings", +] + +# Key bindings can be regular functions or coroutines. +# In both cases, if they return `NotImplemented`, the UI won't be invalidated. +# This is mainly used in case of mouse move events, to prevent excessive +# repainting during mouse move events. +KeyHandlerCallable = Callable[ + ["KeyPressEvent"], + Union["NotImplementedOrNone", Coroutine[Any, Any, "NotImplementedOrNone"]], +] + + +class Binding: + """ + Key binding: (key sequence + handler + filter). + (Immutable binding class.) + + :param record_in_macro: When True, don't record this key binding when a + macro is recorded. + """ + + def __init__( + self, + keys: tuple[Keys | str, ...], + handler: KeyHandlerCallable, + filter: FilterOrBool = True, + eager: FilterOrBool = False, + is_global: FilterOrBool = False, + save_before: Callable[[KeyPressEvent], bool] = (lambda e: True), + record_in_macro: FilterOrBool = True, + ) -> None: + self.keys = keys + self.handler = handler + self.filter = to_filter(filter) + self.eager = to_filter(eager) + self.is_global = to_filter(is_global) + self.save_before = save_before + self.record_in_macro = to_filter(record_in_macro) + + def call(self, event: KeyPressEvent) -> None: + result = self.handler(event) + + # If the handler is a coroutine, create an asyncio task. + if isawaitable(result): + awaitable = cast(Coroutine[Any, Any, "NotImplementedOrNone"], result) + + async def bg_task() -> None: + result = await awaitable + if result != NotImplemented: + event.app.invalidate() + + event.app.create_background_task(bg_task()) + + elif result != NotImplemented: + event.app.invalidate() + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(keys={self.keys!r}, handler={self.handler!r})" + ) + + +# Sequence of keys presses. +KeysTuple = Tuple[Union[Keys, str], ...] + + +class KeyBindingsBase(metaclass=ABCMeta): + """ + Interface for a KeyBindings. + """ + + @property + @abstractmethod + def _version(self) -> Hashable: + """ + For cache invalidation. - This should increase every time that + something changes. + """ + return 0 + + @abstractmethod + def get_bindings_for_keys(self, keys: KeysTuple) -> list[Binding]: + """ + Return a list of key bindings that can handle these keys. + (This return also inactive bindings, so the `filter` still has to be + called, for checking it.) + + :param keys: tuple of keys. + """ + return [] + + @abstractmethod + def get_bindings_starting_with_keys(self, keys: KeysTuple) -> list[Binding]: + """ + Return a list of key bindings that handle a key sequence starting with + `keys`. (It does only return bindings for which the sequences are + longer than `keys`. And like `get_bindings_for_keys`, it also includes + inactive bindings.) + + :param keys: tuple of keys. + """ + return [] + + @property + @abstractmethod + def bindings(self) -> list[Binding]: + """ + List of `Binding` objects. + (These need to be exposed, so that `KeyBindings` objects can be merged + together.) + """ + return [] + + # `add` and `remove` don't have to be part of this interface. + + +T = TypeVar("T", bound=Union[KeyHandlerCallable, Binding]) + + +class KeyBindings(KeyBindingsBase): + """ + A container for a set of key bindings. + + Example usage:: + + kb = KeyBindings() + + @kb.add('c-t') + def _(event): + print('Control-T pressed') + + @kb.add('c-a', 'c-b') + def _(event): + print('Control-A pressed, followed by Control-B') + + @kb.add('c-x', filter=is_searching) + def _(event): + print('Control-X pressed') # Works only if we are searching. + + """ + + def __init__(self) -> None: + self._bindings: list[Binding] = [] + self._get_bindings_for_keys_cache: SimpleCache[KeysTuple, list[Binding]] = ( + SimpleCache(maxsize=10000) + ) + self._get_bindings_starting_with_keys_cache: SimpleCache[ + KeysTuple, list[Binding] + ] = SimpleCache(maxsize=1000) + self.__version = 0 # For cache invalidation. + + def _clear_cache(self) -> None: + self.__version += 1 + self._get_bindings_for_keys_cache.clear() + self._get_bindings_starting_with_keys_cache.clear() + + @property + def bindings(self) -> list[Binding]: + return self._bindings + + @property + def _version(self) -> Hashable: + return self.__version + + def add( + self, + *keys: Keys | str, + filter: FilterOrBool = True, + eager: FilterOrBool = False, + is_global: FilterOrBool = False, + save_before: Callable[[KeyPressEvent], bool] = (lambda e: True), + record_in_macro: FilterOrBool = True, + ) -> Callable[[T], T]: + """ + Decorator for adding a key bindings. + + :param filter: :class:`~prompt_toolkit.filters.Filter` to determine + when this key binding is active. + :param eager: :class:`~prompt_toolkit.filters.Filter` or `bool`. + When True, ignore potential longer matches when this key binding is + hit. E.g. when there is an active eager key binding for Ctrl-X, + execute the handler immediately and ignore the key binding for + Ctrl-X Ctrl-E of which it is a prefix. + :param is_global: When this key bindings is added to a `Container` or + `Control`, make it a global (always active) binding. + :param save_before: Callable that takes an `Event` and returns True if + we should save the current buffer, before handling the event. + (That's the default.) + :param record_in_macro: Record these key bindings when a macro is + being recorded. (True by default.) + """ + assert keys + + keys = tuple(_parse_key(k) for k in keys) + + if isinstance(filter, Never): + # When a filter is Never, it will always stay disabled, so in that + # case don't bother putting it in the key bindings. It will slow + # down every key press otherwise. + def decorator(func: T) -> T: + return func + + else: + + def decorator(func: T) -> T: + if isinstance(func, Binding): + # We're adding an existing Binding object. + self.bindings.append( + Binding( + keys, + func.handler, + filter=func.filter & to_filter(filter), + eager=to_filter(eager) | func.eager, + is_global=to_filter(is_global) | func.is_global, + save_before=func.save_before, + record_in_macro=func.record_in_macro, + ) + ) + else: + self.bindings.append( + Binding( + keys, + cast(KeyHandlerCallable, func), + filter=filter, + eager=eager, + is_global=is_global, + save_before=save_before, + record_in_macro=record_in_macro, + ) + ) + self._clear_cache() + + return func + + return decorator + + def remove(self, *args: Keys | str | KeyHandlerCallable) -> None: + """ + Remove a key binding. + + This expects either a function that was given to `add` method as + parameter or a sequence of key bindings. + + Raises `ValueError` when no bindings was found. + + Usage:: + + remove(handler) # Pass handler. + remove('c-x', 'c-a') # Or pass the key bindings. + """ + found = False + + if callable(args[0]): + assert len(args) == 1 + function = args[0] + + # Remove the given function. + for b in self.bindings: + if b.handler == function: + self.bindings.remove(b) + found = True + + else: + assert len(args) > 0 + args = cast(Tuple[Union[Keys, str]], args) + + # Remove this sequence of key bindings. + keys = tuple(_parse_key(k) for k in args) + + for b in self.bindings: + if b.keys == keys: + self.bindings.remove(b) + found = True + + if found: + self._clear_cache() + else: + # No key binding found for this function. Raise ValueError. + raise ValueError(f"Binding not found: {function!r}") + + # For backwards-compatibility. + add_binding = add + remove_binding = remove + + def get_bindings_for_keys(self, keys: KeysTuple) -> list[Binding]: + """ + Return a list of key bindings that can handle this key. + (This return also inactive bindings, so the `filter` still has to be + called, for checking it.) + + :param keys: tuple of keys. + """ + + def get() -> list[Binding]: + result: list[tuple[int, Binding]] = [] + + for b in self.bindings: + if len(keys) == len(b.keys): + match = True + any_count = 0 + + for i, j in zip(b.keys, keys): + if i != j and i != Keys.Any: + match = False + break + + if i == Keys.Any: + any_count += 1 + + if match: + result.append((any_count, b)) + + # Place bindings that have more 'Any' occurrences in them at the end. + result = sorted(result, key=lambda item: -item[0]) + + return [item[1] for item in result] + + return self._get_bindings_for_keys_cache.get(keys, get) + + def get_bindings_starting_with_keys(self, keys: KeysTuple) -> list[Binding]: + """ + Return a list of key bindings that handle a key sequence starting with + `keys`. (It does only return bindings for which the sequences are + longer than `keys`. And like `get_bindings_for_keys`, it also includes + inactive bindings.) + + :param keys: tuple of keys. + """ + + def get() -> list[Binding]: + result = [] + for b in self.bindings: + if len(keys) < len(b.keys): + match = True + for i, j in zip(b.keys, keys): + if i != j and i != Keys.Any: + match = False + break + if match: + result.append(b) + return result + + return self._get_bindings_starting_with_keys_cache.get(keys, get) + + +def _parse_key(key: Keys | str) -> str | Keys: + """ + Replace key by alias and verify whether it's a valid one. + """ + # Already a parse key? -> Return it. + if isinstance(key, Keys): + return key + + # Lookup aliases. + key = KEY_ALIASES.get(key, key) + + # Replace 'space' by ' ' + if key == "space": + key = " " + + # Return as `Key` object when it's a special key. + try: + return Keys(key) + except ValueError: + pass + + # Final validation. + if len(key) != 1: + raise ValueError(f"Invalid key: {key}") + + return key + + +def key_binding( + filter: FilterOrBool = True, + eager: FilterOrBool = False, + is_global: FilterOrBool = False, + save_before: Callable[[KeyPressEvent], bool] = (lambda event: True), + record_in_macro: FilterOrBool = True, +) -> Callable[[KeyHandlerCallable], Binding]: + """ + Decorator that turn a function into a `Binding` object. This can be added + to a `KeyBindings` object when a key binding is assigned. + """ + assert save_before is None or callable(save_before) + + filter = to_filter(filter) + eager = to_filter(eager) + is_global = to_filter(is_global) + save_before = save_before + record_in_macro = to_filter(record_in_macro) + keys = () + + def decorator(function: KeyHandlerCallable) -> Binding: + return Binding( + keys, + function, + filter=filter, + eager=eager, + is_global=is_global, + save_before=save_before, + record_in_macro=record_in_macro, + ) + + return decorator + + +class _Proxy(KeyBindingsBase): + """ + Common part for ConditionalKeyBindings and _MergedKeyBindings. + """ + + def __init__(self) -> None: + # `KeyBindings` to be synchronized with all the others. + self._bindings2: KeyBindingsBase = KeyBindings() + self._last_version: Hashable = () + + def _update_cache(self) -> None: + """ + If `self._last_version` is outdated, then this should update + the version and `self._bindings2`. + """ + raise NotImplementedError + + # Proxy methods to self._bindings2. + + @property + def bindings(self) -> list[Binding]: + self._update_cache() + return self._bindings2.bindings + + @property + def _version(self) -> Hashable: + self._update_cache() + return self._last_version + + def get_bindings_for_keys(self, keys: KeysTuple) -> list[Binding]: + self._update_cache() + return self._bindings2.get_bindings_for_keys(keys) + + def get_bindings_starting_with_keys(self, keys: KeysTuple) -> list[Binding]: + self._update_cache() + return self._bindings2.get_bindings_starting_with_keys(keys) + + +class ConditionalKeyBindings(_Proxy): + """ + Wraps around a `KeyBindings`. Disable/enable all the key bindings according to + the given (additional) filter.:: + + @Condition + def setting_is_true(): + return True # or False + + registry = ConditionalKeyBindings(key_bindings, setting_is_true) + + When new key bindings are added to this object. They are also + enable/disabled according to the given `filter`. + + :param registries: List of :class:`.KeyBindings` objects. + :param filter: :class:`~prompt_toolkit.filters.Filter` object. + """ + + def __init__( + self, key_bindings: KeyBindingsBase, filter: FilterOrBool = True + ) -> None: + _Proxy.__init__(self) + + self.key_bindings = key_bindings + self.filter = to_filter(filter) + + def _update_cache(self) -> None: + "If the original key bindings was changed. Update our copy version." + expected_version = self.key_bindings._version + + if self._last_version != expected_version: + bindings2 = KeyBindings() + + # Copy all bindings from `self.key_bindings`, adding our condition. + for b in self.key_bindings.bindings: + bindings2.bindings.append( + Binding( + keys=b.keys, + handler=b.handler, + filter=self.filter & b.filter, + eager=b.eager, + is_global=b.is_global, + save_before=b.save_before, + record_in_macro=b.record_in_macro, + ) + ) + + self._bindings2 = bindings2 + self._last_version = expected_version + + +class _MergedKeyBindings(_Proxy): + """ + Merge multiple registries of key bindings into one. + + This class acts as a proxy to multiple :class:`.KeyBindings` objects, but + behaves as if this is just one bigger :class:`.KeyBindings`. + + :param registries: List of :class:`.KeyBindings` objects. + """ + + def __init__(self, registries: Sequence[KeyBindingsBase]) -> None: + _Proxy.__init__(self) + self.registries = registries + + def _update_cache(self) -> None: + """ + If one of the original registries was changed. Update our merged + version. + """ + expected_version = tuple(r._version for r in self.registries) + + if self._last_version != expected_version: + bindings2 = KeyBindings() + + for reg in self.registries: + bindings2.bindings.extend(reg.bindings) + + self._bindings2 = bindings2 + self._last_version = expected_version + + +def merge_key_bindings(bindings: Sequence[KeyBindingsBase]) -> _MergedKeyBindings: + """ + Merge multiple :class:`.Keybinding` objects together. + + Usage:: + + bindings = merge_key_bindings([bindings1, bindings2, ...]) + """ + return _MergedKeyBindings(bindings) + + +class DynamicKeyBindings(_Proxy): + """ + KeyBindings class that can dynamically returns any KeyBindings. + + :param get_key_bindings: Callable that returns a :class:`.KeyBindings` instance. + """ + + def __init__(self, get_key_bindings: Callable[[], KeyBindingsBase | None]) -> None: + self.get_key_bindings = get_key_bindings + self.__version = 0 + self._last_child_version = None + self._dummy = KeyBindings() # Empty key bindings. + + def _update_cache(self) -> None: + key_bindings = self.get_key_bindings() or self._dummy + assert isinstance(key_bindings, KeyBindingsBase) + version = id(key_bindings), key_bindings._version + + self._bindings2 = key_bindings + self._last_version = version + + +class GlobalOnlyKeyBindings(_Proxy): + """ + Wrapper around a :class:`.KeyBindings` object that only exposes the global + key bindings. + """ + + def __init__(self, key_bindings: KeyBindingsBase) -> None: + _Proxy.__init__(self) + self.key_bindings = key_bindings + + def _update_cache(self) -> None: + """ + If one of the original registries was changed. Update our merged + version. + """ + expected_version = self.key_bindings._version + + if self._last_version != expected_version: + bindings2 = KeyBindings() + + for b in self.key_bindings.bindings: + if b.is_global(): + bindings2.bindings.append(b) + + self._bindings2 = bindings2 + self._last_version = expected_version diff --git a/lib/prompt_toolkit/key_binding/key_processor.py b/lib/prompt_toolkit/key_binding/key_processor.py new file mode 100644 index 0000000..e2070a1 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/key_processor.py @@ -0,0 +1,526 @@ +""" +An :class:`~.KeyProcessor` receives callbacks for the keystrokes parsed from +the input in the :class:`~prompt_toolkit.inputstream.InputStream` instance. + +The `KeyProcessor` will according to the implemented keybindings call the +correct callbacks when new key presses are feed through `feed`. +""" + +from __future__ import annotations + +import weakref +from asyncio import Task, sleep +from collections import deque +from typing import TYPE_CHECKING, Any, Generator + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.enums import EditingMode +from prompt_toolkit.filters.app import vi_navigation_mode +from prompt_toolkit.keys import Keys +from prompt_toolkit.utils import Event + +from .key_bindings import Binding, KeyBindingsBase + +if TYPE_CHECKING: + from prompt_toolkit.application import Application + from prompt_toolkit.buffer import Buffer + + +__all__ = [ + "KeyProcessor", + "KeyPress", + "KeyPressEvent", +] + + +class KeyPress: + """ + :param key: A `Keys` instance or text (one character). + :param data: The received string on stdin. (Often vt100 escape codes.) + """ + + def __init__(self, key: Keys | str, data: str | None = None) -> None: + assert isinstance(key, Keys) or len(key) == 1 + + if data is None: + if isinstance(key, Keys): + data = key.value + else: + data = key # 'key' is a one character string. + + self.key = key + self.data = data + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(key={self.key!r}, data={self.data!r})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, KeyPress): + return False + return self.key == other.key and self.data == other.data + + +""" +Helper object to indicate flush operation in the KeyProcessor. +NOTE: the implementation is very similar to the VT100 parser. +""" +_Flush = KeyPress("?", data="_Flush") + + +class KeyProcessor: + """ + Statemachine that receives :class:`KeyPress` instances and according to the + key bindings in the given :class:`KeyBindings`, calls the matching handlers. + + :: + + p = KeyProcessor(key_bindings) + + # Send keys into the processor. + p.feed(KeyPress(Keys.ControlX, '\x18')) + p.feed(KeyPress(Keys.ControlC, '\x03') + + # Process all the keys in the queue. + p.process_keys() + + # Now the ControlX-ControlC callback will be called if this sequence is + # registered in the key bindings. + + :param key_bindings: `KeyBindingsBase` instance. + """ + + def __init__(self, key_bindings: KeyBindingsBase) -> None: + self._bindings = key_bindings + + self.before_key_press = Event(self) + self.after_key_press = Event(self) + + self._flush_wait_task: Task[None] | None = None + + self.reset() + + def reset(self) -> None: + self._previous_key_sequence: list[KeyPress] = [] + self._previous_handler: Binding | None = None + + # The queue of keys not yet send to our _process generator/state machine. + self.input_queue: deque[KeyPress] = deque() + + # The key buffer that is matched in the generator state machine. + # (This is at at most the amount of keys that make up for one key binding.) + self.key_buffer: list[KeyPress] = [] + + #: Readline argument (for repetition of commands.) + #: https://www.gnu.org/software/bash/manual/html_node/Readline-Arguments.html + self.arg: str | None = None + + # Start the processor coroutine. + self._process_coroutine = self._process() + self._process_coroutine.send(None) # type: ignore + + def _get_matches(self, key_presses: list[KeyPress]) -> list[Binding]: + """ + For a list of :class:`KeyPress` instances. Give the matching handlers + that would handle this. + """ + keys = tuple(k.key for k in key_presses) + + # Try match, with mode flag + return [b for b in self._bindings.get_bindings_for_keys(keys) if b.filter()] + + def _is_prefix_of_longer_match(self, key_presses: list[KeyPress]) -> bool: + """ + For a list of :class:`KeyPress` instances. Return True if there is any + handler that is bound to a suffix of this keys. + """ + keys = tuple(k.key for k in key_presses) + + # Get the filters for all the key bindings that have a longer match. + # Note that we transform it into a `set`, because we don't care about + # the actual bindings and executing it more than once doesn't make + # sense. (Many key bindings share the same filter.) + filters = { + b.filter for b in self._bindings.get_bindings_starting_with_keys(keys) + } + + # When any key binding is active, return True. + return any(f() for f in filters) + + def _process(self) -> Generator[None, KeyPress, None]: + """ + Coroutine implementing the key match algorithm. Key strokes are sent + into this generator, and it calls the appropriate handlers. + """ + buffer = self.key_buffer + retry = False + + while True: + flush = False + + if retry: + retry = False + else: + key = yield + if key is _Flush: + flush = True + else: + buffer.append(key) + + # If we have some key presses, check for matches. + if buffer: + matches = self._get_matches(buffer) + + if flush: + is_prefix_of_longer_match = False + else: + is_prefix_of_longer_match = self._is_prefix_of_longer_match(buffer) + + # When eager matches were found, give priority to them and also + # ignore all the longer matches. + eager_matches = [m for m in matches if m.eager()] + + if eager_matches: + matches = eager_matches + is_prefix_of_longer_match = False + + # Exact matches found, call handler. + if not is_prefix_of_longer_match and matches: + self._call_handler(matches[-1], key_sequence=buffer[:]) + del buffer[:] # Keep reference. + + # No match found. + elif not is_prefix_of_longer_match and not matches: + retry = True + found = False + + # Loop over the input, try longest match first and shift. + for i in range(len(buffer), 0, -1): + matches = self._get_matches(buffer[:i]) + if matches: + self._call_handler(matches[-1], key_sequence=buffer[:i]) + del buffer[:i] + found = True + break + + if not found: + del buffer[:1] + + def feed(self, key_press: KeyPress, first: bool = False) -> None: + """ + Add a new :class:`KeyPress` to the input queue. + (Don't forget to call `process_keys` in order to process the queue.) + + :param first: If true, insert before everything else. + """ + if first: + self.input_queue.appendleft(key_press) + else: + self.input_queue.append(key_press) + + def feed_multiple(self, key_presses: list[KeyPress], first: bool = False) -> None: + """ + :param first: If true, insert before everything else. + """ + if first: + self.input_queue.extendleft(reversed(key_presses)) + else: + self.input_queue.extend(key_presses) + + def process_keys(self) -> None: + """ + Process all the keys in the `input_queue`. + (To be called after `feed`.) + + Note: because of the `feed`/`process_keys` separation, it is + possible to call `feed` from inside a key binding. + This function keeps looping until the queue is empty. + """ + app = get_app() + + def not_empty() -> bool: + # When the application result is set, stop processing keys. (E.g. + # if ENTER was received, followed by a few additional key strokes, + # leave the other keys in the queue.) + if app.is_done: + # But if there are still CPRResponse keys in the queue, these + # need to be processed. + return any(k for k in self.input_queue if k.key == Keys.CPRResponse) + else: + return bool(self.input_queue) + + def get_next() -> KeyPress: + if app.is_done: + # Only process CPR responses. Everything else is typeahead. + cpr = [k for k in self.input_queue if k.key == Keys.CPRResponse][0] + self.input_queue.remove(cpr) + return cpr + else: + return self.input_queue.popleft() + + is_flush = False + + while not_empty(): + # Process next key. + key_press = get_next() + + is_flush = key_press is _Flush + is_cpr = key_press.key == Keys.CPRResponse + + if not is_flush and not is_cpr: + self.before_key_press.fire() + + try: + self._process_coroutine.send(key_press) + except Exception: + # If for some reason something goes wrong in the parser, (maybe + # an exception was raised) restart the processor for next time. + self.reset() + self.empty_queue() + raise + + if not is_flush and not is_cpr: + self.after_key_press.fire() + + # Skip timeout if the last key was flush. + if not is_flush: + self._start_timeout() + + def empty_queue(self) -> list[KeyPress]: + """ + Empty the input queue. Return the unprocessed input. + """ + key_presses = list(self.input_queue) + self.input_queue.clear() + + # Filter out CPRs. We don't want to return these. + key_presses = [k for k in key_presses if k.key != Keys.CPRResponse] + return key_presses + + def _call_handler(self, handler: Binding, key_sequence: list[KeyPress]) -> None: + app = get_app() + was_recording_emacs = app.emacs_state.is_recording + was_recording_vi = bool(app.vi_state.recording_register) + was_temporary_navigation_mode = app.vi_state.temporary_navigation_mode + arg = self.arg + self.arg = None + + event = KeyPressEvent( + weakref.ref(self), + arg=arg, + key_sequence=key_sequence, + previous_key_sequence=self._previous_key_sequence, + is_repeat=(handler == self._previous_handler), + ) + + # Save the state of the current buffer. + if handler.save_before(event): + event.app.current_buffer.save_to_undo_stack() + + # Call handler. + from prompt_toolkit.buffer import EditReadOnlyBuffer + + try: + handler.call(event) + self._fix_vi_cursor_position(event) + + except EditReadOnlyBuffer: + # When a key binding does an attempt to change a buffer which is + # read-only, we can ignore that. We sound a bell and go on. + app.output.bell() + + if was_temporary_navigation_mode: + self._leave_vi_temp_navigation_mode(event) + + self._previous_key_sequence = key_sequence + self._previous_handler = handler + + # Record the key sequence in our macro. (Only if we're in macro mode + # before and after executing the key.) + if handler.record_in_macro(): + if app.emacs_state.is_recording and was_recording_emacs: + recording = app.emacs_state.current_recording + if recording is not None: # Should always be true, given that + # `was_recording_emacs` is set. + recording.extend(key_sequence) + + if app.vi_state.recording_register and was_recording_vi: + for k in key_sequence: + app.vi_state.current_recording += k.data + + def _fix_vi_cursor_position(self, event: KeyPressEvent) -> None: + """ + After every command, make sure that if we are in Vi navigation mode, we + never put the cursor after the last character of a line. (Unless it's + an empty line.) + """ + app = event.app + buff = app.current_buffer + preferred_column = buff.preferred_column + + if ( + vi_navigation_mode() + and buff.document.is_cursor_at_the_end_of_line + and len(buff.document.current_line) > 0 + ): + buff.cursor_position -= 1 + + # Set the preferred_column for arrow up/down again. + # (This was cleared after changing the cursor position.) + buff.preferred_column = preferred_column + + def _leave_vi_temp_navigation_mode(self, event: KeyPressEvent) -> None: + """ + If we're in Vi temporary navigation (normal) mode, return to + insert/replace mode after executing one action. + """ + app = event.app + + if app.editing_mode == EditingMode.VI: + # Not waiting for a text object and no argument has been given. + if app.vi_state.operator_func is None and self.arg is None: + app.vi_state.temporary_navigation_mode = False + + def _start_timeout(self) -> None: + """ + Start auto flush timeout. Similar to Vim's `timeoutlen` option. + + Start a background coroutine with a timer. When this timeout expires + and no key was pressed in the meantime, we flush all data in the queue + and call the appropriate key binding handlers. + """ + app = get_app() + timeout = app.timeoutlen + + if timeout is None: + return + + async def wait() -> None: + "Wait for timeout." + # This sleep can be cancelled. In that case we don't flush. + await sleep(timeout) + + if len(self.key_buffer) > 0: + # (No keys pressed in the meantime.) + flush_keys() + + def flush_keys() -> None: + "Flush keys." + self.feed(_Flush) + self.process_keys() + + # Automatically flush keys. + if self._flush_wait_task: + self._flush_wait_task.cancel() + self._flush_wait_task = app.create_background_task(wait()) + + def send_sigint(self) -> None: + """ + Send SIGINT. Immediately call the SIGINT key handler. + """ + self.feed(KeyPress(key=Keys.SIGINT), first=True) + self.process_keys() + + +class KeyPressEvent: + """ + Key press event, delivered to key bindings. + + :param key_processor_ref: Weak reference to the `KeyProcessor`. + :param arg: Repetition argument. + :param key_sequence: List of `KeyPress` instances. + :param previouskey_sequence: Previous list of `KeyPress` instances. + :param is_repeat: True when the previous event was delivered to the same handler. + """ + + def __init__( + self, + key_processor_ref: weakref.ReferenceType[KeyProcessor], + arg: str | None, + key_sequence: list[KeyPress], + previous_key_sequence: list[KeyPress], + is_repeat: bool, + ) -> None: + self._key_processor_ref = key_processor_ref + self.key_sequence = key_sequence + self.previous_key_sequence = previous_key_sequence + + #: True when the previous key sequence was handled by the same handler. + self.is_repeat = is_repeat + + self._arg = arg + self._app = get_app() + + def __repr__(self) -> str: + return f"KeyPressEvent(arg={self.arg!r}, key_sequence={self.key_sequence!r}, is_repeat={self.is_repeat!r})" + + @property + def data(self) -> str: + return self.key_sequence[-1].data + + @property + def key_processor(self) -> KeyProcessor: + processor = self._key_processor_ref() + if processor is None: + raise Exception("KeyProcessor was lost. This should not happen.") + return processor + + @property + def app(self) -> Application[Any]: + """ + The current `Application` object. + """ + return self._app + + @property + def current_buffer(self) -> Buffer: + """ + The current buffer. + """ + return self.app.current_buffer + + @property + def arg(self) -> int: + """ + Repetition argument. + """ + if self._arg == "-": + return -1 + + result = int(self._arg or 1) + + # Don't exceed a million. + if int(result) >= 1000000: + result = 1 + + return result + + @property + def arg_present(self) -> bool: + """ + True if repetition argument was explicitly provided. + """ + return self._arg is not None + + def append_to_arg_count(self, data: str) -> None: + """ + Add digit to the input argument. + + :param data: the typed digit as string + """ + assert data in "-0123456789" + current = self._arg + + if data == "-": + assert current is None or current == "-" + result = data + elif current is None: + result = data + else: + result = f"{current}{data}" + + self.key_processor.arg = result + + @property + def cli(self) -> Application[Any]: + "For backward-compatibility." + return self.app diff --git a/lib/prompt_toolkit/key_binding/vi_state.py b/lib/prompt_toolkit/key_binding/vi_state.py new file mode 100644 index 0000000..0543911 --- /dev/null +++ b/lib/prompt_toolkit/key_binding/vi_state.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from enum import Enum +from typing import TYPE_CHECKING, Callable + +from prompt_toolkit.clipboard import ClipboardData + +if TYPE_CHECKING: + from .bindings.vi import TextObject + from .key_processor import KeyPressEvent + +__all__ = [ + "InputMode", + "CharacterFind", + "ViState", +] + + +class InputMode(str, Enum): + value: str + + INSERT = "vi-insert" + INSERT_MULTIPLE = "vi-insert-multiple" + NAVIGATION = "vi-navigation" # Normal mode. + REPLACE = "vi-replace" + REPLACE_SINGLE = "vi-replace-single" + + +class CharacterFind: + def __init__(self, character: str, backwards: bool = False) -> None: + self.character = character + self.backwards = backwards + + +class ViState: + """ + Mutable class to hold the state of the Vi navigation. + """ + + def __init__(self) -> None: + #: None or CharacterFind instance. (This is used to repeat the last + #: search in Vi mode, by pressing the 'n' or 'N' in navigation mode.) + self.last_character_find: CharacterFind | None = None + + # When an operator is given and we are waiting for text object, + # -- e.g. in the case of 'dw', after the 'd' --, an operator callback + # is set here. + self.operator_func: None | (Callable[[KeyPressEvent, TextObject], None]) = None + self.operator_arg: int | None = None + + #: Named registers. Maps register name (e.g. 'a') to + #: :class:`ClipboardData` instances. + self.named_registers: dict[str, ClipboardData] = {} + + #: The Vi mode we're currently in to. + self.__input_mode = InputMode.INSERT + + #: Waiting for digraph. + self.waiting_for_digraph = False + self.digraph_symbol1: str | None = None # (None or a symbol.) + + #: When true, make ~ act as an operator. + self.tilde_operator = False + + #: Register in which we are recording a macro. + #: `None` when not recording anything. + # Note that the recording is only stored in the register after the + # recording is stopped. So we record in a separate `current_recording` + # variable. + self.recording_register: str | None = None + self.current_recording: str = "" + + # Temporary navigation (normal) mode. + # This happens when control-o has been pressed in insert or replace + # mode. The user can now do one navigation action and we'll return back + # to insert/replace. + self.temporary_navigation_mode = False + + @property + def input_mode(self) -> InputMode: + "Get `InputMode`." + return self.__input_mode + + @input_mode.setter + def input_mode(self, value: InputMode) -> None: + "Set `InputMode`." + if value == InputMode.NAVIGATION: + self.waiting_for_digraph = False + self.operator_func = None + self.operator_arg = None + + self.__input_mode = value + + def reset(self) -> None: + """ + Reset state, go back to the given mode. INSERT by default. + """ + # Go back to insert mode. + self.input_mode = InputMode.INSERT + + self.waiting_for_digraph = False + self.operator_func = None + self.operator_arg = None + + # Reset recording state. + self.recording_register = None + self.current_recording = "" diff --git a/lib/prompt_toolkit/keys.py b/lib/prompt_toolkit/keys.py new file mode 100644 index 0000000..ee52aee --- /dev/null +++ b/lib/prompt_toolkit/keys.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "Keys", + "ALL_KEYS", +] + + +class Keys(str, Enum): + """ + List of keys for use in key bindings. + + Note that this is an "StrEnum", all values can be compared against + strings. + """ + + value: str + + Escape = "escape" # Also Control-[ + ShiftEscape = "s-escape" + + ControlAt = "c-@" # Also Control-Space. + + ControlA = "c-a" + ControlB = "c-b" + ControlC = "c-c" + ControlD = "c-d" + ControlE = "c-e" + ControlF = "c-f" + ControlG = "c-g" + ControlH = "c-h" + ControlI = "c-i" # Tab + ControlJ = "c-j" # Newline + ControlK = "c-k" + ControlL = "c-l" + ControlM = "c-m" # Carriage return + ControlN = "c-n" + ControlO = "c-o" + ControlP = "c-p" + ControlQ = "c-q" + ControlR = "c-r" + ControlS = "c-s" + ControlT = "c-t" + ControlU = "c-u" + ControlV = "c-v" + ControlW = "c-w" + ControlX = "c-x" + ControlY = "c-y" + ControlZ = "c-z" + + Control1 = "c-1" + Control2 = "c-2" + Control3 = "c-3" + Control4 = "c-4" + Control5 = "c-5" + Control6 = "c-6" + Control7 = "c-7" + Control8 = "c-8" + Control9 = "c-9" + Control0 = "c-0" + + ControlShift1 = "c-s-1" + ControlShift2 = "c-s-2" + ControlShift3 = "c-s-3" + ControlShift4 = "c-s-4" + ControlShift5 = "c-s-5" + ControlShift6 = "c-s-6" + ControlShift7 = "c-s-7" + ControlShift8 = "c-s-8" + ControlShift9 = "c-s-9" + ControlShift0 = "c-s-0" + + ControlBackslash = "c-\\" + ControlSquareClose = "c-]" + ControlCircumflex = "c-^" + ControlUnderscore = "c-_" + + Left = "left" + Right = "right" + Up = "up" + Down = "down" + Home = "home" + End = "end" + Insert = "insert" + Delete = "delete" + PageUp = "pageup" + PageDown = "pagedown" + + ControlLeft = "c-left" + ControlRight = "c-right" + ControlUp = "c-up" + ControlDown = "c-down" + ControlHome = "c-home" + ControlEnd = "c-end" + ControlInsert = "c-insert" + ControlDelete = "c-delete" + ControlPageUp = "c-pageup" + ControlPageDown = "c-pagedown" + + ShiftLeft = "s-left" + ShiftRight = "s-right" + ShiftUp = "s-up" + ShiftDown = "s-down" + ShiftHome = "s-home" + ShiftEnd = "s-end" + ShiftInsert = "s-insert" + ShiftDelete = "s-delete" + ShiftPageUp = "s-pageup" + ShiftPageDown = "s-pagedown" + + ControlShiftLeft = "c-s-left" + ControlShiftRight = "c-s-right" + ControlShiftUp = "c-s-up" + ControlShiftDown = "c-s-down" + ControlShiftHome = "c-s-home" + ControlShiftEnd = "c-s-end" + ControlShiftInsert = "c-s-insert" + ControlShiftDelete = "c-s-delete" + ControlShiftPageUp = "c-s-pageup" + ControlShiftPageDown = "c-s-pagedown" + + BackTab = "s-tab" # shift + tab + + F1 = "f1" + F2 = "f2" + F3 = "f3" + F4 = "f4" + F5 = "f5" + F6 = "f6" + F7 = "f7" + F8 = "f8" + F9 = "f9" + F10 = "f10" + F11 = "f11" + F12 = "f12" + F13 = "f13" + F14 = "f14" + F15 = "f15" + F16 = "f16" + F17 = "f17" + F18 = "f18" + F19 = "f19" + F20 = "f20" + F21 = "f21" + F22 = "f22" + F23 = "f23" + F24 = "f24" + + ControlF1 = "c-f1" + ControlF2 = "c-f2" + ControlF3 = "c-f3" + ControlF4 = "c-f4" + ControlF5 = "c-f5" + ControlF6 = "c-f6" + ControlF7 = "c-f7" + ControlF8 = "c-f8" + ControlF9 = "c-f9" + ControlF10 = "c-f10" + ControlF11 = "c-f11" + ControlF12 = "c-f12" + ControlF13 = "c-f13" + ControlF14 = "c-f14" + ControlF15 = "c-f15" + ControlF16 = "c-f16" + ControlF17 = "c-f17" + ControlF18 = "c-f18" + ControlF19 = "c-f19" + ControlF20 = "c-f20" + ControlF21 = "c-f21" + ControlF22 = "c-f22" + ControlF23 = "c-f23" + ControlF24 = "c-f24" + + # Matches any key. + Any = "" + + # Special. + ScrollUp = "" + ScrollDown = "" + + CPRResponse = "" + Vt100MouseEvent = "" + WindowsMouseEvent = "" + BracketedPaste = "" + + SIGINT = "" + + # For internal use: key which is ignored. + # (The key binding for this key should not do anything.) + Ignore = "" + + # Some 'Key' aliases (for backwards-compatibility). + ControlSpace = ControlAt + Tab = ControlI + Enter = ControlM + Backspace = ControlH + + # ShiftControl was renamed to ControlShift in + # 888fcb6fa4efea0de8333177e1bbc792f3ff3c24 (20 Feb 2020). + ShiftControlLeft = ControlShiftLeft + ShiftControlRight = ControlShiftRight + ShiftControlHome = ControlShiftHome + ShiftControlEnd = ControlShiftEnd + + +ALL_KEYS: list[str] = [k.value for k in Keys] + + +# Aliases. +KEY_ALIASES: dict[str, str] = { + "backspace": "c-h", + "c-space": "c-@", + "enter": "c-m", + "tab": "c-i", + # ShiftControl was renamed to ControlShift. + "s-c-left": "c-s-left", + "s-c-right": "c-s-right", + "s-c-home": "c-s-home", + "s-c-end": "c-s-end", +} diff --git a/lib/prompt_toolkit/layout/__init__.py b/lib/prompt_toolkit/layout/__init__.py new file mode 100644 index 0000000..7cd0c77 --- /dev/null +++ b/lib/prompt_toolkit/layout/__init__.py @@ -0,0 +1,147 @@ +""" +Command line layout definitions +------------------------------- + +The layout of a command line interface is defined by a Container instance. +There are two main groups of classes here. Containers and controls: + +- A container can contain other containers or controls, it can have multiple + children and it decides about the dimensions. +- A control is responsible for rendering the actual content to a screen. + A control can propose some dimensions, but it's the container who decides + about the dimensions -- or when the control consumes more space -- which part + of the control will be visible. + + +Container classes:: + + - Container (Abstract base class) + |- HSplit (Horizontal split) + |- VSplit (Vertical split) + |- FloatContainer (Container which can also contain menus and other floats) + `- Window (Container which contains one actual control + +Control classes:: + + - UIControl (Abstract base class) + |- FormattedTextControl (Renders formatted text, or a simple list of text fragments) + `- BufferControl (Renders an input buffer.) + + +Usually, you end up wrapping every control inside a `Window` object, because +that's the only way to render it in a layout. + +There are some prepared toolbars which are ready to use:: + +- SystemToolbar (Shows the 'system' input buffer, for entering system commands.) +- ArgToolbar (Shows the input 'arg', for repetition of input commands.) +- SearchToolbar (Shows the 'search' input buffer, for incremental search.) +- CompletionsToolbar (Shows the completions of the current buffer.) +- ValidationToolbar (Shows validation errors of the current buffer.) + +And one prepared menu: + +- CompletionsMenu + +""" + +from __future__ import annotations + +from .containers import ( + AnyContainer, + ColorColumn, + ConditionalContainer, + Container, + DynamicContainer, + Float, + FloatContainer, + HorizontalAlign, + HSplit, + ScrollOffsets, + VerticalAlign, + VSplit, + Window, + WindowAlign, + WindowRenderInfo, + is_container, + to_container, + to_window, +) +from .controls import ( + BufferControl, + DummyControl, + FormattedTextControl, + SearchBufferControl, + UIContent, + UIControl, +) +from .dimension import ( + AnyDimension, + D, + Dimension, + is_dimension, + max_layout_dimensions, + sum_layout_dimensions, + to_dimension, +) +from .layout import InvalidLayoutError, Layout, walk +from .margins import ( + ConditionalMargin, + Margin, + NumberedMargin, + PromptMargin, + ScrollbarMargin, +) +from .menus import CompletionsMenu, MultiColumnCompletionsMenu +from .scrollable_pane import ScrollablePane + +__all__ = [ + # Layout. + "Layout", + "InvalidLayoutError", + "walk", + # Dimensions. + "AnyDimension", + "Dimension", + "D", + "sum_layout_dimensions", + "max_layout_dimensions", + "to_dimension", + "is_dimension", + # Containers. + "AnyContainer", + "Container", + "HorizontalAlign", + "VerticalAlign", + "HSplit", + "VSplit", + "FloatContainer", + "Float", + "WindowAlign", + "Window", + "WindowRenderInfo", + "ConditionalContainer", + "ScrollOffsets", + "ColorColumn", + "to_container", + "to_window", + "is_container", + "DynamicContainer", + "ScrollablePane", + # Controls. + "BufferControl", + "SearchBufferControl", + "DummyControl", + "FormattedTextControl", + "UIControl", + "UIContent", + # Margins. + "Margin", + "NumberedMargin", + "ScrollbarMargin", + "ConditionalMargin", + "PromptMargin", + # Menus. + "CompletionsMenu", + "MultiColumnCompletionsMenu", +] diff --git a/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..0e64bd8 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc new file mode 100644 index 0000000..51ed57b Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc new file mode 100644 index 0000000..86f18c4 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc new file mode 100644 index 0000000..9037b34 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc new file mode 100644 index 0000000..8201e3b Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc new file mode 100644 index 0000000..d496262 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc new file mode 100644 index 0000000..d017d65 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc new file mode 100644 index 0000000..9fe6314 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc new file mode 100644 index 0000000..2d197b0 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc new file mode 100644 index 0000000..9d88ab1 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc new file mode 100644 index 0000000..299f1b7 Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc new file mode 100644 index 0000000..ce8cb8c Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc b/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..ce6cebf Binary files /dev/null and b/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/layout/containers.py b/lib/prompt_toolkit/layout/containers.py new file mode 100644 index 0000000..f6fe381 --- /dev/null +++ b/lib/prompt_toolkit/layout/containers.py @@ -0,0 +1,2766 @@ +""" +Container for the layout. +(Containers can contain other containers or user interface controls.) +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from enum import Enum +from functools import partial +from typing import TYPE_CHECKING, Callable, Sequence, Union, cast + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.cache import SimpleCache +from prompt_toolkit.data_structures import Point +from prompt_toolkit.filters import ( + FilterOrBool, + emacs_insert_mode, + to_filter, + vi_insert_mode, +) +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + to_formatted_text, +) +from prompt_toolkit.formatted_text.utils import ( + fragment_list_to_text, + fragment_list_width, +) +from prompt_toolkit.key_binding import KeyBindingsBase +from prompt_toolkit.mouse_events import MouseEvent, MouseEventType +from prompt_toolkit.utils import get_cwidth, take_using_weights, to_int, to_str + +from .controls import ( + DummyControl, + FormattedTextControl, + GetLinePrefixCallable, + UIContent, + UIControl, +) +from .dimension import ( + AnyDimension, + Dimension, + max_layout_dimensions, + sum_layout_dimensions, + to_dimension, +) +from .margins import Margin +from .mouse_handlers import MouseHandlers +from .screen import _CHAR_CACHE, Screen, WritePosition +from .utils import explode_text_fragments + +if TYPE_CHECKING: + from typing_extensions import Protocol, TypeGuard + + from prompt_toolkit.key_binding.key_bindings import NotImplementedOrNone + + +__all__ = [ + "AnyContainer", + "Container", + "HorizontalAlign", + "VerticalAlign", + "HSplit", + "VSplit", + "FloatContainer", + "Float", + "WindowAlign", + "Window", + "WindowRenderInfo", + "ConditionalContainer", + "ScrollOffsets", + "ColorColumn", + "to_container", + "to_window", + "is_container", + "DynamicContainer", +] + + +class Container(metaclass=ABCMeta): + """ + Base class for user interface layout. + """ + + @abstractmethod + def reset(self) -> None: + """ + Reset the state of this container and all the children. + (E.g. reset scroll offsets, etc...) + """ + + @abstractmethod + def preferred_width(self, max_available_width: int) -> Dimension: + """ + Return a :class:`~prompt_toolkit.layout.Dimension` that represents the + desired width for this container. + """ + + @abstractmethod + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + """ + Return a :class:`~prompt_toolkit.layout.Dimension` that represents the + desired height for this container. + """ + + @abstractmethod + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + """ + Write the actual content to the screen. + + :param screen: :class:`~prompt_toolkit.layout.screen.Screen` + :param mouse_handlers: :class:`~prompt_toolkit.layout.mouse_handlers.MouseHandlers`. + :param parent_style: Style string to pass to the :class:`.Window` + object. This will be applied to all content of the windows. + :class:`.VSplit` and :class:`.HSplit` can use it to pass their + style down to the windows that they contain. + :param z_index: Used for propagating z_index from parent to child. + """ + + def is_modal(self) -> bool: + """ + When this container is modal, key bindings from parent containers are + not taken into account if a user control in this container is focused. + """ + return False + + def get_key_bindings(self) -> KeyBindingsBase | None: + """ + Returns a :class:`.KeyBindings` object. These bindings become active when any + user control in this container has the focus, except if any containers + between this container and the focused user control is modal. + """ + return None + + @abstractmethod + def get_children(self) -> list[Container]: + """ + Return the list of child :class:`.Container` objects. + """ + return [] + + +if TYPE_CHECKING: + + class MagicContainer(Protocol): + """ + Any object that implements ``__pt_container__`` represents a container. + """ + + def __pt_container__(self) -> AnyContainer: ... + + +AnyContainer = Union[Container, "MagicContainer"] + + +def _window_too_small() -> Window: + "Create a `Window` that displays the 'Window too small' text." + return Window( + FormattedTextControl(text=[("class:window-too-small", " Window too small... ")]) + ) + + +class VerticalAlign(Enum): + "Alignment for `HSplit`." + + TOP = "TOP" + CENTER = "CENTER" + BOTTOM = "BOTTOM" + JUSTIFY = "JUSTIFY" + + +class HorizontalAlign(Enum): + "Alignment for `VSplit`." + + LEFT = "LEFT" + CENTER = "CENTER" + RIGHT = "RIGHT" + JUSTIFY = "JUSTIFY" + + +class _Split(Container): + """ + The common parts of `VSplit` and `HSplit`. + """ + + def __init__( + self, + children: Sequence[AnyContainer], + window_too_small: Container | None = None, + padding: AnyDimension = Dimension.exact(0), + padding_char: str | None = None, + padding_style: str = "", + width: AnyDimension = None, + height: AnyDimension = None, + z_index: int | None = None, + modal: bool = False, + key_bindings: KeyBindingsBase | None = None, + style: str | Callable[[], str] = "", + ) -> None: + self.children = [to_container(c) for c in children] + self.window_too_small = window_too_small or _window_too_small() + self.padding = padding + self.padding_char = padding_char + self.padding_style = padding_style + + self.width = width + self.height = height + self.z_index = z_index + + self.modal = modal + self.key_bindings = key_bindings + self.style = style + + def is_modal(self) -> bool: + return self.modal + + def get_key_bindings(self) -> KeyBindingsBase | None: + return self.key_bindings + + def get_children(self) -> list[Container]: + return self.children + + +class HSplit(_Split): + """ + Several layouts, one stacked above/under the other. :: + + +--------------------+ + | | + +--------------------+ + | | + +--------------------+ + + By default, this doesn't display a horizontal line between the children, + but if this is something you need, then create a HSplit as follows:: + + HSplit(children=[ ... ], padding_char='-', + padding=1, padding_style='#ffff00') + + :param children: List of child :class:`.Container` objects. + :param window_too_small: A :class:`.Container` object that is displayed if + there is not enough space for all the children. By default, this is a + "Window too small" message. + :param align: `VerticalAlign` value. + :param width: When given, use this width instead of looking at the children. + :param height: When given, use this height instead of looking at the children. + :param z_index: (int or None) When specified, this can be used to bring + element in front of floating elements. `None` means: inherit from parent. + :param style: A style string. + :param modal: ``True`` or ``False``. + :param key_bindings: ``None`` or a :class:`.KeyBindings` object. + + :param padding: (`Dimension` or int), size to be used for the padding. + :param padding_char: Character to be used for filling in the padding. + :param padding_style: Style to applied to the padding. + """ + + def __init__( + self, + children: Sequence[AnyContainer], + window_too_small: Container | None = None, + align: VerticalAlign = VerticalAlign.JUSTIFY, + padding: AnyDimension = 0, + padding_char: str | None = None, + padding_style: str = "", + width: AnyDimension = None, + height: AnyDimension = None, + z_index: int | None = None, + modal: bool = False, + key_bindings: KeyBindingsBase | None = None, + style: str | Callable[[], str] = "", + ) -> None: + super().__init__( + children=children, + window_too_small=window_too_small, + padding=padding, + padding_char=padding_char, + padding_style=padding_style, + width=width, + height=height, + z_index=z_index, + modal=modal, + key_bindings=key_bindings, + style=style, + ) + + self.align = align + + self._children_cache: SimpleCache[tuple[Container, ...], list[Container]] = ( + SimpleCache(maxsize=1) + ) + self._remaining_space_window = Window() # Dummy window. + + def preferred_width(self, max_available_width: int) -> Dimension: + if self.width is not None: + return to_dimension(self.width) + + if self.children: + dimensions = [c.preferred_width(max_available_width) for c in self.children] + return max_layout_dimensions(dimensions) + else: + return Dimension() + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + if self.height is not None: + return to_dimension(self.height) + + dimensions = [ + c.preferred_height(width, max_available_height) for c in self._all_children + ] + return sum_layout_dimensions(dimensions) + + def reset(self) -> None: + for c in self.children: + c.reset() + + @property + def _all_children(self) -> list[Container]: + """ + List of child objects, including padding. + """ + + def get() -> list[Container]: + result: list[Container] = [] + + # Padding Top. + if self.align in (VerticalAlign.CENTER, VerticalAlign.BOTTOM): + result.append(Window(width=Dimension(preferred=0))) + + # The children with padding. + for child in self.children: + result.append(child) + result.append( + Window( + height=self.padding, + char=self.padding_char, + style=self.padding_style, + ) + ) + if result: + result.pop() + + # Padding right. + if self.align in (VerticalAlign.CENTER, VerticalAlign.TOP): + result.append(Window(width=Dimension(preferred=0))) + + return result + + return self._children_cache.get(tuple(self.children), get) + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + """ + Render the prompt to a `Screen` instance. + + :param screen: The :class:`~prompt_toolkit.layout.screen.Screen` class + to which the output has to be written. + """ + sizes = self._divide_heights(write_position) + style = parent_style + " " + to_str(self.style) + z_index = z_index if self.z_index is None else self.z_index + + if sizes is None: + self.window_too_small.write_to_screen( + screen, mouse_handlers, write_position, style, erase_bg, z_index + ) + else: + # + ypos = write_position.ypos + xpos = write_position.xpos + width = write_position.width + + # Draw child panes. + for s, c in zip(sizes, self._all_children): + c.write_to_screen( + screen, + mouse_handlers, + WritePosition(xpos, ypos, width, s), + style, + erase_bg, + z_index, + ) + ypos += s + + # Fill in the remaining space. This happens when a child control + # refuses to take more space and we don't have any padding. Adding a + # dummy child control for this (in `self._all_children`) is not + # desired, because in some situations, it would take more space, even + # when it's not required. This is required to apply the styling. + remaining_height = write_position.ypos + write_position.height - ypos + if remaining_height > 0: + self._remaining_space_window.write_to_screen( + screen, + mouse_handlers, + WritePosition(xpos, ypos, width, remaining_height), + style, + erase_bg, + z_index, + ) + + def _divide_heights(self, write_position: WritePosition) -> list[int] | None: + """ + Return the heights for all rows. + Or None when there is not enough space. + """ + if not self.children: + return [] + + width = write_position.width + height = write_position.height + + # Calculate heights. + dimensions = [c.preferred_height(width, height) for c in self._all_children] + + # Sum dimensions + sum_dimensions = sum_layout_dimensions(dimensions) + + # If there is not enough space for both. + # Don't do anything. + if sum_dimensions.min > height: + return None + + # Find optimal sizes. (Start with minimal size, increase until we cover + # the whole height.) + sizes = [d.min for d in dimensions] + + child_generator = take_using_weights( + items=list(range(len(dimensions))), weights=[d.weight for d in dimensions] + ) + + i = next(child_generator) + + # Increase until we meet at least the 'preferred' size. + preferred_stop = min(height, sum_dimensions.preferred) + preferred_dimensions = [d.preferred for d in dimensions] + + while sum(sizes) < preferred_stop: + if sizes[i] < preferred_dimensions[i]: + sizes[i] += 1 + i = next(child_generator) + + # Increase until we use all the available space. (or until "max") + if not get_app().is_done: + max_stop = min(height, sum_dimensions.max) + max_dimensions = [d.max for d in dimensions] + + while sum(sizes) < max_stop: + if sizes[i] < max_dimensions[i]: + sizes[i] += 1 + i = next(child_generator) + + return sizes + + +class VSplit(_Split): + """ + Several layouts, one stacked left/right of the other. :: + + +---------+----------+ + | | | + | | | + +---------+----------+ + + By default, this doesn't display a vertical line between the children, but + if this is something you need, then create a HSplit as follows:: + + VSplit(children=[ ... ], padding_char='|', + padding=1, padding_style='#ffff00') + + :param children: List of child :class:`.Container` objects. + :param window_too_small: A :class:`.Container` object that is displayed if + there is not enough space for all the children. By default, this is a + "Window too small" message. + :param align: `HorizontalAlign` value. + :param width: When given, use this width instead of looking at the children. + :param height: When given, use this height instead of looking at the children. + :param z_index: (int or None) When specified, this can be used to bring + element in front of floating elements. `None` means: inherit from parent. + :param style: A style string. + :param modal: ``True`` or ``False``. + :param key_bindings: ``None`` or a :class:`.KeyBindings` object. + + :param padding: (`Dimension` or int), size to be used for the padding. + :param padding_char: Character to be used for filling in the padding. + :param padding_style: Style to applied to the padding. + """ + + def __init__( + self, + children: Sequence[AnyContainer], + window_too_small: Container | None = None, + align: HorizontalAlign = HorizontalAlign.JUSTIFY, + padding: AnyDimension = 0, + padding_char: str | None = None, + padding_style: str = "", + width: AnyDimension = None, + height: AnyDimension = None, + z_index: int | None = None, + modal: bool = False, + key_bindings: KeyBindingsBase | None = None, + style: str | Callable[[], str] = "", + ) -> None: + super().__init__( + children=children, + window_too_small=window_too_small, + padding=padding, + padding_char=padding_char, + padding_style=padding_style, + width=width, + height=height, + z_index=z_index, + modal=modal, + key_bindings=key_bindings, + style=style, + ) + + self.align = align + + self._children_cache: SimpleCache[tuple[Container, ...], list[Container]] = ( + SimpleCache(maxsize=1) + ) + self._remaining_space_window = Window() # Dummy window. + + def preferred_width(self, max_available_width: int) -> Dimension: + if self.width is not None: + return to_dimension(self.width) + + dimensions = [ + c.preferred_width(max_available_width) for c in self._all_children + ] + + return sum_layout_dimensions(dimensions) + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + if self.height is not None: + return to_dimension(self.height) + + # At the point where we want to calculate the heights, the widths have + # already been decided. So we can trust `width` to be the actual + # `width` that's going to be used for the rendering. So, + # `divide_widths` is supposed to use all of the available width. + # Using only the `preferred` width caused a bug where the reported + # height was more than required. (we had a `BufferControl` which did + # wrap lines because of the smaller width returned by `_divide_widths`. + + sizes = self._divide_widths(width) + children = self._all_children + + if sizes is None: + return Dimension() + else: + dimensions = [ + c.preferred_height(s, max_available_height) + for s, c in zip(sizes, children) + ] + return max_layout_dimensions(dimensions) + + def reset(self) -> None: + for c in self.children: + c.reset() + + @property + def _all_children(self) -> list[Container]: + """ + List of child objects, including padding. + """ + + def get() -> list[Container]: + result: list[Container] = [] + + # Padding left. + if self.align in (HorizontalAlign.CENTER, HorizontalAlign.RIGHT): + result.append(Window(width=Dimension(preferred=0))) + + # The children with padding. + for child in self.children: + result.append(child) + result.append( + Window( + width=self.padding, + char=self.padding_char, + style=self.padding_style, + ) + ) + if result: + result.pop() + + # Padding right. + if self.align in (HorizontalAlign.CENTER, HorizontalAlign.LEFT): + result.append(Window(width=Dimension(preferred=0))) + + return result + + return self._children_cache.get(tuple(self.children), get) + + def _divide_widths(self, width: int) -> list[int] | None: + """ + Return the widths for all columns. + Or None when there is not enough space. + """ + children = self._all_children + + if not children: + return [] + + # Calculate widths. + dimensions = [c.preferred_width(width) for c in children] + preferred_dimensions = [d.preferred for d in dimensions] + + # Sum dimensions + sum_dimensions = sum_layout_dimensions(dimensions) + + # If there is not enough space for both. + # Don't do anything. + if sum_dimensions.min > width: + return None + + # Find optimal sizes. (Start with minimal size, increase until we cover + # the whole width.) + sizes = [d.min for d in dimensions] + + child_generator = take_using_weights( + items=list(range(len(dimensions))), weights=[d.weight for d in dimensions] + ) + + i = next(child_generator) + + # Increase until we meet at least the 'preferred' size. + preferred_stop = min(width, sum_dimensions.preferred) + + while sum(sizes) < preferred_stop: + if sizes[i] < preferred_dimensions[i]: + sizes[i] += 1 + i = next(child_generator) + + # Increase until we use all the available space. + max_dimensions = [d.max for d in dimensions] + max_stop = min(width, sum_dimensions.max) + + while sum(sizes) < max_stop: + if sizes[i] < max_dimensions[i]: + sizes[i] += 1 + i = next(child_generator) + + return sizes + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + """ + Render the prompt to a `Screen` instance. + + :param screen: The :class:`~prompt_toolkit.layout.screen.Screen` class + to which the output has to be written. + """ + if not self.children: + return + + children = self._all_children + sizes = self._divide_widths(write_position.width) + style = parent_style + " " + to_str(self.style) + z_index = z_index if self.z_index is None else self.z_index + + # If there is not enough space. + if sizes is None: + self.window_too_small.write_to_screen( + screen, mouse_handlers, write_position, style, erase_bg, z_index + ) + return + + # Calculate heights, take the largest possible, but not larger than + # write_position.height. + heights = [ + child.preferred_height(width, write_position.height).preferred + for width, child in zip(sizes, children) + ] + height = max(write_position.height, min(write_position.height, max(heights))) + + # + ypos = write_position.ypos + xpos = write_position.xpos + + # Draw all child panes. + for s, c in zip(sizes, children): + c.write_to_screen( + screen, + mouse_handlers, + WritePosition(xpos, ypos, s, height), + style, + erase_bg, + z_index, + ) + xpos += s + + # Fill in the remaining space. This happens when a child control + # refuses to take more space and we don't have any padding. Adding a + # dummy child control for this (in `self._all_children`) is not + # desired, because in some situations, it would take more space, even + # when it's not required. This is required to apply the styling. + remaining_width = write_position.xpos + write_position.width - xpos + if remaining_width > 0: + self._remaining_space_window.write_to_screen( + screen, + mouse_handlers, + WritePosition(xpos, ypos, remaining_width, height), + style, + erase_bg, + z_index, + ) + + +class FloatContainer(Container): + """ + Container which can contain another container for the background, as well + as a list of floating containers on top of it. + + Example Usage:: + + FloatContainer(content=Window(...), + floats=[ + Float(xcursor=True, + ycursor=True, + content=CompletionsMenu(...)) + ]) + + :param z_index: (int or None) When specified, this can be used to bring + element in front of floating elements. `None` means: inherit from parent. + This is the z_index for the whole `Float` container as a whole. + """ + + def __init__( + self, + content: AnyContainer, + floats: list[Float], + modal: bool = False, + key_bindings: KeyBindingsBase | None = None, + style: str | Callable[[], str] = "", + z_index: int | None = None, + ) -> None: + self.content = to_container(content) + self.floats = floats + + self.modal = modal + self.key_bindings = key_bindings + self.style = style + self.z_index = z_index + + def reset(self) -> None: + self.content.reset() + + for f in self.floats: + f.content.reset() + + def preferred_width(self, max_available_width: int) -> Dimension: + return self.content.preferred_width(max_available_width) + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + """ + Return the preferred height of the float container. + (We don't care about the height of the floats, they should always fit + into the dimensions provided by the container.) + """ + return self.content.preferred_height(width, max_available_height) + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + style = parent_style + " " + to_str(self.style) + z_index = z_index if self.z_index is None else self.z_index + + self.content.write_to_screen( + screen, mouse_handlers, write_position, style, erase_bg, z_index + ) + + for number, fl in enumerate(self.floats): + # z_index of a Float is computed by summing the z_index of the + # container and the `Float`. + new_z_index = (z_index or 0) + fl.z_index + style = parent_style + " " + to_str(self.style) + + # If the float that we have here, is positioned relative to the + # cursor position, but the Window that specifies the cursor + # position is not drawn yet, because it's a Float itself, we have + # to postpone this calculation. (This is a work-around, but good + # enough for now.) + postpone = fl.xcursor is not None or fl.ycursor is not None + + if postpone: + new_z_index = ( + number + 10**8 + ) # Draw as late as possible, but keep the order. + screen.draw_with_z_index( + z_index=new_z_index, + draw_func=partial( + self._draw_float, + fl, + screen, + mouse_handlers, + write_position, + style, + erase_bg, + new_z_index, + ), + ) + else: + self._draw_float( + fl, + screen, + mouse_handlers, + write_position, + style, + erase_bg, + new_z_index, + ) + + def _draw_float( + self, + fl: Float, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + "Draw a single Float." + # When a menu_position was given, use this instead of the cursor + # position. (These cursor positions are absolute, translate again + # relative to the write_position.) + # Note: This should be inside the for-loop, because one float could + # set the cursor position to be used for the next one. + cpos = screen.get_menu_position( + fl.attach_to_window or get_app().layout.current_window + ) + cursor_position = Point( + x=cpos.x - write_position.xpos, y=cpos.y - write_position.ypos + ) + + fl_width = fl.get_width() + fl_height = fl.get_height() + width: int + height: int + xpos: int + ypos: int + + # Left & width given. + if fl.left is not None and fl_width is not None: + xpos = fl.left + width = fl_width + # Left & right given -> calculate width. + elif fl.left is not None and fl.right is not None: + xpos = fl.left + width = write_position.width - fl.left - fl.right + # Width & right given -> calculate left. + elif fl_width is not None and fl.right is not None: + xpos = write_position.width - fl.right - fl_width + width = fl_width + # Near x position of cursor. + elif fl.xcursor: + if fl_width is None: + width = fl.content.preferred_width(write_position.width).preferred + width = min(write_position.width, width) + else: + width = fl_width + + xpos = cursor_position.x + if xpos + width > write_position.width: + xpos = max(0, write_position.width - width) + # Only width given -> center horizontally. + elif fl_width: + xpos = int((write_position.width - fl_width) / 2) + width = fl_width + # Otherwise, take preferred width from float content. + else: + width = fl.content.preferred_width(write_position.width).preferred + + if fl.left is not None: + xpos = fl.left + elif fl.right is not None: + xpos = max(0, write_position.width - width - fl.right) + else: # Center horizontally. + xpos = max(0, int((write_position.width - width) / 2)) + + # Trim. + width = min(width, write_position.width - xpos) + + # Top & height given. + if fl.top is not None and fl_height is not None: + ypos = fl.top + height = fl_height + # Top & bottom given -> calculate height. + elif fl.top is not None and fl.bottom is not None: + ypos = fl.top + height = write_position.height - fl.top - fl.bottom + # Height & bottom given -> calculate top. + elif fl_height is not None and fl.bottom is not None: + ypos = write_position.height - fl_height - fl.bottom + height = fl_height + # Near cursor. + elif fl.ycursor: + ypos = cursor_position.y + (0 if fl.allow_cover_cursor else 1) + + if fl_height is None: + height = fl.content.preferred_height( + width, write_position.height + ).preferred + else: + height = fl_height + + # Reduce height if not enough space. (We can use the height + # when the content requires it.) + if height > write_position.height - ypos: + if write_position.height - ypos + 1 >= ypos: + # When the space below the cursor is more than + # the space above, just reduce the height. + height = write_position.height - ypos + else: + # Otherwise, fit the float above the cursor. + height = min(height, cursor_position.y) + ypos = cursor_position.y - height + + # Only height given -> center vertically. + elif fl_height: + ypos = int((write_position.height - fl_height) / 2) + height = fl_height + # Otherwise, take preferred height from content. + else: + height = fl.content.preferred_height(width, write_position.height).preferred + + if fl.top is not None: + ypos = fl.top + elif fl.bottom is not None: + ypos = max(0, write_position.height - height - fl.bottom) + else: # Center vertically. + ypos = max(0, int((write_position.height - height) / 2)) + + # Trim. + height = min(height, write_position.height - ypos) + + # Write float. + # (xpos and ypos can be negative: a float can be partially visible.) + if height > 0 and width > 0: + wp = WritePosition( + xpos=xpos + write_position.xpos, + ypos=ypos + write_position.ypos, + width=width, + height=height, + ) + + if not fl.hide_when_covering_content or self._area_is_empty(screen, wp): + fl.content.write_to_screen( + screen, + mouse_handlers, + wp, + style, + erase_bg=not fl.transparent(), + z_index=z_index, + ) + + def _area_is_empty(self, screen: Screen, write_position: WritePosition) -> bool: + """ + Return True when the area below the write position is still empty. + (For floats that should not hide content underneath.) + """ + wp = write_position + + for y in range(wp.ypos, wp.ypos + wp.height): + if y in screen.data_buffer: + row = screen.data_buffer[y] + + for x in range(wp.xpos, wp.xpos + wp.width): + c = row[x] + if c.char != " ": + return False + + return True + + def is_modal(self) -> bool: + return self.modal + + def get_key_bindings(self) -> KeyBindingsBase | None: + return self.key_bindings + + def get_children(self) -> list[Container]: + children = [self.content] + children.extend(f.content for f in self.floats) + return children + + +class Float: + """ + Float for use in a :class:`.FloatContainer`. + Except for the `content` parameter, all other options are optional. + + :param content: :class:`.Container` instance. + + :param width: :class:`.Dimension` or callable which returns a :class:`.Dimension`. + :param height: :class:`.Dimension` or callable which returns a :class:`.Dimension`. + + :param left: Distance to the left edge of the :class:`.FloatContainer`. + :param right: Distance to the right edge of the :class:`.FloatContainer`. + :param top: Distance to the top of the :class:`.FloatContainer`. + :param bottom: Distance to the bottom of the :class:`.FloatContainer`. + + :param attach_to_window: Attach to the cursor from this window, instead of + the current window. + :param hide_when_covering_content: Hide the float when it covers content underneath. + :param allow_cover_cursor: When `False`, make sure to display the float + below the cursor. Not on top of the indicated position. + :param z_index: Z-index position. For a Float, this needs to be at least + one. It is relative to the z_index of the parent container. + :param transparent: :class:`.Filter` indicating whether this float needs to be + drawn transparently. + """ + + def __init__( + self, + content: AnyContainer, + top: int | None = None, + right: int | None = None, + bottom: int | None = None, + left: int | None = None, + width: int | Callable[[], int] | None = None, + height: int | Callable[[], int] | None = None, + xcursor: bool = False, + ycursor: bool = False, + attach_to_window: AnyContainer | None = None, + hide_when_covering_content: bool = False, + allow_cover_cursor: bool = False, + z_index: int = 1, + transparent: bool = False, + ) -> None: + assert z_index >= 1 + + self.left = left + self.right = right + self.top = top + self.bottom = bottom + + self.width = width + self.height = height + + self.xcursor = xcursor + self.ycursor = ycursor + + self.attach_to_window = ( + to_window(attach_to_window) if attach_to_window else None + ) + + self.content = to_container(content) + self.hide_when_covering_content = hide_when_covering_content + self.allow_cover_cursor = allow_cover_cursor + self.z_index = z_index + self.transparent = to_filter(transparent) + + def get_width(self) -> int | None: + if callable(self.width): + return self.width() + return self.width + + def get_height(self) -> int | None: + if callable(self.height): + return self.height() + return self.height + + def __repr__(self) -> str: + return f"Float(content={self.content!r})" + + +class WindowRenderInfo: + """ + Render information for the last render time of this control. + It stores mapping information between the input buffers (in case of a + :class:`~prompt_toolkit.layout.controls.BufferControl`) and the actual + render position on the output screen. + + (Could be used for implementation of the Vi 'H' and 'L' key bindings as + well as implementing mouse support.) + + :param ui_content: The original :class:`.UIContent` instance that contains + the whole input, without clipping. (ui_content) + :param horizontal_scroll: The horizontal scroll of the :class:`.Window` instance. + :param vertical_scroll: The vertical scroll of the :class:`.Window` instance. + :param window_width: The width of the window that displays the content, + without the margins. + :param window_height: The height of the window that displays the content. + :param configured_scroll_offsets: The scroll offsets as configured for the + :class:`Window` instance. + :param visible_line_to_row_col: Mapping that maps the row numbers on the + displayed screen (starting from zero for the first visible line) to + (row, col) tuples pointing to the row and column of the :class:`.UIContent`. + :param rowcol_to_yx: Mapping that maps (row, column) tuples representing + coordinates of the :class:`UIContent` to (y, x) absolute coordinates at + the rendered screen. + """ + + def __init__( + self, + window: Window, + ui_content: UIContent, + horizontal_scroll: int, + vertical_scroll: int, + window_width: int, + window_height: int, + configured_scroll_offsets: ScrollOffsets, + visible_line_to_row_col: dict[int, tuple[int, int]], + rowcol_to_yx: dict[tuple[int, int], tuple[int, int]], + x_offset: int, + y_offset: int, + wrap_lines: bool, + ) -> None: + self.window = window + self.ui_content = ui_content + self.vertical_scroll = vertical_scroll + self.window_width = window_width # Width without margins. + self.window_height = window_height + + self.configured_scroll_offsets = configured_scroll_offsets + self.visible_line_to_row_col = visible_line_to_row_col + self.wrap_lines = wrap_lines + + self._rowcol_to_yx = rowcol_to_yx # row/col from input to absolute y/x + # screen coordinates. + self._x_offset = x_offset + self._y_offset = y_offset + + @property + def visible_line_to_input_line(self) -> dict[int, int]: + return { + visible_line: rowcol[0] + for visible_line, rowcol in self.visible_line_to_row_col.items() + } + + @property + def cursor_position(self) -> Point: + """ + Return the cursor position coordinates, relative to the left/top corner + of the rendered screen. + """ + cpos = self.ui_content.cursor_position + try: + y, x = self._rowcol_to_yx[cpos.y, cpos.x] + except KeyError: + # For `DummyControl` for instance, the content can be empty, and so + # will `_rowcol_to_yx` be. Return 0/0 by default. + return Point(x=0, y=0) + else: + return Point(x=x - self._x_offset, y=y - self._y_offset) + + @property + def applied_scroll_offsets(self) -> ScrollOffsets: + """ + Return a :class:`.ScrollOffsets` instance that indicates the actual + offset. This can be less than or equal to what's configured. E.g, when + the cursor is completely at the top, the top offset will be zero rather + than what's configured. + """ + if self.displayed_lines[0] == 0: + top = 0 + else: + # Get row where the cursor is displayed. + y = self.input_line_to_visible_line[self.ui_content.cursor_position.y] + top = min(y, self.configured_scroll_offsets.top) + + return ScrollOffsets( + top=top, + bottom=min( + self.ui_content.line_count - self.displayed_lines[-1] - 1, + self.configured_scroll_offsets.bottom, + ), + # For left/right, it probably doesn't make sense to return something. + # (We would have to calculate the widths of all the lines and keep + # double width characters in mind.) + left=0, + right=0, + ) + + @property + def displayed_lines(self) -> list[int]: + """ + List of all the visible rows. (Line numbers of the input buffer.) + The last line may not be entirely visible. + """ + return sorted(row for row, col in self.visible_line_to_row_col.values()) + + @property + def input_line_to_visible_line(self) -> dict[int, int]: + """ + Return the dictionary mapping the line numbers of the input buffer to + the lines of the screen. When a line spans several rows at the screen, + the first row appears in the dictionary. + """ + result: dict[int, int] = {} + for k, v in self.visible_line_to_input_line.items(): + if v in result: + result[v] = min(result[v], k) + else: + result[v] = k + return result + + def first_visible_line(self, after_scroll_offset: bool = False) -> int: + """ + Return the line number (0 based) of the input document that corresponds + with the first visible line. + """ + if after_scroll_offset: + return self.displayed_lines[self.applied_scroll_offsets.top] + else: + return self.displayed_lines[0] + + def last_visible_line(self, before_scroll_offset: bool = False) -> int: + """ + Like `first_visible_line`, but for the last visible line. + """ + if before_scroll_offset: + return self.displayed_lines[-1 - self.applied_scroll_offsets.bottom] + else: + return self.displayed_lines[-1] + + def center_visible_line( + self, before_scroll_offset: bool = False, after_scroll_offset: bool = False + ) -> int: + """ + Like `first_visible_line`, but for the center visible line. + """ + return ( + self.first_visible_line(after_scroll_offset) + + ( + self.last_visible_line(before_scroll_offset) + - self.first_visible_line(after_scroll_offset) + ) + // 2 + ) + + @property + def content_height(self) -> int: + """ + The full height of the user control. + """ + return self.ui_content.line_count + + @property + def full_height_visible(self) -> bool: + """ + True when the full height is visible (There is no vertical scroll.) + """ + return ( + self.vertical_scroll == 0 + and self.last_visible_line() == self.content_height + ) + + @property + def top_visible(self) -> bool: + """ + True when the top of the buffer is visible. + """ + return self.vertical_scroll == 0 + + @property + def bottom_visible(self) -> bool: + """ + True when the bottom of the buffer is visible. + """ + return self.last_visible_line() == self.content_height - 1 + + @property + def vertical_scroll_percentage(self) -> int: + """ + Vertical scroll as a percentage. (0 means: the top is visible, + 100 means: the bottom is visible.) + """ + if self.bottom_visible: + return 100 + else: + return 100 * self.vertical_scroll // self.content_height + + def get_height_for_line(self, lineno: int) -> int: + """ + Return the height of the given line. + (The height that it would take, if this line became visible.) + """ + if self.wrap_lines: + return self.ui_content.get_height_for_line( + lineno, self.window_width, self.window.get_line_prefix + ) + else: + return 1 + + +class ScrollOffsets: + """ + Scroll offsets for the :class:`.Window` class. + + Note that left/right offsets only make sense if line wrapping is disabled. + """ + + def __init__( + self, + top: int | Callable[[], int] = 0, + bottom: int | Callable[[], int] = 0, + left: int | Callable[[], int] = 0, + right: int | Callable[[], int] = 0, + ) -> None: + self._top = top + self._bottom = bottom + self._left = left + self._right = right + + @property + def top(self) -> int: + return to_int(self._top) + + @property + def bottom(self) -> int: + return to_int(self._bottom) + + @property + def left(self) -> int: + return to_int(self._left) + + @property + def right(self) -> int: + return to_int(self._right) + + def __repr__(self) -> str: + return f"ScrollOffsets(top={self._top!r}, bottom={self._bottom!r}, left={self._left!r}, right={self._right!r})" + + +class ColorColumn: + """ + Column for a :class:`.Window` to be colored. + """ + + def __init__(self, position: int, style: str = "class:color-column") -> None: + self.position = position + self.style = style + + +_in_insert_mode = vi_insert_mode | emacs_insert_mode + + +class WindowAlign(Enum): + """ + Alignment of the Window content. + + Note that this is different from `HorizontalAlign` and `VerticalAlign`, + which are used for the alignment of the child containers in respectively + `VSplit` and `HSplit`. + """ + + LEFT = "LEFT" + RIGHT = "RIGHT" + CENTER = "CENTER" + + +class Window(Container): + """ + Container that holds a control. + + :param content: :class:`.UIControl` instance. + :param width: :class:`.Dimension` instance or callable. + :param height: :class:`.Dimension` instance or callable. + :param z_index: When specified, this can be used to bring element in front + of floating elements. + :param dont_extend_width: When `True`, don't take up more width then the + preferred width reported by the control. + :param dont_extend_height: When `True`, don't take up more width then the + preferred height reported by the control. + :param ignore_content_width: A `bool` or :class:`.Filter` instance. Ignore + the :class:`.UIContent` width when calculating the dimensions. + :param ignore_content_height: A `bool` or :class:`.Filter` instance. Ignore + the :class:`.UIContent` height when calculating the dimensions. + :param left_margins: A list of :class:`.Margin` instance to be displayed on + the left. For instance: :class:`~prompt_toolkit.layout.NumberedMargin` + can be one of them in order to show line numbers. + :param right_margins: Like `left_margins`, but on the other side. + :param scroll_offsets: :class:`.ScrollOffsets` instance, representing the + preferred amount of lines/columns to be always visible before/after the + cursor. When both top and bottom are a very high number, the cursor + will be centered vertically most of the time. + :param allow_scroll_beyond_bottom: A `bool` or + :class:`.Filter` instance. When True, allow scrolling so far, that the + top part of the content is not visible anymore, while there is still + empty space available at the bottom of the window. In the Vi editor for + instance, this is possible. You will see tildes while the top part of + the body is hidden. + :param wrap_lines: A `bool` or :class:`.Filter` instance. When True, don't + scroll horizontally, but wrap lines instead. + :param get_vertical_scroll: Callable that takes this window + instance as input and returns a preferred vertical scroll. + (When this is `None`, the scroll is only determined by the last and + current cursor position.) + :param get_horizontal_scroll: Callable that takes this window + instance as input and returns a preferred vertical scroll. + :param always_hide_cursor: A `bool` or + :class:`.Filter` instance. When True, never display the cursor, even + when the user control specifies a cursor position. + :param cursorline: A `bool` or :class:`.Filter` instance. When True, + display a cursorline. + :param cursorcolumn: A `bool` or :class:`.Filter` instance. When True, + display a cursorcolumn. + :param colorcolumns: A list of :class:`.ColorColumn` instances that + describe the columns to be highlighted, or a callable that returns such + a list. + :param align: :class:`.WindowAlign` value or callable that returns an + :class:`.WindowAlign` value. alignment of content. + :param style: A style string. Style to be applied to all the cells in this + window. (This can be a callable that returns a string.) + :param char: (string) Character to be used for filling the background. This + can also be a callable that returns a character. + :param get_line_prefix: None or a callable that returns formatted text to + be inserted before a line. It takes a line number (int) and a + wrap_count and returns formatted text. This can be used for + implementation of line continuations, things like Vim "breakindent" and + so on. + """ + + def __init__( + self, + content: UIControl | None = None, + width: AnyDimension = None, + height: AnyDimension = None, + z_index: int | None = None, + dont_extend_width: FilterOrBool = False, + dont_extend_height: FilterOrBool = False, + ignore_content_width: FilterOrBool = False, + ignore_content_height: FilterOrBool = False, + left_margins: Sequence[Margin] | None = None, + right_margins: Sequence[Margin] | None = None, + scroll_offsets: ScrollOffsets | None = None, + allow_scroll_beyond_bottom: FilterOrBool = False, + wrap_lines: FilterOrBool = False, + get_vertical_scroll: Callable[[Window], int] | None = None, + get_horizontal_scroll: Callable[[Window], int] | None = None, + always_hide_cursor: FilterOrBool = False, + cursorline: FilterOrBool = False, + cursorcolumn: FilterOrBool = False, + colorcolumns: ( + None | list[ColorColumn] | Callable[[], list[ColorColumn]] + ) = None, + align: WindowAlign | Callable[[], WindowAlign] = WindowAlign.LEFT, + style: str | Callable[[], str] = "", + char: None | str | Callable[[], str] = None, + get_line_prefix: GetLinePrefixCallable | None = None, + ) -> None: + self.allow_scroll_beyond_bottom = to_filter(allow_scroll_beyond_bottom) + self.always_hide_cursor = to_filter(always_hide_cursor) + self.wrap_lines = to_filter(wrap_lines) + self.cursorline = to_filter(cursorline) + self.cursorcolumn = to_filter(cursorcolumn) + + self.content = content or DummyControl() + self.dont_extend_width = to_filter(dont_extend_width) + self.dont_extend_height = to_filter(dont_extend_height) + self.ignore_content_width = to_filter(ignore_content_width) + self.ignore_content_height = to_filter(ignore_content_height) + self.left_margins = left_margins or [] + self.right_margins = right_margins or [] + self.scroll_offsets = scroll_offsets or ScrollOffsets() + self.get_vertical_scroll = get_vertical_scroll + self.get_horizontal_scroll = get_horizontal_scroll + self.colorcolumns = colorcolumns or [] + self.align = align + self.style = style + self.char = char + self.get_line_prefix = get_line_prefix + + self.width = width + self.height = height + self.z_index = z_index + + # Cache for the screens generated by the margin. + self._ui_content_cache: SimpleCache[tuple[int, int, int], UIContent] = ( + SimpleCache(maxsize=8) + ) + self._margin_width_cache: SimpleCache[tuple[Margin, int], int] = SimpleCache( + maxsize=1 + ) + + self.reset() + + def __repr__(self) -> str: + return f"Window(content={self.content!r})" + + def reset(self) -> None: + self.content.reset() + + #: Scrolling position of the main content. + self.vertical_scroll = 0 + self.horizontal_scroll = 0 + + # Vertical scroll 2: this is the vertical offset that a line is + # scrolled if a single line (the one that contains the cursor) consumes + # all of the vertical space. + self.vertical_scroll_2 = 0 + + #: Keep render information (mappings between buffer input and render + #: output.) + self.render_info: WindowRenderInfo | None = None + + def _get_margin_width(self, margin: Margin) -> int: + """ + Return the width for this margin. + (Calculate only once per render time.) + """ + + # Margin.get_width, needs to have a UIContent instance. + def get_ui_content() -> UIContent: + return self._get_ui_content(width=0, height=0) + + def get_width() -> int: + return margin.get_width(get_ui_content) + + key = (margin, get_app().render_counter) + return self._margin_width_cache.get(key, get_width) + + def _get_total_margin_width(self) -> int: + """ + Calculate and return the width of the margin (left + right). + """ + return sum(self._get_margin_width(m) for m in self.left_margins) + sum( + self._get_margin_width(m) for m in self.right_margins + ) + + def preferred_width(self, max_available_width: int) -> Dimension: + """ + Calculate the preferred width for this window. + """ + + def preferred_content_width() -> int | None: + """Content width: is only calculated if no exact width for the + window was given.""" + if self.ignore_content_width(): + return None + + # Calculate the width of the margin. + total_margin_width = self._get_total_margin_width() + + # Window of the content. (Can be `None`.) + preferred_width = self.content.preferred_width( + max_available_width - total_margin_width + ) + + if preferred_width is not None: + # Include width of the margins. + preferred_width += total_margin_width + return preferred_width + + # Merge. + return self._merge_dimensions( + dimension=to_dimension(self.width), + get_preferred=preferred_content_width, + dont_extend=self.dont_extend_width(), + ) + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + """ + Calculate the preferred height for this window. + """ + + def preferred_content_height() -> int | None: + """Content height: is only calculated if no exact height for the + window was given.""" + if self.ignore_content_height(): + return None + + total_margin_width = self._get_total_margin_width() + wrap_lines = self.wrap_lines() + + return self.content.preferred_height( + width - total_margin_width, + max_available_height, + wrap_lines, + self.get_line_prefix, + ) + + return self._merge_dimensions( + dimension=to_dimension(self.height), + get_preferred=preferred_content_height, + dont_extend=self.dont_extend_height(), + ) + + @staticmethod + def _merge_dimensions( + dimension: Dimension | None, + get_preferred: Callable[[], int | None], + dont_extend: bool = False, + ) -> Dimension: + """ + Take the Dimension from this `Window` class and the received preferred + size from the `UIControl` and return a `Dimension` to report to the + parent container. + """ + dimension = dimension or Dimension() + + # When a preferred dimension was explicitly given to the Window, + # ignore the UIControl. + preferred: int | None + + if dimension.preferred_specified: + preferred = dimension.preferred + else: + # Otherwise, calculate the preferred dimension from the UI control + # content. + preferred = get_preferred() + + # When a 'preferred' dimension is given by the UIControl, make sure + # that it stays within the bounds of the Window. + if preferred is not None: + if dimension.max_specified: + preferred = min(preferred, dimension.max) + + if dimension.min_specified: + preferred = max(preferred, dimension.min) + + # When a `dont_extend` flag has been given, use the preferred dimension + # also as the max dimension. + max_: int | None + min_: int | None + + if dont_extend and preferred is not None: + max_ = min(dimension.max, preferred) + else: + max_ = dimension.max if dimension.max_specified else None + + min_ = dimension.min if dimension.min_specified else None + + return Dimension( + min=min_, max=max_, preferred=preferred, weight=dimension.weight + ) + + def _get_ui_content(self, width: int, height: int) -> UIContent: + """ + Create a `UIContent` instance. + """ + + def get_content() -> UIContent: + return self.content.create_content(width=width, height=height) + + key = (get_app().render_counter, width, height) + return self._ui_content_cache.get(key, get_content) + + def _get_digraph_char(self) -> str | None: + "Return `False`, or the Digraph symbol to be used." + app = get_app() + if app.quoted_insert: + return "^" + if app.vi_state.waiting_for_digraph: + if app.vi_state.digraph_symbol1: + return app.vi_state.digraph_symbol1 + return "?" + return None + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + """ + Write window to screen. This renders the user control, the margins and + copies everything over to the absolute position at the given screen. + """ + # If dont_extend_width/height was given. Then reduce width/height in + # WritePosition if the parent wanted us to paint in a bigger area. + # (This happens if this window is bundled with another window in a + # HSplit/VSplit, but with different size requirements.) + write_position = WritePosition( + xpos=write_position.xpos, + ypos=write_position.ypos, + width=write_position.width, + height=write_position.height, + ) + + if self.dont_extend_width(): + write_position.width = min( + write_position.width, + self.preferred_width(write_position.width).preferred, + ) + + if self.dont_extend_height(): + write_position.height = min( + write_position.height, + self.preferred_height( + write_position.width, write_position.height + ).preferred, + ) + + # Draw + z_index = z_index if self.z_index is None else self.z_index + + draw_func = partial( + self._write_to_screen_at_index, + screen, + mouse_handlers, + write_position, + parent_style, + erase_bg, + ) + + if z_index is None or z_index <= 0: + # When no z_index is given, draw right away. + draw_func() + else: + # Otherwise, postpone. + screen.draw_with_z_index(z_index=z_index, draw_func=draw_func) + + def _write_to_screen_at_index( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + ) -> None: + # Don't bother writing invisible windows. + # (We save some time, but also avoid applying last-line styling.) + if write_position.height <= 0 or write_position.width <= 0: + return + + # Calculate margin sizes. + left_margin_widths = [self._get_margin_width(m) for m in self.left_margins] + right_margin_widths = [self._get_margin_width(m) for m in self.right_margins] + total_margin_width = sum(left_margin_widths + right_margin_widths) + + # Render UserControl. + ui_content = self.content.create_content( + write_position.width - total_margin_width, write_position.height + ) + assert isinstance(ui_content, UIContent) + + # Scroll content. + wrap_lines = self.wrap_lines() + self._scroll( + ui_content, write_position.width - total_margin_width, write_position.height + ) + + # Erase background and fill with `char`. + self._fill_bg(screen, write_position, erase_bg) + + # Resolve `align` attribute. + align = self.align() if callable(self.align) else self.align + + # Write body + visible_line_to_row_col, rowcol_to_yx = self._copy_body( + ui_content, + screen, + write_position, + sum(left_margin_widths), + write_position.width - total_margin_width, + self.vertical_scroll, + self.horizontal_scroll, + wrap_lines=wrap_lines, + highlight_lines=True, + vertical_scroll_2=self.vertical_scroll_2, + always_hide_cursor=self.always_hide_cursor(), + has_focus=get_app().layout.current_control == self.content, + align=align, + get_line_prefix=self.get_line_prefix, + ) + + # Remember render info. (Set before generating the margins. They need this.) + x_offset = write_position.xpos + sum(left_margin_widths) + y_offset = write_position.ypos + + render_info = WindowRenderInfo( + window=self, + ui_content=ui_content, + horizontal_scroll=self.horizontal_scroll, + vertical_scroll=self.vertical_scroll, + window_width=write_position.width - total_margin_width, + window_height=write_position.height, + configured_scroll_offsets=self.scroll_offsets, + visible_line_to_row_col=visible_line_to_row_col, + rowcol_to_yx=rowcol_to_yx, + x_offset=x_offset, + y_offset=y_offset, + wrap_lines=wrap_lines, + ) + self.render_info = render_info + + # Set mouse handlers. + def mouse_handler(mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Wrapper around the mouse_handler of the `UIControl` that turns + screen coordinates into line coordinates. + Returns `NotImplemented` if no UI invalidation should be done. + """ + # Don't handle mouse events outside of the current modal part of + # the UI. + if self not in get_app().layout.walk_through_modal_area(): + return NotImplemented + + # Find row/col position first. + yx_to_rowcol = {v: k for k, v in rowcol_to_yx.items()} + y = mouse_event.position.y + x = mouse_event.position.x + + # If clicked below the content area, look for a position in the + # last line instead. + max_y = write_position.ypos + len(visible_line_to_row_col) - 1 + y = min(max_y, y) + result: NotImplementedOrNone + + while x >= 0: + try: + row, col = yx_to_rowcol[y, x] + except KeyError: + # Try again. (When clicking on the right side of double + # width characters, or on the right side of the input.) + x -= 1 + else: + # Found position, call handler of UIControl. + result = self.content.mouse_handler( + MouseEvent( + position=Point(x=col, y=row), + event_type=mouse_event.event_type, + button=mouse_event.button, + modifiers=mouse_event.modifiers, + ) + ) + break + else: + # nobreak. + # (No x/y coordinate found for the content. This happens in + # case of a DummyControl, that does not have any content. + # Report (0,0) instead.) + result = self.content.mouse_handler( + MouseEvent( + position=Point(x=0, y=0), + event_type=mouse_event.event_type, + button=mouse_event.button, + modifiers=mouse_event.modifiers, + ) + ) + + # If it returns NotImplemented, handle it here. + if result == NotImplemented: + result = self._mouse_handler(mouse_event) + + return result + + mouse_handlers.set_mouse_handler_for_range( + x_min=write_position.xpos + sum(left_margin_widths), + x_max=write_position.xpos + write_position.width - total_margin_width, + y_min=write_position.ypos, + y_max=write_position.ypos + write_position.height, + handler=mouse_handler, + ) + + # Render and copy margins. + move_x = 0 + + def render_margin(m: Margin, width: int) -> UIContent: + "Render margin. Return `Screen`." + # Retrieve margin fragments. + fragments = m.create_margin(render_info, width, write_position.height) + + # Turn it into a UIContent object. + # already rendered those fragments using this size.) + return FormattedTextControl(fragments).create_content( + width + 1, write_position.height + ) + + for m, width in zip(self.left_margins, left_margin_widths): + if width > 0: # (ConditionalMargin returns a zero width. -- Don't render.) + # Create screen for margin. + margin_content = render_margin(m, width) + + # Copy and shift X. + self._copy_margin(margin_content, screen, write_position, move_x, width) + move_x += width + + move_x = write_position.width - sum(right_margin_widths) + + for m, width in zip(self.right_margins, right_margin_widths): + # Create screen for margin. + margin_content = render_margin(m, width) + + # Copy and shift X. + self._copy_margin(margin_content, screen, write_position, move_x, width) + move_x += width + + # Apply 'self.style' + self._apply_style(screen, write_position, parent_style) + + # Tell the screen that this user control has been painted at this + # position. + screen.visible_windows_to_write_positions[self] = write_position + + def _copy_body( + self, + ui_content: UIContent, + new_screen: Screen, + write_position: WritePosition, + move_x: int, + width: int, + vertical_scroll: int = 0, + horizontal_scroll: int = 0, + wrap_lines: bool = False, + highlight_lines: bool = False, + vertical_scroll_2: int = 0, + always_hide_cursor: bool = False, + has_focus: bool = False, + align: WindowAlign = WindowAlign.LEFT, + get_line_prefix: Callable[[int, int], AnyFormattedText] | None = None, + ) -> tuple[dict[int, tuple[int, int]], dict[tuple[int, int], tuple[int, int]]]: + """ + Copy the UIContent into the output screen. + Return (visible_line_to_row_col, rowcol_to_yx) tuple. + + :param get_line_prefix: None or a callable that takes a line number + (int) and a wrap_count (int) and returns formatted text. + """ + xpos = write_position.xpos + move_x + ypos = write_position.ypos + line_count = ui_content.line_count + new_buffer = new_screen.data_buffer + empty_char = _CHAR_CACHE["", ""] + + # Map visible line number to (row, col) of input. + # 'col' will always be zero if line wrapping is off. + visible_line_to_row_col: dict[int, tuple[int, int]] = {} + + # Maps (row, col) from the input to (y, x) screen coordinates. + rowcol_to_yx: dict[tuple[int, int], tuple[int, int]] = {} + + def copy_line( + line: StyleAndTextTuples, + lineno: int, + x: int, + y: int, + is_input: bool = False, + ) -> tuple[int, int]: + """ + Copy over a single line to the output screen. This can wrap over + multiple lines in the output. It will call the prefix (prompt) + function before every line. + """ + if is_input: + current_rowcol_to_yx = rowcol_to_yx + else: + current_rowcol_to_yx = {} # Throwaway dictionary. + + # Draw line prefix. + if is_input and get_line_prefix: + prompt = to_formatted_text(get_line_prefix(lineno, 0)) + x, y = copy_line(prompt, lineno, x, y, is_input=False) + + # Scroll horizontally. + skipped = 0 # Characters skipped because of horizontal scrolling. + if horizontal_scroll and is_input: + h_scroll = horizontal_scroll + line = explode_text_fragments(line) + while h_scroll > 0 and line: + h_scroll -= get_cwidth(line[0][1]) + skipped += 1 + del line[:1] # Remove first character. + + x -= h_scroll # When scrolling over double width character, + # this can end up being negative. + + # Align this line. (Note that this doesn't work well when we use + # get_line_prefix and that function returns variable width prefixes.) + if align == WindowAlign.CENTER: + line_width = fragment_list_width(line) + if line_width < width: + x += (width - line_width) // 2 + elif align == WindowAlign.RIGHT: + line_width = fragment_list_width(line) + if line_width < width: + x += width - line_width + + col = 0 + wrap_count = 0 + for style, text, *_ in line: + new_buffer_row = new_buffer[y + ypos] + + # Remember raw VT escape sequences. (E.g. FinalTerm's + # escape sequences.) + if "[ZeroWidthEscape]" in style: + new_screen.zero_width_escapes[y + ypos][x + xpos] += text + continue + + for c in text: + char = _CHAR_CACHE[c, style] + char_width = char.width + + # Wrap when the line width is exceeded. + if wrap_lines and x + char_width > width: + visible_line_to_row_col[y + 1] = ( + lineno, + visible_line_to_row_col[y][1] + x, + ) + y += 1 + wrap_count += 1 + x = 0 + + # Insert line prefix (continuation prompt). + if is_input and get_line_prefix: + prompt = to_formatted_text( + get_line_prefix(lineno, wrap_count) + ) + x, y = copy_line(prompt, lineno, x, y, is_input=False) + + new_buffer_row = new_buffer[y + ypos] + + if y >= write_position.height: + return x, y # Break out of all for loops. + + # Set character in screen and shift 'x'. + if x >= 0 and y >= 0 and x < width: + new_buffer_row[x + xpos] = char + + # When we print a multi width character, make sure + # to erase the neighbors positions in the screen. + # (The empty string if different from everything, + # so next redraw this cell will repaint anyway.) + if char_width > 1: + for i in range(1, char_width): + new_buffer_row[x + xpos + i] = empty_char + + # If this is a zero width characters, then it's + # probably part of a decomposed unicode character. + # See: https://en.wikipedia.org/wiki/Unicode_equivalence + # Merge it in the previous cell. + elif char_width == 0: + # Handle all character widths. If the previous + # character is a multiwidth character, then + # merge it two positions back. + for pw in [2, 1]: # Previous character width. + if ( + x - pw >= 0 + and new_buffer_row[x + xpos - pw].width == pw + ): + prev_char = new_buffer_row[x + xpos - pw] + char2 = _CHAR_CACHE[ + prev_char.char + c, prev_char.style + ] + new_buffer_row[x + xpos - pw] = char2 + + # Keep track of write position for each character. + current_rowcol_to_yx[lineno, col + skipped] = ( + y + ypos, + x + xpos, + ) + + col += 1 + x += char_width + return x, y + + # Copy content. + def copy() -> int: + y = -vertical_scroll_2 + lineno = vertical_scroll + + while y < write_position.height and lineno < line_count: + # Take the next line and copy it in the real screen. + line = ui_content.get_line(lineno) + + visible_line_to_row_col[y] = (lineno, horizontal_scroll) + + # Copy margin and actual line. + x = 0 + x, y = copy_line(line, lineno, x, y, is_input=True) + + lineno += 1 + y += 1 + return y + + copy() + + def cursor_pos_to_screen_pos(row: int, col: int) -> Point: + "Translate row/col from UIContent to real Screen coordinates." + try: + y, x = rowcol_to_yx[row, col] + except KeyError: + # Normally this should never happen. (It is a bug, if it happens.) + # But to be sure, return (0, 0) + return Point(x=0, y=0) + + # raise ValueError( + # 'Invalid position. row=%r col=%r, vertical_scroll=%r, ' + # 'horizontal_scroll=%r, height=%r' % + # (row, col, vertical_scroll, horizontal_scroll, write_position.height)) + else: + return Point(x=x, y=y) + + # Set cursor and menu positions. + if ui_content.cursor_position: + screen_cursor_position = cursor_pos_to_screen_pos( + ui_content.cursor_position.y, ui_content.cursor_position.x + ) + + if has_focus: + new_screen.set_cursor_position(self, screen_cursor_position) + + if always_hide_cursor: + new_screen.show_cursor = False + else: + new_screen.show_cursor = ui_content.show_cursor + + self._highlight_digraph(new_screen) + + if highlight_lines: + self._highlight_cursorlines( + new_screen, + screen_cursor_position, + xpos, + ypos, + width, + write_position.height, + ) + + # Draw input characters from the input processor queue. + if has_focus and ui_content.cursor_position: + self._show_key_processor_key_buffer(new_screen) + + # Set menu position. + if ui_content.menu_position: + new_screen.set_menu_position( + self, + cursor_pos_to_screen_pos( + ui_content.menu_position.y, ui_content.menu_position.x + ), + ) + + # Update output screen height. + new_screen.height = max(new_screen.height, ypos + write_position.height) + + return visible_line_to_row_col, rowcol_to_yx + + def _fill_bg( + self, screen: Screen, write_position: WritePosition, erase_bg: bool + ) -> None: + """ + Erase/fill the background. + (Useful for floats and when a `char` has been given.) + """ + char: str | None + if callable(self.char): + char = self.char() + else: + char = self.char + + if erase_bg or char: + wp = write_position + char_obj = _CHAR_CACHE[char or " ", ""] + + for y in range(wp.ypos, wp.ypos + wp.height): + row = screen.data_buffer[y] + for x in range(wp.xpos, wp.xpos + wp.width): + row[x] = char_obj + + def _apply_style( + self, new_screen: Screen, write_position: WritePosition, parent_style: str + ) -> None: + # Apply `self.style`. + style = parent_style + " " + to_str(self.style) + + new_screen.fill_area(write_position, style=style, after=False) + + # Apply the 'last-line' class to the last line of each Window. This can + # be used to apply an 'underline' to the user control. + wp = WritePosition( + write_position.xpos, + write_position.ypos + write_position.height - 1, + write_position.width, + 1, + ) + new_screen.fill_area(wp, "class:last-line", after=True) + + def _highlight_digraph(self, new_screen: Screen) -> None: + """ + When we are in Vi digraph mode, put a question mark underneath the + cursor. + """ + digraph_char = self._get_digraph_char() + if digraph_char: + cpos = new_screen.get_cursor_position(self) + new_screen.data_buffer[cpos.y][cpos.x] = _CHAR_CACHE[ + digraph_char, "class:digraph" + ] + + def _show_key_processor_key_buffer(self, new_screen: Screen) -> None: + """ + When the user is typing a key binding that consists of several keys, + display the last pressed key if the user is in insert mode and the key + is meaningful to be displayed. + E.g. Some people want to bind 'jj' to escape in Vi insert mode. But the + first 'j' needs to be displayed in order to get some feedback. + """ + app = get_app() + key_buffer = app.key_processor.key_buffer + + if key_buffer and _in_insert_mode() and not app.is_done: + # The textual data for the given key. (Can be a VT100 escape + # sequence.) + data = key_buffer[-1].data + + # Display only if this is a 1 cell width character. + if get_cwidth(data) == 1: + cpos = new_screen.get_cursor_position(self) + new_screen.data_buffer[cpos.y][cpos.x] = _CHAR_CACHE[ + data, "class:partial-key-binding" + ] + + def _highlight_cursorlines( + self, new_screen: Screen, cpos: Point, x: int, y: int, width: int, height: int + ) -> None: + """ + Highlight cursor row/column. + """ + cursor_line_style = " class:cursor-line " + cursor_column_style = " class:cursor-column " + + data_buffer = new_screen.data_buffer + + # Highlight cursor line. + if self.cursorline(): + row = data_buffer[cpos.y] + for x in range(x, x + width): + original_char = row[x] + row[x] = _CHAR_CACHE[ + original_char.char, original_char.style + cursor_line_style + ] + + # Highlight cursor column. + if self.cursorcolumn(): + for y2 in range(y, y + height): + row = data_buffer[y2] + original_char = row[cpos.x] + row[cpos.x] = _CHAR_CACHE[ + original_char.char, original_char.style + cursor_column_style + ] + + # Highlight color columns + colorcolumns = self.colorcolumns + if callable(colorcolumns): + colorcolumns = colorcolumns() + + for cc in colorcolumns: + assert isinstance(cc, ColorColumn) + column = cc.position + + if column < x + width: # Only draw when visible. + color_column_style = " " + cc.style + + for y2 in range(y, y + height): + row = data_buffer[y2] + original_char = row[column + x] + row[column + x] = _CHAR_CACHE[ + original_char.char, original_char.style + color_column_style + ] + + def _copy_margin( + self, + margin_content: UIContent, + new_screen: Screen, + write_position: WritePosition, + move_x: int, + width: int, + ) -> None: + """ + Copy characters from the margin screen to the real screen. + """ + xpos = write_position.xpos + move_x + ypos = write_position.ypos + + margin_write_position = WritePosition(xpos, ypos, width, write_position.height) + self._copy_body(margin_content, new_screen, margin_write_position, 0, width) + + def _scroll(self, ui_content: UIContent, width: int, height: int) -> None: + """ + Scroll body. Ensure that the cursor is visible. + """ + if self.wrap_lines(): + func = self._scroll_when_linewrapping + else: + func = self._scroll_without_linewrapping + + func(ui_content, width, height) + + def _scroll_when_linewrapping( + self, ui_content: UIContent, width: int, height: int + ) -> None: + """ + Scroll to make sure the cursor position is visible and that we maintain + the requested scroll offset. + + Set `self.horizontal_scroll/vertical_scroll`. + """ + scroll_offsets_bottom = self.scroll_offsets.bottom + scroll_offsets_top = self.scroll_offsets.top + + # We don't have horizontal scrolling. + self.horizontal_scroll = 0 + + def get_line_height(lineno: int) -> int: + return ui_content.get_height_for_line(lineno, width, self.get_line_prefix) + + # When there is no space, reset `vertical_scroll_2` to zero and abort. + # This can happen if the margin is bigger than the window width. + # Otherwise the text height will become "infinite" (a big number) and + # the copy_line will spend a huge amount of iterations trying to render + # nothing. + if width <= 0: + self.vertical_scroll = ui_content.cursor_position.y + self.vertical_scroll_2 = 0 + return + + # If the current line consumes more than the whole window height, + # then we have to scroll vertically inside this line. (We don't take + # the scroll offsets into account for this.) + # Also, ignore the scroll offsets in this case. Just set the vertical + # scroll to this line. + line_height = get_line_height(ui_content.cursor_position.y) + if line_height > height - scroll_offsets_top: + # Calculate the height of the text before the cursor (including + # line prefixes). + text_before_height = ui_content.get_height_for_line( + ui_content.cursor_position.y, + width, + self.get_line_prefix, + slice_stop=ui_content.cursor_position.x, + ) + + # Adjust scroll offset. + self.vertical_scroll = ui_content.cursor_position.y + self.vertical_scroll_2 = min( + text_before_height - 1, # Keep the cursor visible. + line_height + - height, # Avoid blank lines at the bottom when scrolling up again. + self.vertical_scroll_2, + ) + self.vertical_scroll_2 = max( + 0, text_before_height - height, self.vertical_scroll_2 + ) + return + else: + self.vertical_scroll_2 = 0 + + # Current line doesn't consume the whole height. Take scroll offsets into account. + def get_min_vertical_scroll() -> int: + # Make sure that the cursor line is not below the bottom. + # (Calculate how many lines can be shown between the cursor and the .) + used_height = 0 + prev_lineno = ui_content.cursor_position.y + + for lineno in range(ui_content.cursor_position.y, -1, -1): + used_height += get_line_height(lineno) + + if used_height > height - scroll_offsets_bottom: + return prev_lineno + else: + prev_lineno = lineno + return 0 + + def get_max_vertical_scroll() -> int: + # Make sure that the cursor line is not above the top. + prev_lineno = ui_content.cursor_position.y + used_height = 0 + + for lineno in range(ui_content.cursor_position.y - 1, -1, -1): + used_height += get_line_height(lineno) + + if used_height > scroll_offsets_top: + return prev_lineno + else: + prev_lineno = lineno + return prev_lineno + + def get_topmost_visible() -> int: + """ + Calculate the upper most line that can be visible, while the bottom + is still visible. We should not allow scroll more than this if + `allow_scroll_beyond_bottom` is false. + """ + prev_lineno = ui_content.line_count - 1 + used_height = 0 + for lineno in range(ui_content.line_count - 1, -1, -1): + used_height += get_line_height(lineno) + if used_height > height: + return prev_lineno + else: + prev_lineno = lineno + return prev_lineno + + # Scroll vertically. (Make sure that the whole line which contains the + # cursor is visible. + topmost_visible = get_topmost_visible() + + # Note: the `min(topmost_visible, ...)` is to make sure that we + # don't require scrolling up because of the bottom scroll offset, + # when we are at the end of the document. + self.vertical_scroll = max( + self.vertical_scroll, min(topmost_visible, get_min_vertical_scroll()) + ) + self.vertical_scroll = min(self.vertical_scroll, get_max_vertical_scroll()) + + # Disallow scrolling beyond bottom? + if not self.allow_scroll_beyond_bottom(): + self.vertical_scroll = min(self.vertical_scroll, topmost_visible) + + def _scroll_without_linewrapping( + self, ui_content: UIContent, width: int, height: int + ) -> None: + """ + Scroll to make sure the cursor position is visible and that we maintain + the requested scroll offset. + + Set `self.horizontal_scroll/vertical_scroll`. + """ + cursor_position = ui_content.cursor_position or Point(x=0, y=0) + + # Without line wrapping, we will never have to scroll vertically inside + # a single line. + self.vertical_scroll_2 = 0 + + if ui_content.line_count == 0: + self.vertical_scroll = 0 + self.horizontal_scroll = 0 + return + else: + current_line_text = fragment_list_to_text( + ui_content.get_line(cursor_position.y) + ) + + def do_scroll( + current_scroll: int, + scroll_offset_start: int, + scroll_offset_end: int, + cursor_pos: int, + window_size: int, + content_size: int, + ) -> int: + "Scrolling algorithm. Used for both horizontal and vertical scrolling." + # Calculate the scroll offset to apply. + # This can obviously never be more than have the screen size. Also, when the + # cursor appears at the top or bottom, we don't apply the offset. + scroll_offset_start = int( + min(scroll_offset_start, window_size / 2, cursor_pos) + ) + scroll_offset_end = int( + min(scroll_offset_end, window_size / 2, content_size - 1 - cursor_pos) + ) + + # Prevent negative scroll offsets. + if current_scroll < 0: + current_scroll = 0 + + # Scroll back if we scrolled to much and there's still space to show more of the document. + if ( + not self.allow_scroll_beyond_bottom() + and current_scroll > content_size - window_size + ): + current_scroll = max(0, content_size - window_size) + + # Scroll up if cursor is before visible part. + if current_scroll > cursor_pos - scroll_offset_start: + current_scroll = max(0, cursor_pos - scroll_offset_start) + + # Scroll down if cursor is after visible part. + if current_scroll < (cursor_pos + 1) - window_size + scroll_offset_end: + current_scroll = (cursor_pos + 1) - window_size + scroll_offset_end + + return current_scroll + + # When a preferred scroll is given, take that first into account. + if self.get_vertical_scroll: + self.vertical_scroll = self.get_vertical_scroll(self) + assert isinstance(self.vertical_scroll, int) + if self.get_horizontal_scroll: + self.horizontal_scroll = self.get_horizontal_scroll(self) + assert isinstance(self.horizontal_scroll, int) + + # Update horizontal/vertical scroll to make sure that the cursor + # remains visible. + offsets = self.scroll_offsets + + self.vertical_scroll = do_scroll( + current_scroll=self.vertical_scroll, + scroll_offset_start=offsets.top, + scroll_offset_end=offsets.bottom, + cursor_pos=ui_content.cursor_position.y, + window_size=height, + content_size=ui_content.line_count, + ) + + if self.get_line_prefix: + current_line_prefix_width = fragment_list_width( + to_formatted_text(self.get_line_prefix(ui_content.cursor_position.y, 0)) + ) + else: + current_line_prefix_width = 0 + + self.horizontal_scroll = do_scroll( + current_scroll=self.horizontal_scroll, + scroll_offset_start=offsets.left, + scroll_offset_end=offsets.right, + cursor_pos=get_cwidth(current_line_text[: ui_content.cursor_position.x]), + window_size=width - current_line_prefix_width, + # We can only analyze the current line. Calculating the width off + # all the lines is too expensive. + content_size=max( + get_cwidth(current_line_text), self.horizontal_scroll + width + ), + ) + + def _mouse_handler(self, mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Mouse handler. Called when the UI control doesn't handle this + particular event. + + Return `NotImplemented` if nothing was done as a consequence of this + key binding (no UI invalidate required in that case). + """ + if mouse_event.event_type == MouseEventType.SCROLL_DOWN: + self._scroll_down() + return None + elif mouse_event.event_type == MouseEventType.SCROLL_UP: + self._scroll_up() + return None + + return NotImplemented + + def _scroll_down(self) -> None: + "Scroll window down." + info = self.render_info + + if info is None: + return + + if self.vertical_scroll < info.content_height - info.window_height: + if info.cursor_position.y <= info.configured_scroll_offsets.top: + self.content.move_cursor_down() + + self.vertical_scroll += 1 + + def _scroll_up(self) -> None: + "Scroll window up." + info = self.render_info + + if info is None: + return + + if info.vertical_scroll > 0: + # TODO: not entirely correct yet in case of line wrapping and long lines. + if ( + info.cursor_position.y + >= info.window_height - 1 - info.configured_scroll_offsets.bottom + ): + self.content.move_cursor_up() + + self.vertical_scroll -= 1 + + def get_key_bindings(self) -> KeyBindingsBase | None: + return self.content.get_key_bindings() + + def get_children(self) -> list[Container]: + return [] + + +class ConditionalContainer(Container): + """ + Wrapper around any other container that can change the visibility. The + received `filter` determines whether the given container should be + displayed or not. + + :param content: :class:`.Container` instance. + :param filter: :class:`.Filter` instance. + """ + + def __init__( + self, + content: AnyContainer, + filter: FilterOrBool, + alternative_content: AnyContainer | None = None, + ) -> None: + self.content = to_container(content) + self.alternative_content = ( + to_container(alternative_content) + if alternative_content is not None + else None + ) + self.filter = to_filter(filter) + + def __repr__(self) -> str: + return f"ConditionalContainer({self.content!r}, filter={self.filter!r})" + + def reset(self) -> None: + self.content.reset() + + def preferred_width(self, max_available_width: int) -> Dimension: + if self.filter(): + return self.content.preferred_width(max_available_width) + elif self.alternative_content is not None: + return self.alternative_content.preferred_width(max_available_width) + else: + return Dimension.zero() + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + if self.filter(): + return self.content.preferred_height(width, max_available_height) + elif self.alternative_content is not None: + return self.alternative_content.preferred_height( + width, max_available_height + ) + else: + return Dimension.zero() + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + if self.filter(): + return self.content.write_to_screen( + screen, mouse_handlers, write_position, parent_style, erase_bg, z_index + ) + elif self.alternative_content is not None: + return self.alternative_content.write_to_screen( + screen, + mouse_handlers, + write_position, + parent_style, + erase_bg, + z_index, + ) + + def get_children(self) -> list[Container]: + result = [self.content] + if self.alternative_content is not None: + result.append(self.alternative_content) + return result + + +class DynamicContainer(Container): + """ + Container class that dynamically returns any Container. + + :param get_container: Callable that returns a :class:`.Container` instance + or any widget with a ``__pt_container__`` method. + """ + + def __init__(self, get_container: Callable[[], AnyContainer]) -> None: + self.get_container = get_container + + def _get_container(self) -> Container: + """ + Return the current container object. + + We call `to_container`, because `get_container` can also return a + widget with a ``__pt_container__`` method. + """ + obj = self.get_container() + return to_container(obj) + + def reset(self) -> None: + self._get_container().reset() + + def preferred_width(self, max_available_width: int) -> Dimension: + return self._get_container().preferred_width(max_available_width) + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + return self._get_container().preferred_height(width, max_available_height) + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + self._get_container().write_to_screen( + screen, mouse_handlers, write_position, parent_style, erase_bg, z_index + ) + + def is_modal(self) -> bool: + return False + + def get_key_bindings(self) -> KeyBindingsBase | None: + # Key bindings will be collected when `layout.walk()` finds the child + # container. + return None + + def get_children(self) -> list[Container]: + # Here we have to return the current active container itself, not its + # children. Otherwise, we run into issues where `layout.walk()` will + # never see an object of type `Window` if this contains a window. We + # can't/shouldn't proxy the "isinstance" check. + return [self._get_container()] + + +def to_container(container: AnyContainer) -> Container: + """ + Make sure that the given object is a :class:`.Container`. + """ + if isinstance(container, Container): + return container + elif hasattr(container, "__pt_container__"): + return to_container(container.__pt_container__()) + else: + raise ValueError(f"Not a container object: {container!r}") + + +def to_window(container: AnyContainer) -> Window: + """ + Make sure that the given argument is a :class:`.Window`. + """ + if isinstance(container, Window): + return container + elif hasattr(container, "__pt_container__"): + return to_window(cast("MagicContainer", container).__pt_container__()) + else: + raise ValueError(f"Not a Window object: {container!r}.") + + +def is_container(value: object) -> TypeGuard[AnyContainer]: + """ + Checks whether the given value is a container object + (for use in assert statements). + """ + if isinstance(value, Container): + return True + if hasattr(value, "__pt_container__"): + return is_container(cast("MagicContainer", value).__pt_container__()) + return False diff --git a/lib/prompt_toolkit/layout/controls.py b/lib/prompt_toolkit/layout/controls.py new file mode 100644 index 0000000..5083c82 --- /dev/null +++ b/lib/prompt_toolkit/layout/controls.py @@ -0,0 +1,956 @@ +""" +User interface Controls for the layout. +""" + +from __future__ import annotations + +import time +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Callable, Hashable, Iterable, NamedTuple + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.cache import SimpleCache +from prompt_toolkit.data_structures import Point +from prompt_toolkit.document import Document +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + to_formatted_text, +) +from prompt_toolkit.formatted_text.utils import ( + fragment_list_to_text, + fragment_list_width, + split_lines, +) +from prompt_toolkit.lexers import Lexer, SimpleLexer +from prompt_toolkit.mouse_events import MouseButton, MouseEvent, MouseEventType +from prompt_toolkit.search import SearchState +from prompt_toolkit.selection import SelectionType +from prompt_toolkit.utils import get_cwidth + +from .processors import ( + DisplayMultipleCursors, + HighlightIncrementalSearchProcessor, + HighlightSearchProcessor, + HighlightSelectionProcessor, + Processor, + TransformationInput, + merge_processors, +) + +if TYPE_CHECKING: + from prompt_toolkit.key_binding.key_bindings import ( + KeyBindingsBase, + NotImplementedOrNone, + ) + from prompt_toolkit.utils import Event + + +__all__ = [ + "BufferControl", + "SearchBufferControl", + "DummyControl", + "FormattedTextControl", + "UIControl", + "UIContent", +] + +GetLinePrefixCallable = Callable[[int, int], AnyFormattedText] + + +class UIControl(metaclass=ABCMeta): + """ + Base class for all user interface controls. + """ + + def reset(self) -> None: + # Default reset. (Doesn't have to be implemented.) + pass + + def preferred_width(self, max_available_width: int) -> int | None: + return None + + def preferred_height( + self, + width: int, + max_available_height: int, + wrap_lines: bool, + get_line_prefix: GetLinePrefixCallable | None, + ) -> int | None: + return None + + def is_focusable(self) -> bool: + """ + Tell whether this user control is focusable. + """ + return False + + @abstractmethod + def create_content(self, width: int, height: int) -> UIContent: + """ + Generate the content for this user control. + + Returns a :class:`.UIContent` instance. + """ + + def mouse_handler(self, mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Handle mouse events. + + When `NotImplemented` is returned, it means that the given event is not + handled by the `UIControl` itself. The `Window` or key bindings can + decide to handle this event as scrolling or changing focus. + + :param mouse_event: `MouseEvent` instance. + """ + return NotImplemented + + def move_cursor_down(self) -> None: + """ + Request to move the cursor down. + This happens when scrolling down and the cursor is completely at the + top. + """ + + def move_cursor_up(self) -> None: + """ + Request to move the cursor up. + """ + + def get_key_bindings(self) -> KeyBindingsBase | None: + """ + The key bindings that are specific for this user control. + + Return a :class:`.KeyBindings` object if some key bindings are + specified, or `None` otherwise. + """ + + def get_invalidate_events(self) -> Iterable[Event[object]]: + """ + Return a list of `Event` objects. This can be a generator. + (The application collects all these events, in order to bind redraw + handlers to these events.) + """ + return [] + + +class UIContent: + """ + Content generated by a user control. This content consists of a list of + lines. + + :param get_line: Callable that takes a line number and returns the current + line. This is a list of (style_str, text) tuples. + :param line_count: The number of lines. + :param cursor_position: a :class:`.Point` for the cursor position. + :param menu_position: a :class:`.Point` for the menu position. + :param show_cursor: Make the cursor visible. + """ + + def __init__( + self, + get_line: Callable[[int], StyleAndTextTuples] = (lambda i: []), + line_count: int = 0, + cursor_position: Point | None = None, + menu_position: Point | None = None, + show_cursor: bool = True, + ): + self.get_line = get_line + self.line_count = line_count + self.cursor_position = cursor_position or Point(x=0, y=0) + self.menu_position = menu_position + self.show_cursor = show_cursor + + # Cache for line heights. Maps cache key -> height + self._line_heights_cache: dict[Hashable, int] = {} + + def __getitem__(self, lineno: int) -> StyleAndTextTuples: + "Make it iterable (iterate line by line)." + if lineno < self.line_count: + return self.get_line(lineno) + else: + raise IndexError + + def get_height_for_line( + self, + lineno: int, + width: int, + get_line_prefix: GetLinePrefixCallable | None, + slice_stop: int | None = None, + ) -> int: + """ + Return the height that a given line would need if it is rendered in a + space with the given width (using line wrapping). + + :param get_line_prefix: None or a `Window.get_line_prefix` callable + that returns the prefix to be inserted before this line. + :param slice_stop: Wrap only "line[:slice_stop]" and return that + partial result. This is needed for scrolling the window correctly + when line wrapping. + :returns: The computed height. + """ + # Instead of using `get_line_prefix` as key, we use render_counter + # instead. This is more reliable, because this function could still be + # the same, while the content would change over time. + key = get_app().render_counter, lineno, width, slice_stop + + try: + return self._line_heights_cache[key] + except KeyError: + if width == 0: + height = 10**8 + else: + # Calculate line width first. + line = fragment_list_to_text(self.get_line(lineno))[:slice_stop] + text_width = get_cwidth(line) + + if get_line_prefix: + # Add prefix width. + text_width += fragment_list_width( + to_formatted_text(get_line_prefix(lineno, 0)) + ) + + # Slower path: compute path when there's a line prefix. + height = 1 + + # Keep wrapping as long as the line doesn't fit. + # Keep adding new prefixes for every wrapped line. + while text_width > width: + height += 1 + text_width -= width + + fragments2 = to_formatted_text( + get_line_prefix(lineno, height - 1) + ) + prefix_width = get_cwidth(fragment_list_to_text(fragments2)) + + if prefix_width >= width: # Prefix doesn't fit. + height = 10**8 + break + + text_width += prefix_width + else: + # Fast path: compute height when there's no line prefix. + try: + quotient, remainder = divmod(text_width, width) + except ZeroDivisionError: + height = 10**8 + else: + if remainder: + quotient += 1 # Like math.ceil. + height = max(1, quotient) + + # Cache and return + self._line_heights_cache[key] = height + return height + + +class FormattedTextControl(UIControl): + """ + Control that displays formatted text. This can be either plain text, an + :class:`~prompt_toolkit.formatted_text.HTML` object an + :class:`~prompt_toolkit.formatted_text.ANSI` object, a list of ``(style_str, + text)`` tuples or a callable that takes no argument and returns one of + those, depending on how you prefer to do the formatting. See + ``prompt_toolkit.layout.formatted_text`` for more information. + + (It's mostly optimized for rather small widgets, like toolbars, menus, etc...) + + When this UI control has the focus, the cursor will be shown in the upper + left corner of this control by default. There are two ways for specifying + the cursor position: + + - Pass a `get_cursor_position` function which returns a `Point` instance + with the current cursor position. + + - If the (formatted) text is passed as a list of ``(style, text)`` tuples + and there is one that looks like ``('[SetCursorPosition]', '')``, then + this will specify the cursor position. + + Mouse support: + + The list of fragments can also contain tuples of three items, looking like: + (style_str, text, handler). When mouse support is enabled and the user + clicks on this fragment, then the given handler is called. That handler + should accept two inputs: (Application, MouseEvent) and it should + either handle the event or return `NotImplemented` in case we want the + containing Window to handle this event. + + :param focusable: `bool` or :class:`.Filter`: Tell whether this control is + focusable. + + :param text: Text or formatted text to be displayed. + :param style: Style string applied to the content. (If you want to style + the whole :class:`~prompt_toolkit.layout.Window`, pass the style to the + :class:`~prompt_toolkit.layout.Window` instead.) + :param key_bindings: a :class:`.KeyBindings` object. + :param get_cursor_position: A callable that returns the cursor position as + a `Point` instance. + """ + + def __init__( + self, + text: AnyFormattedText = "", + style: str = "", + focusable: FilterOrBool = False, + key_bindings: KeyBindingsBase | None = None, + show_cursor: bool = True, + modal: bool = False, + get_cursor_position: Callable[[], Point | None] | None = None, + ) -> None: + self.text = text # No type check on 'text'. This is done dynamically. + self.style = style + self.focusable = to_filter(focusable) + + # Key bindings. + self.key_bindings = key_bindings + self.show_cursor = show_cursor + self.modal = modal + self.get_cursor_position = get_cursor_position + + #: Cache for the content. + self._content_cache: SimpleCache[Hashable, UIContent] = SimpleCache(maxsize=18) + self._fragment_cache: SimpleCache[int, StyleAndTextTuples] = SimpleCache( + maxsize=1 + ) + # Only cache one fragment list. We don't need the previous item. + + # Render info for the mouse support. + self._fragments: StyleAndTextTuples | None = None + + def reset(self) -> None: + self._fragments = None + + def is_focusable(self) -> bool: + return self.focusable() + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.text!r})" + + def _get_formatted_text_cached(self) -> StyleAndTextTuples: + """ + Get fragments, but only retrieve fragments once during one render run. + (This function is called several times during one rendering, because + we also need those for calculating the dimensions.) + """ + return self._fragment_cache.get( + get_app().render_counter, lambda: to_formatted_text(self.text, self.style) + ) + + def preferred_width(self, max_available_width: int) -> int: + """ + Return the preferred width for this control. + That is the width of the longest line. + """ + text = fragment_list_to_text(self._get_formatted_text_cached()) + line_lengths = [get_cwidth(l) for l in text.split("\n")] + return max(line_lengths) + + def preferred_height( + self, + width: int, + max_available_height: int, + wrap_lines: bool, + get_line_prefix: GetLinePrefixCallable | None, + ) -> int | None: + """ + Return the preferred height for this control. + """ + content = self.create_content(width, None) + if wrap_lines: + height = 0 + for i in range(content.line_count): + height += content.get_height_for_line(i, width, get_line_prefix) + if height >= max_available_height: + return max_available_height + return height + else: + return content.line_count + + def create_content(self, width: int, height: int | None) -> UIContent: + # Get fragments + fragments_with_mouse_handlers = self._get_formatted_text_cached() + fragment_lines_with_mouse_handlers = list( + split_lines(fragments_with_mouse_handlers) + ) + + # Strip mouse handlers from fragments. + fragment_lines: list[StyleAndTextTuples] = [ + [(item[0], item[1]) for item in line] + for line in fragment_lines_with_mouse_handlers + ] + + # Keep track of the fragments with mouse handler, for later use in + # `mouse_handler`. + self._fragments = fragments_with_mouse_handlers + + # If there is a `[SetCursorPosition]` in the fragment list, set the + # cursor position here. + def get_cursor_position( + fragment: str = "[SetCursorPosition]", + ) -> Point | None: + for y, line in enumerate(fragment_lines): + x = 0 + for style_str, text, *_ in line: + if fragment in style_str: + return Point(x=x, y=y) + x += len(text) + return None + + # If there is a `[SetMenuPosition]`, set the menu over here. + def get_menu_position() -> Point | None: + return get_cursor_position("[SetMenuPosition]") + + cursor_position = (self.get_cursor_position or get_cursor_position)() + + # Create content, or take it from the cache. + key = (tuple(fragments_with_mouse_handlers), width, cursor_position) + + def get_content() -> UIContent: + return UIContent( + get_line=lambda i: fragment_lines[i], + line_count=len(fragment_lines), + show_cursor=self.show_cursor, + cursor_position=cursor_position, + menu_position=get_menu_position(), + ) + + return self._content_cache.get(key, get_content) + + def mouse_handler(self, mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Handle mouse events. + + (When the fragment list contained mouse handlers and the user clicked on + on any of these, the matching handler is called. This handler can still + return `NotImplemented` in case we want the + :class:`~prompt_toolkit.layout.Window` to handle this particular + event.) + """ + if self._fragments: + # Read the generator. + fragments_for_line = list(split_lines(self._fragments)) + + try: + fragments = fragments_for_line[mouse_event.position.y] + except IndexError: + return NotImplemented + else: + # Find position in the fragment list. + xpos = mouse_event.position.x + + # Find mouse handler for this character. + count = 0 + for item in fragments: + count += len(item[1]) + if count > xpos: + if len(item) >= 3: + # Handler found. Call it. + # (Handler can return NotImplemented, so return + # that result.) + handler = item[2] + return handler(mouse_event) + else: + break + + # Otherwise, don't handle here. + return NotImplemented + + def is_modal(self) -> bool: + return self.modal + + def get_key_bindings(self) -> KeyBindingsBase | None: + return self.key_bindings + + +class DummyControl(UIControl): + """ + A dummy control object that doesn't paint any content. + + Useful for filling a :class:`~prompt_toolkit.layout.Window`. (The + `fragment` and `char` attributes of the `Window` class can be used to + define the filling.) + """ + + def create_content(self, width: int, height: int) -> UIContent: + def get_line(i: int) -> StyleAndTextTuples: + return [] + + return UIContent(get_line=get_line, line_count=100**100) # Something very big. + + def is_focusable(self) -> bool: + return False + + +class _ProcessedLine(NamedTuple): + fragments: StyleAndTextTuples + source_to_display: Callable[[int], int] + display_to_source: Callable[[int], int] + + +class BufferControl(UIControl): + """ + Control for visualizing the content of a :class:`.Buffer`. + + :param buffer: The :class:`.Buffer` object to be displayed. + :param input_processors: A list of + :class:`~prompt_toolkit.layout.processors.Processor` objects. + :param include_default_input_processors: When True, include the default + processors for highlighting of selection, search and displaying of + multiple cursors. + :param lexer: :class:`.Lexer` instance for syntax highlighting. + :param preview_search: `bool` or :class:`.Filter`: Show search while + typing. When this is `True`, probably you want to add a + ``HighlightIncrementalSearchProcessor`` as well. Otherwise only the + cursor position will move, but the text won't be highlighted. + :param focusable: `bool` or :class:`.Filter`: Tell whether this control is focusable. + :param focus_on_click: Focus this buffer when it's click, but not yet focused. + :param key_bindings: a :class:`.KeyBindings` object. + """ + + def __init__( + self, + buffer: Buffer | None = None, + input_processors: list[Processor] | None = None, + include_default_input_processors: bool = True, + lexer: Lexer | None = None, + preview_search: FilterOrBool = False, + focusable: FilterOrBool = True, + search_buffer_control: ( + None | SearchBufferControl | Callable[[], SearchBufferControl] + ) = None, + menu_position: Callable[[], int | None] | None = None, + focus_on_click: FilterOrBool = False, + key_bindings: KeyBindingsBase | None = None, + ): + self.input_processors = input_processors + self.include_default_input_processors = include_default_input_processors + + self.default_input_processors = [ + HighlightSearchProcessor(), + HighlightIncrementalSearchProcessor(), + HighlightSelectionProcessor(), + DisplayMultipleCursors(), + ] + + self.preview_search = to_filter(preview_search) + self.focusable = to_filter(focusable) + self.focus_on_click = to_filter(focus_on_click) + + self.buffer = buffer or Buffer() + self.menu_position = menu_position + self.lexer = lexer or SimpleLexer() + self.key_bindings = key_bindings + self._search_buffer_control = search_buffer_control + + #: Cache for the lexer. + #: Often, due to cursor movement, undo/redo and window resizing + #: operations, it happens that a short time, the same document has to be + #: lexed. This is a fairly easy way to cache such an expensive operation. + self._fragment_cache: SimpleCache[ + Hashable, Callable[[int], StyleAndTextTuples] + ] = SimpleCache(maxsize=8) + + self._last_click_timestamp: float | None = None + self._last_get_processed_line: Callable[[int], _ProcessedLine] | None = None + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} buffer={self.buffer!r} at {id(self)!r}>" + + @property + def search_buffer_control(self) -> SearchBufferControl | None: + result: SearchBufferControl | None + + if callable(self._search_buffer_control): + result = self._search_buffer_control() + else: + result = self._search_buffer_control + + assert result is None or isinstance(result, SearchBufferControl) + return result + + @property + def search_buffer(self) -> Buffer | None: + control = self.search_buffer_control + if control is not None: + return control.buffer + return None + + @property + def search_state(self) -> SearchState: + """ + Return the `SearchState` for searching this `BufferControl`. This is + always associated with the search control. If one search bar is used + for searching multiple `BufferControls`, then they share the same + `SearchState`. + """ + search_buffer_control = self.search_buffer_control + if search_buffer_control: + return search_buffer_control.searcher_search_state + else: + return SearchState() + + def is_focusable(self) -> bool: + return self.focusable() + + def preferred_width(self, max_available_width: int) -> int | None: + """ + This should return the preferred width. + + Note: We don't specify a preferred width according to the content, + because it would be too expensive. Calculating the preferred + width can be done by calculating the longest line, but this would + require applying all the processors to each line. This is + unfeasible for a larger document, and doing it for small + documents only would result in inconsistent behavior. + """ + return None + + def preferred_height( + self, + width: int, + max_available_height: int, + wrap_lines: bool, + get_line_prefix: GetLinePrefixCallable | None, + ) -> int | None: + # Calculate the content height, if it was drawn on a screen with the + # given width. + height = 0 + content = self.create_content(width, height=1) # Pass a dummy '1' as height. + + # When line wrapping is off, the height should be equal to the amount + # of lines. + if not wrap_lines: + return content.line_count + + # When the number of lines exceeds the max_available_height, just + # return max_available_height. No need to calculate anything. + if content.line_count >= max_available_height: + return max_available_height + + for i in range(content.line_count): + height += content.get_height_for_line(i, width, get_line_prefix) + + if height >= max_available_height: + return max_available_height + + return height + + def _get_formatted_text_for_line_func( + self, document: Document + ) -> Callable[[int], StyleAndTextTuples]: + """ + Create a function that returns the fragments for a given line. + """ + + # Cache using `document.text`. + def get_formatted_text_for_line() -> Callable[[int], StyleAndTextTuples]: + return self.lexer.lex_document(document) + + key = (document.text, self.lexer.invalidation_hash()) + return self._fragment_cache.get(key, get_formatted_text_for_line) + + def _create_get_processed_line_func( + self, document: Document, width: int, height: int + ) -> Callable[[int], _ProcessedLine]: + """ + Create a function that takes a line number of the current document and + returns a _ProcessedLine(processed_fragments, source_to_display, display_to_source) + tuple. + """ + # Merge all input processors together. + input_processors = self.input_processors or [] + if self.include_default_input_processors: + input_processors = self.default_input_processors + input_processors + + merged_processor = merge_processors(input_processors) + + def transform( + lineno: int, + fragments: StyleAndTextTuples, + get_line: Callable[[int], StyleAndTextTuples], + ) -> _ProcessedLine: + "Transform the fragments for a given line number." + + # Get cursor position at this line. + def source_to_display(i: int) -> int: + """X position from the buffer to the x position in the + processed fragment list. By default, we start from the 'identity' + operation.""" + return i + + transformation = merged_processor.apply_transformation( + TransformationInput( + self, + document, + lineno, + source_to_display, + fragments, + width, + height, + get_line, + ) + ) + + return _ProcessedLine( + transformation.fragments, + transformation.source_to_display, + transformation.display_to_source, + ) + + def create_func() -> Callable[[int], _ProcessedLine]: + get_line = self._get_formatted_text_for_line_func(document) + cache: dict[int, _ProcessedLine] = {} + + def get_processed_line(i: int) -> _ProcessedLine: + try: + return cache[i] + except KeyError: + processed_line = transform(i, get_line(i), get_line) + cache[i] = processed_line + return processed_line + + return get_processed_line + + return create_func() + + def create_content( + self, width: int, height: int, preview_search: bool = False + ) -> UIContent: + """ + Create a UIContent. + """ + buffer = self.buffer + + # Trigger history loading of the buffer. We do this during the + # rendering of the UI here, because it needs to happen when an + # `Application` with its event loop is running. During the rendering of + # the buffer control is the earliest place we can achieve this, where + # we're sure the right event loop is active, and don't require user + # interaction (like in a key binding). + buffer.load_history_if_not_yet_loaded() + + # Get the document to be shown. If we are currently searching (the + # search buffer has focus, and the preview_search filter is enabled), + # then use the search document, which has possibly a different + # text/cursor position.) + search_control = self.search_buffer_control + preview_now = preview_search or bool( + # Only if this feature is enabled. + self.preview_search() + and + # And something was typed in the associated search field. + search_control + and search_control.buffer.text + and + # And we are searching in this control. (Many controls can point to + # the same search field, like in Pyvim.) + get_app().layout.search_target_buffer_control == self + ) + + if preview_now and search_control is not None: + ss = self.search_state + + document = buffer.document_for_search( + SearchState( + text=search_control.buffer.text, + direction=ss.direction, + ignore_case=ss.ignore_case, + ) + ) + else: + document = buffer.document + + get_processed_line = self._create_get_processed_line_func( + document, width, height + ) + self._last_get_processed_line = get_processed_line + + def translate_rowcol(row: int, col: int) -> Point: + "Return the content column for this coordinate." + return Point(x=get_processed_line(row).source_to_display(col), y=row) + + def get_line(i: int) -> StyleAndTextTuples: + "Return the fragments for a given line number." + fragments = get_processed_line(i).fragments + + # Add a space at the end, because that is a possible cursor + # position. (When inserting after the input.) We should do this on + # all the lines, not just the line containing the cursor. (Because + # otherwise, line wrapping/scrolling could change when moving the + # cursor around.) + fragments = fragments + [("", " ")] + return fragments + + content = UIContent( + get_line=get_line, + line_count=document.line_count, + cursor_position=translate_rowcol( + document.cursor_position_row, document.cursor_position_col + ), + ) + + # If there is an auto completion going on, use that start point for a + # pop-up menu position. (But only when this buffer has the focus -- + # there is only one place for a menu, determined by the focused buffer.) + if get_app().layout.current_control == self: + menu_position = self.menu_position() if self.menu_position else None + if menu_position is not None: + assert isinstance(menu_position, int) + menu_row, menu_col = buffer.document.translate_index_to_position( + menu_position + ) + content.menu_position = translate_rowcol(menu_row, menu_col) + elif buffer.complete_state: + # Position for completion menu. + # Note: We use 'min', because the original cursor position could be + # behind the input string when the actual completion is for + # some reason shorter than the text we had before. (A completion + # can change and shorten the input.) + menu_row, menu_col = buffer.document.translate_index_to_position( + min( + buffer.cursor_position, + buffer.complete_state.original_document.cursor_position, + ) + ) + content.menu_position = translate_rowcol(menu_row, menu_col) + else: + content.menu_position = None + + return content + + def mouse_handler(self, mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Mouse handler for this control. + """ + buffer = self.buffer + position = mouse_event.position + + # Focus buffer when clicked. + if get_app().layout.current_control == self: + if self._last_get_processed_line: + processed_line = self._last_get_processed_line(position.y) + + # Translate coordinates back to the cursor position of the + # original input. + xpos = processed_line.display_to_source(position.x) + index = buffer.document.translate_row_col_to_index(position.y, xpos) + + # Set the cursor position. + if mouse_event.event_type == MouseEventType.MOUSE_DOWN: + buffer.exit_selection() + buffer.cursor_position = index + + elif ( + mouse_event.event_type == MouseEventType.MOUSE_MOVE + and mouse_event.button != MouseButton.NONE + ): + # Click and drag to highlight a selection + if ( + buffer.selection_state is None + and abs(buffer.cursor_position - index) > 0 + ): + buffer.start_selection(selection_type=SelectionType.CHARACTERS) + buffer.cursor_position = index + + elif mouse_event.event_type == MouseEventType.MOUSE_UP: + # When the cursor was moved to another place, select the text. + # (The >1 is actually a small but acceptable workaround for + # selecting text in Vi navigation mode. In navigation mode, + # the cursor can never be after the text, so the cursor + # will be repositioned automatically.) + if abs(buffer.cursor_position - index) > 1: + if buffer.selection_state is None: + buffer.start_selection( + selection_type=SelectionType.CHARACTERS + ) + buffer.cursor_position = index + + # Select word around cursor on double click. + # Two MOUSE_UP events in a short timespan are considered a double click. + double_click = ( + self._last_click_timestamp + and time.time() - self._last_click_timestamp < 0.3 + ) + self._last_click_timestamp = time.time() + + if double_click: + start, end = buffer.document.find_boundaries_of_current_word() + buffer.cursor_position += start + buffer.start_selection(selection_type=SelectionType.CHARACTERS) + buffer.cursor_position += end - start + else: + # Don't handle scroll events here. + return NotImplemented + + # Not focused, but focusing on click events. + else: + if ( + self.focus_on_click() + and mouse_event.event_type == MouseEventType.MOUSE_UP + ): + # Focus happens on mouseup. (If we did this on mousedown, the + # up event will be received at the point where this widget is + # focused and be handled anyway.) + get_app().layout.current_control = self + else: + return NotImplemented + + return None + + def move_cursor_down(self) -> None: + b = self.buffer + b.cursor_position += b.document.get_cursor_down_position() + + def move_cursor_up(self) -> None: + b = self.buffer + b.cursor_position += b.document.get_cursor_up_position() + + def get_key_bindings(self) -> KeyBindingsBase | None: + """ + When additional key bindings are given. Return these. + """ + return self.key_bindings + + def get_invalidate_events(self) -> Iterable[Event[object]]: + """ + Return the Window invalidate events. + """ + # Whenever the buffer changes, the UI has to be updated. + yield self.buffer.on_text_changed + yield self.buffer.on_cursor_position_changed + + yield self.buffer.on_completions_changed + yield self.buffer.on_suggestion_set + + +class SearchBufferControl(BufferControl): + """ + :class:`.BufferControl` which is used for searching another + :class:`.BufferControl`. + + :param ignore_case: Search case insensitive. + """ + + def __init__( + self, + buffer: Buffer | None = None, + input_processors: list[Processor] | None = None, + lexer: Lexer | None = None, + focus_on_click: FilterOrBool = False, + key_bindings: KeyBindingsBase | None = None, + ignore_case: FilterOrBool = False, + ): + super().__init__( + buffer=buffer, + input_processors=input_processors, + lexer=lexer, + focus_on_click=focus_on_click, + key_bindings=key_bindings, + ) + + # If this BufferControl is used as a search field for one or more other + # BufferControls, then represents the search state. + self.searcher_search_state = SearchState(ignore_case=ignore_case) diff --git a/lib/prompt_toolkit/layout/dimension.py b/lib/prompt_toolkit/layout/dimension.py new file mode 100644 index 0000000..eec6c69 --- /dev/null +++ b/lib/prompt_toolkit/layout/dimension.py @@ -0,0 +1,216 @@ +""" +Layout dimensions are used to give the minimum, maximum and preferred +dimensions for containers and controls. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, Union + +__all__ = [ + "Dimension", + "D", + "sum_layout_dimensions", + "max_layout_dimensions", + "AnyDimension", + "to_dimension", + "is_dimension", +] + +if TYPE_CHECKING: + from typing_extensions import TypeGuard + + +class Dimension: + """ + Specified dimension (width/height) of a user control or window. + + The layout engine tries to honor the preferred size. If that is not + possible, because the terminal is larger or smaller, it tries to keep in + between min and max. + + :param min: Minimum size. + :param max: Maximum size. + :param weight: For a VSplit/HSplit, the actual size will be determined + by taking the proportion of weights from all the children. + E.g. When there are two children, one with a weight of 1, + and the other with a weight of 2, the second will always be + twice as big as the first, if the min/max values allow it. + :param preferred: Preferred size. + """ + + def __init__( + self, + min: int | None = None, + max: int | None = None, + weight: int | None = None, + preferred: int | None = None, + ) -> None: + if weight is not None: + assert weight >= 0 # Also cannot be a float. + + assert min is None or min >= 0 + assert max is None or max >= 0 + assert preferred is None or preferred >= 0 + + self.min_specified = min is not None + self.max_specified = max is not None + self.preferred_specified = preferred is not None + self.weight_specified = weight is not None + + if min is None: + min = 0 # Smallest possible value. + if max is None: # 0-values are allowed, so use "is None" + max = 1000**10 # Something huge. + if preferred is None: + preferred = min + if weight is None: + weight = 1 + + self.min = min + self.max = max + self.preferred = preferred + self.weight = weight + + # Don't allow situations where max < min. (This would be a bug.) + if max < min: + raise ValueError("Invalid Dimension: max < min.") + + # Make sure that the 'preferred' size is always in the min..max range. + if self.preferred < self.min: + self.preferred = self.min + + if self.preferred > self.max: + self.preferred = self.max + + @classmethod + def exact(cls, amount: int) -> Dimension: + """ + Return a :class:`.Dimension` with an exact size. (min, max and + preferred set to ``amount``). + """ + return cls(min=amount, max=amount, preferred=amount) + + @classmethod + def zero(cls) -> Dimension: + """ + Create a dimension that represents a zero size. (Used for 'invisible' + controls.) + """ + return cls.exact(amount=0) + + def __repr__(self) -> str: + fields = [] + if self.min_specified: + fields.append(f"min={self.min!r}") + if self.max_specified: + fields.append(f"max={self.max!r}") + if self.preferred_specified: + fields.append(f"preferred={self.preferred!r}") + if self.weight_specified: + fields.append(f"weight={self.weight!r}") + + return "Dimension({})".format(", ".join(fields)) + + +def sum_layout_dimensions(dimensions: list[Dimension]) -> Dimension: + """ + Sum a list of :class:`.Dimension` instances. + """ + min = sum(d.min for d in dimensions) + max = sum(d.max for d in dimensions) + preferred = sum(d.preferred for d in dimensions) + + return Dimension(min=min, max=max, preferred=preferred) + + +def max_layout_dimensions(dimensions: list[Dimension]) -> Dimension: + """ + Take the maximum of a list of :class:`.Dimension` instances. + Used when we have a HSplit/VSplit, and we want to get the best width/height.) + """ + if not len(dimensions): + return Dimension.zero() + + # If all dimensions are size zero. Return zero. + # (This is important for HSplit/VSplit, to report the right values to their + # parent when all children are invisible.) + if all(d.preferred == 0 and d.max == 0 for d in dimensions): + return Dimension.zero() + + # Ignore empty dimensions. (They should not reduce the size of others.) + dimensions = [d for d in dimensions if d.preferred != 0 and d.max != 0] + + if dimensions: + # Take the highest minimum dimension. + min_ = max(d.min for d in dimensions) + + # For the maximum, we would prefer not to go larger than then smallest + # 'max' value, unless other dimensions have a bigger preferred value. + # This seems to work best: + # - We don't want that a widget with a small height in a VSplit would + # shrink other widgets in the split. + # If it doesn't work well enough, then it's up to the UI designer to + # explicitly pass dimensions. + max_ = min(d.max for d in dimensions) + max_ = max(max_, max(d.preferred for d in dimensions)) + + # Make sure that min>=max. In some scenarios, when certain min..max + # ranges don't have any overlap, we can end up in such an impossible + # situation. In that case, give priority to the max value. + # E.g. taking (1..5) and (8..9) would return (8..5). Instead take (8..8). + if min_ > max_: + max_ = min_ + + preferred = max(d.preferred for d in dimensions) + + return Dimension(min=min_, max=max_, preferred=preferred) + else: + return Dimension() + + +# Anything that can be converted to a dimension. +AnyDimension = Union[ + None, # None is a valid dimension that will fit anything. + int, + Dimension, + # Callable[[], 'AnyDimension'] # Recursive definition not supported by mypy. + Callable[[], Any], +] + + +def to_dimension(value: AnyDimension) -> Dimension: + """ + Turn the given object into a `Dimension` object. + """ + if value is None: + return Dimension() + if isinstance(value, int): + return Dimension.exact(value) + if isinstance(value, Dimension): + return value + if callable(value): + return to_dimension(value()) + + raise ValueError("Not an integer or Dimension object.") + + +def is_dimension(value: object) -> TypeGuard[AnyDimension]: + """ + Test whether the given value could be a valid dimension. + (For usage in an assertion. It's not guaranteed in case of a callable.) + """ + if value is None: + return True + if callable(value): + return True # Assume it's a callable that doesn't take arguments. + if isinstance(value, (int, Dimension)): + return True + return False + + +# Common alias. +D = Dimension + +# For backward-compatibility. +LayoutDimension = Dimension diff --git a/lib/prompt_toolkit/layout/dummy.py b/lib/prompt_toolkit/layout/dummy.py new file mode 100644 index 0000000..1ee3e6c --- /dev/null +++ b/lib/prompt_toolkit/layout/dummy.py @@ -0,0 +1,40 @@ +""" +Dummy layout. Used when somebody creates an `Application` without specifying a +`Layout`. +""" + +from __future__ import annotations + +from prompt_toolkit.formatted_text import HTML +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPressEvent + +from .containers import Window +from .controls import FormattedTextControl +from .dimension import D +from .layout import Layout + +__all__ = [ + "create_dummy_layout", +] + +E = KeyPressEvent + + +def create_dummy_layout() -> Layout: + """ + Create a dummy layout for use in an 'Application' that doesn't have a + layout specified. When ENTER is pressed, the application quits. + """ + kb = KeyBindings() + + @kb.add("enter") + def enter(event: E) -> None: + event.app.exit() + + control = FormattedTextControl( + HTML("No layout specified. Press ENTER to quit."), + key_bindings=kb, + ) + window = Window(content=control, height=D(min=1)) + return Layout(container=window, focused_element=window) diff --git a/lib/prompt_toolkit/layout/layout.py b/lib/prompt_toolkit/layout/layout.py new file mode 100644 index 0000000..f9b7110 --- /dev/null +++ b/lib/prompt_toolkit/layout/layout.py @@ -0,0 +1,412 @@ +""" +Wrapper for the layout. +""" + +from __future__ import annotations + +from typing import Generator, Iterable, Union + +from prompt_toolkit.buffer import Buffer + +from .containers import ( + AnyContainer, + ConditionalContainer, + Container, + Window, + to_container, +) +from .controls import BufferControl, SearchBufferControl, UIControl + +__all__ = [ + "Layout", + "InvalidLayoutError", + "walk", +] + +FocusableElement = Union[str, Buffer, UIControl, AnyContainer] + + +class Layout: + """ + The layout for a prompt_toolkit + :class:`~prompt_toolkit.application.Application`. + This also keeps track of which user control is focused. + + :param container: The "root" container for the layout. + :param focused_element: element to be focused initially. (Can be anything + the `focus` function accepts.) + """ + + def __init__( + self, + container: AnyContainer, + focused_element: FocusableElement | None = None, + ) -> None: + self.container = to_container(container) + self._stack: list[Window] = [] + + # Map search BufferControl back to the original BufferControl. + # This is used to keep track of when exactly we are searching, and for + # applying the search. + # When a link exists in this dictionary, that means the search is + # currently active. + # Map: search_buffer_control -> original buffer control. + self.search_links: dict[SearchBufferControl, BufferControl] = {} + + # Mapping that maps the children in the layout to their parent. + # This relationship is calculated dynamically, each time when the UI + # is rendered. (UI elements have only references to their children.) + self._child_to_parent: dict[Container, Container] = {} + + if focused_element is None: + try: + self._stack.append(next(self.find_all_windows())) + except StopIteration as e: + raise InvalidLayoutError( + "Invalid layout. The layout does not contain any Window object." + ) from e + else: + self.focus(focused_element) + + # List of visible windows. + self.visible_windows: list[Window] = [] # List of `Window` objects. + + def __repr__(self) -> str: + return f"Layout({self.container!r}, current_window={self.current_window!r})" + + def find_all_windows(self) -> Generator[Window, None, None]: + """ + Find all the :class:`.UIControl` objects in this layout. + """ + for item in self.walk(): + if isinstance(item, Window): + yield item + + def find_all_controls(self) -> Iterable[UIControl]: + for container in self.find_all_windows(): + yield container.content + + def focus(self, value: FocusableElement) -> None: + """ + Focus the given UI element. + + `value` can be either: + + - a :class:`.UIControl` + - a :class:`.Buffer` instance or the name of a :class:`.Buffer` + - a :class:`.Window` + - Any container object. In this case we will focus the :class:`.Window` + from this container that was focused most recent, or the very first + focusable :class:`.Window` of the container. + """ + # BufferControl by buffer name. + if isinstance(value, str): + for control in self.find_all_controls(): + if isinstance(control, BufferControl) and control.buffer.name == value: + self.focus(control) + return + raise ValueError(f"Couldn't find Buffer in the current layout: {value!r}.") + + # BufferControl by buffer object. + elif isinstance(value, Buffer): + for control in self.find_all_controls(): + if isinstance(control, BufferControl) and control.buffer == value: + self.focus(control) + return + raise ValueError(f"Couldn't find Buffer in the current layout: {value!r}.") + + # Focus UIControl. + elif isinstance(value, UIControl): + if value not in self.find_all_controls(): + raise ValueError( + "Invalid value. Container does not appear in the layout." + ) + if not value.is_focusable(): + raise ValueError("Invalid value. UIControl is not focusable.") + + self.current_control = value + + # Otherwise, expecting any Container object. + else: + value = to_container(value) + + if isinstance(value, Window): + # This is a `Window`: focus that. + if value not in self.find_all_windows(): + raise ValueError( + f"Invalid value. Window does not appear in the layout: {value!r}" + ) + + self.current_window = value + else: + # Focus a window in this container. + # If we have many windows as part of this container, and some + # of them have been focused before, take the last focused + # item. (This is very useful when the UI is composed of more + # complex sub components.) + windows = [] + for c in walk(value, skip_hidden=True): + if isinstance(c, Window) and c.content.is_focusable(): + windows.append(c) + + # Take the first one that was focused before. + for w in reversed(self._stack): + if w in windows: + self.current_window = w + return + + # None was focused before: take the very first focusable window. + if windows: + self.current_window = windows[0] + return + + raise ValueError( + f"Invalid value. Container cannot be focused: {value!r}" + ) + + def has_focus(self, value: FocusableElement) -> bool: + """ + Check whether the given control has the focus. + :param value: :class:`.UIControl` or :class:`.Window` instance. + """ + if isinstance(value, str): + if self.current_buffer is None: + return False + return self.current_buffer.name == value + if isinstance(value, Buffer): + return self.current_buffer == value + if isinstance(value, UIControl): + return self.current_control == value + else: + value = to_container(value) + if isinstance(value, Window): + return self.current_window == value + else: + # Check whether this "container" is focused. This is true if + # one of the elements inside is focused. + for element in walk(value): + if element == self.current_window: + return True + return False + + @property + def current_control(self) -> UIControl: + """ + Get the :class:`.UIControl` to currently has the focus. + """ + return self._stack[-1].content + + @current_control.setter + def current_control(self, control: UIControl) -> None: + """ + Set the :class:`.UIControl` to receive the focus. + """ + for window in self.find_all_windows(): + if window.content == control: + self.current_window = window + return + + raise ValueError("Control not found in the user interface.") + + @property + def current_window(self) -> Window: + "Return the :class:`.Window` object that is currently focused." + return self._stack[-1] + + @current_window.setter + def current_window(self, value: Window) -> None: + "Set the :class:`.Window` object to be currently focused." + self._stack.append(value) + + @property + def is_searching(self) -> bool: + "True if we are searching right now." + return self.current_control in self.search_links + + @property + def search_target_buffer_control(self) -> BufferControl | None: + """ + Return the :class:`.BufferControl` in which we are searching or `None`. + """ + # Not every `UIControl` is a `BufferControl`. This only applies to + # `BufferControl`. + control = self.current_control + + if isinstance(control, SearchBufferControl): + return self.search_links.get(control) + else: + return None + + def get_focusable_windows(self) -> Iterable[Window]: + """ + Return all the :class:`.Window` objects which are focusable (in the + 'modal' area). + """ + for w in self.walk_through_modal_area(): + if isinstance(w, Window) and w.content.is_focusable(): + yield w + + def get_visible_focusable_windows(self) -> list[Window]: + """ + Return a list of :class:`.Window` objects that are focusable. + """ + # focusable windows are windows that are visible, but also part of the + # modal container. Make sure to keep the ordering. + visible_windows = self.visible_windows + return [w for w in self.get_focusable_windows() if w in visible_windows] + + @property + def current_buffer(self) -> Buffer | None: + """ + The currently focused :class:`~.Buffer` or `None`. + """ + ui_control = self.current_control + if isinstance(ui_control, BufferControl): + return ui_control.buffer + return None + + def get_buffer_by_name(self, buffer_name: str) -> Buffer | None: + """ + Look in the layout for a buffer with the given name. + Return `None` when nothing was found. + """ + for w in self.walk(): + if isinstance(w, Window) and isinstance(w.content, BufferControl): + if w.content.buffer.name == buffer_name: + return w.content.buffer + return None + + @property + def buffer_has_focus(self) -> bool: + """ + Return `True` if the currently focused control is a + :class:`.BufferControl`. (For instance, used to determine whether the + default key bindings should be active or not.) + """ + ui_control = self.current_control + return isinstance(ui_control, BufferControl) + + @property + def previous_control(self) -> UIControl: + """ + Get the :class:`.UIControl` to previously had the focus. + """ + try: + return self._stack[-2].content + except IndexError: + return self._stack[-1].content + + def focus_last(self) -> None: + """ + Give the focus to the last focused control. + """ + if len(self._stack) > 1: + self._stack = self._stack[:-1] + + def focus_next(self) -> None: + """ + Focus the next visible/focusable Window. + """ + windows = self.get_visible_focusable_windows() + + if len(windows) > 0: + try: + index = windows.index(self.current_window) + except ValueError: + index = 0 + else: + index = (index + 1) % len(windows) + + self.focus(windows[index]) + + def focus_previous(self) -> None: + """ + Focus the previous visible/focusable Window. + """ + windows = self.get_visible_focusable_windows() + + if len(windows) > 0: + try: + index = windows.index(self.current_window) + except ValueError: + index = 0 + else: + index = (index - 1) % len(windows) + + self.focus(windows[index]) + + def walk(self) -> Iterable[Container]: + """ + Walk through all the layout nodes (and their children) and yield them. + """ + yield from walk(self.container) + + def walk_through_modal_area(self) -> Iterable[Container]: + """ + Walk through all the containers which are in the current 'modal' part + of the layout. + """ + # Go up in the tree, and find the root. (it will be a part of the + # layout, if the focus is in a modal part.) + root: Container = self.current_window + while not root.is_modal() and root in self._child_to_parent: + root = self._child_to_parent[root] + + yield from walk(root) + + def update_parents_relations(self) -> None: + """ + Update child->parent relationships mapping. + """ + parents = {} + + def walk(e: Container) -> None: + for c in e.get_children(): + parents[c] = e + walk(c) + + walk(self.container) + + self._child_to_parent = parents + + def reset(self) -> None: + # Remove all search links when the UI starts. + # (Important, for instance when control-c is been pressed while + # searching. The prompt cancels, but next `run()` call the search + # links are still there.) + self.search_links.clear() + + self.container.reset() + + def get_parent(self, container: Container) -> Container | None: + """ + Return the parent container for the given container, or ``None``, if it + wasn't found. + """ + try: + return self._child_to_parent[container] + except KeyError: + return None + + +class InvalidLayoutError(Exception): + pass + + +def walk(container: Container, skip_hidden: bool = False) -> Iterable[Container]: + """ + Walk through layout, starting at this container. + """ + # When `skip_hidden` is set, don't go into disabled ConditionalContainer containers. + if ( + skip_hidden + and isinstance(container, ConditionalContainer) + and not container.filter() + ): + return + + yield container + + for c in container.get_children(): + # yield from walk(c) + yield from walk(c, skip_hidden=skip_hidden) diff --git a/lib/prompt_toolkit/layout/margins.py b/lib/prompt_toolkit/layout/margins.py new file mode 100644 index 0000000..737a74d --- /dev/null +++ b/lib/prompt_toolkit/layout/margins.py @@ -0,0 +1,304 @@ +""" +Margin implementations for a :class:`~prompt_toolkit.layout.containers.Window`. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Callable + +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.formatted_text import ( + StyleAndTextTuples, + fragment_list_to_text, + to_formatted_text, +) +from prompt_toolkit.utils import get_cwidth + +from .controls import UIContent + +if TYPE_CHECKING: + from .containers import WindowRenderInfo + +__all__ = [ + "Margin", + "NumberedMargin", + "ScrollbarMargin", + "ConditionalMargin", + "PromptMargin", +] + + +class Margin(metaclass=ABCMeta): + """ + Base interface for a margin. + """ + + @abstractmethod + def get_width(self, get_ui_content: Callable[[], UIContent]) -> int: + """ + Return the width that this margin is going to consume. + + :param get_ui_content: Callable that asks the user control to create + a :class:`.UIContent` instance. This can be used for instance to + obtain the number of lines. + """ + return 0 + + @abstractmethod + def create_margin( + self, window_render_info: WindowRenderInfo, width: int, height: int + ) -> StyleAndTextTuples: + """ + Creates a margin. + This should return a list of (style_str, text) tuples. + + :param window_render_info: + :class:`~prompt_toolkit.layout.containers.WindowRenderInfo` + instance, generated after rendering and copying the visible part of + the :class:`~prompt_toolkit.layout.controls.UIControl` into the + :class:`~prompt_toolkit.layout.containers.Window`. + :param width: The width that's available for this margin. (As reported + by :meth:`.get_width`.) + :param height: The height that's available for this margin. (The height + of the :class:`~prompt_toolkit.layout.containers.Window`.) + """ + return [] + + +class NumberedMargin(Margin): + """ + Margin that displays the line numbers. + + :param relative: Number relative to the cursor position. Similar to the Vi + 'relativenumber' option. + :param display_tildes: Display tildes after the end of the document, just + like Vi does. + """ + + def __init__( + self, relative: FilterOrBool = False, display_tildes: FilterOrBool = False + ) -> None: + self.relative = to_filter(relative) + self.display_tildes = to_filter(display_tildes) + + def get_width(self, get_ui_content: Callable[[], UIContent]) -> int: + line_count = get_ui_content().line_count + return max(3, len(f"{line_count}") + 1) + + def create_margin( + self, window_render_info: WindowRenderInfo, width: int, height: int + ) -> StyleAndTextTuples: + relative = self.relative() + + style = "class:line-number" + style_current = "class:line-number.current" + + # Get current line number. + current_lineno = window_render_info.ui_content.cursor_position.y + + # Construct margin. + result: StyleAndTextTuples = [] + last_lineno = None + + for y, lineno in enumerate(window_render_info.displayed_lines): + # Only display line number if this line is not a continuation of the previous line. + if lineno != last_lineno: + if lineno is None: + pass + elif lineno == current_lineno: + # Current line. + if relative: + # Left align current number in relative mode. + result.append((style_current, "%i" % (lineno + 1))) + else: + result.append( + (style_current, ("%i " % (lineno + 1)).rjust(width)) + ) + else: + # Other lines. + if relative: + lineno = abs(lineno - current_lineno) - 1 + + result.append((style, ("%i " % (lineno + 1)).rjust(width))) + + last_lineno = lineno + result.append(("", "\n")) + + # Fill with tildes. + if self.display_tildes(): + while y < window_render_info.window_height: + result.append(("class:tilde", "~\n")) + y += 1 + + return result + + +class ConditionalMargin(Margin): + """ + Wrapper around other :class:`.Margin` classes to show/hide them. + """ + + def __init__(self, margin: Margin, filter: FilterOrBool) -> None: + self.margin = margin + self.filter = to_filter(filter) + + def get_width(self, get_ui_content: Callable[[], UIContent]) -> int: + if self.filter(): + return self.margin.get_width(get_ui_content) + else: + return 0 + + def create_margin( + self, window_render_info: WindowRenderInfo, width: int, height: int + ) -> StyleAndTextTuples: + if width and self.filter(): + return self.margin.create_margin(window_render_info, width, height) + else: + return [] + + +class ScrollbarMargin(Margin): + """ + Margin displaying a scrollbar. + + :param display_arrows: Display scroll up/down arrows. + """ + + def __init__( + self, + display_arrows: FilterOrBool = False, + up_arrow_symbol: str = "^", + down_arrow_symbol: str = "v", + ) -> None: + self.display_arrows = to_filter(display_arrows) + self.up_arrow_symbol = up_arrow_symbol + self.down_arrow_symbol = down_arrow_symbol + + def get_width(self, get_ui_content: Callable[[], UIContent]) -> int: + return 1 + + def create_margin( + self, window_render_info: WindowRenderInfo, width: int, height: int + ) -> StyleAndTextTuples: + content_height = window_render_info.content_height + window_height = window_render_info.window_height + display_arrows = self.display_arrows() + + if display_arrows: + window_height -= 2 + + try: + fraction_visible = len(window_render_info.displayed_lines) / float( + content_height + ) + fraction_above = window_render_info.vertical_scroll / float(content_height) + + scrollbar_height = int( + min(window_height, max(1, window_height * fraction_visible)) + ) + scrollbar_top = int(window_height * fraction_above) + except ZeroDivisionError: + return [] + else: + + def is_scroll_button(row: int) -> bool: + "True if we should display a button on this row." + return scrollbar_top <= row <= scrollbar_top + scrollbar_height + + # Up arrow. + result: StyleAndTextTuples = [] + if display_arrows: + result.extend( + [ + ("class:scrollbar.arrow", self.up_arrow_symbol), + ("class:scrollbar", "\n"), + ] + ) + + # Scrollbar body. + scrollbar_background = "class:scrollbar.background" + scrollbar_background_start = "class:scrollbar.background,scrollbar.start" + scrollbar_button = "class:scrollbar.button" + scrollbar_button_end = "class:scrollbar.button,scrollbar.end" + + for i in range(window_height): + if is_scroll_button(i): + if not is_scroll_button(i + 1): + # Give the last cell a different style, because we + # want to underline this. + result.append((scrollbar_button_end, " ")) + else: + result.append((scrollbar_button, " ")) + else: + if is_scroll_button(i + 1): + result.append((scrollbar_background_start, " ")) + else: + result.append((scrollbar_background, " ")) + result.append(("", "\n")) + + # Down arrow + if display_arrows: + result.append(("class:scrollbar.arrow", self.down_arrow_symbol)) + + return result + + +class PromptMargin(Margin): + """ + [Deprecated] + + Create margin that displays a prompt. + This can display one prompt at the first line, and a continuation prompt + (e.g, just dots) on all the following lines. + + This `PromptMargin` implementation has been largely superseded in favor of + the `get_line_prefix` attribute of `Window`. The reason is that a margin is + always a fixed width, while `get_line_prefix` can return a variable width + prefix in front of every line, making it more powerful, especially for line + continuations. + + :param get_prompt: Callable returns formatted text or a list of + `(style_str, type)` tuples to be shown as the prompt at the first line. + :param get_continuation: Callable that takes three inputs. The width (int), + line_number (int), and is_soft_wrap (bool). It should return formatted + text or a list of `(style_str, type)` tuples for the next lines of the + input. + """ + + def __init__( + self, + get_prompt: Callable[[], StyleAndTextTuples], + get_continuation: None + | (Callable[[int, int, bool], StyleAndTextTuples]) = None, + ) -> None: + self.get_prompt = get_prompt + self.get_continuation = get_continuation + + def get_width(self, get_ui_content: Callable[[], UIContent]) -> int: + "Width to report to the `Window`." + # Take the width from the first line. + text = fragment_list_to_text(self.get_prompt()) + return get_cwidth(text) + + def create_margin( + self, window_render_info: WindowRenderInfo, width: int, height: int + ) -> StyleAndTextTuples: + get_continuation = self.get_continuation + result: StyleAndTextTuples = [] + + # First line. + result.extend(to_formatted_text(self.get_prompt())) + + # Next lines. + if get_continuation: + last_y = None + + for y in window_render_info.displayed_lines[1:]: + result.append(("", "\n")) + result.extend( + to_formatted_text(get_continuation(width, y, y == last_y)) + ) + last_y = y + + return result diff --git a/lib/prompt_toolkit/layout/menus.py b/lib/prompt_toolkit/layout/menus.py new file mode 100644 index 0000000..612e8ab --- /dev/null +++ b/lib/prompt_toolkit/layout/menus.py @@ -0,0 +1,748 @@ +from __future__ import annotations + +import math +from itertools import zip_longest +from typing import TYPE_CHECKING, Callable, Iterable, Sequence, TypeVar, cast +from weakref import WeakKeyDictionary + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.buffer import CompletionState +from prompt_toolkit.completion import Completion +from prompt_toolkit.data_structures import Point +from prompt_toolkit.filters import ( + Condition, + FilterOrBool, + has_completions, + is_done, + to_filter, +) +from prompt_toolkit.formatted_text import ( + StyleAndTextTuples, + fragment_list_width, + to_formatted_text, +) +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.layout.utils import explode_text_fragments +from prompt_toolkit.mouse_events import MouseEvent, MouseEventType +from prompt_toolkit.utils import get_cwidth + +from .containers import ConditionalContainer, HSplit, ScrollOffsets, Window +from .controls import GetLinePrefixCallable, UIContent, UIControl +from .dimension import Dimension +from .margins import ScrollbarMargin + +if TYPE_CHECKING: + from prompt_toolkit.key_binding.key_bindings import ( + KeyBindings, + NotImplementedOrNone, + ) + + +__all__ = [ + "CompletionsMenu", + "MultiColumnCompletionsMenu", +] + +E = KeyPressEvent + + +class CompletionsMenuControl(UIControl): + """ + Helper for drawing the complete menu to the screen. + + :param scroll_offset: Number (integer) representing the preferred amount of + completions to be displayed before and after the current one. When this + is a very high number, the current completion will be shown in the + middle most of the time. + """ + + # Preferred minimum size of the menu control. + # The CompletionsMenu class defines a width of 8, and there is a scrollbar + # of 1.) + MIN_WIDTH = 7 + + def has_focus(self) -> bool: + return False + + def preferred_width(self, max_available_width: int) -> int | None: + complete_state = get_app().current_buffer.complete_state + if complete_state: + menu_width = self._get_menu_width(500, complete_state) + menu_meta_width = self._get_menu_meta_width(500, complete_state) + + return menu_width + menu_meta_width + else: + return 0 + + def preferred_height( + self, + width: int, + max_available_height: int, + wrap_lines: bool, + get_line_prefix: GetLinePrefixCallable | None, + ) -> int | None: + complete_state = get_app().current_buffer.complete_state + if complete_state: + return len(complete_state.completions) + else: + return 0 + + def create_content(self, width: int, height: int) -> UIContent: + """ + Create a UIContent object for this control. + """ + complete_state = get_app().current_buffer.complete_state + if complete_state: + completions = complete_state.completions + index = complete_state.complete_index # Can be None! + + # Calculate width of completions menu. + menu_width = self._get_menu_width(width, complete_state) + menu_meta_width = self._get_menu_meta_width( + width - menu_width, complete_state + ) + show_meta = self._show_meta(complete_state) + + def get_line(i: int) -> StyleAndTextTuples: + c = completions[i] + is_current_completion = i == index + result = _get_menu_item_fragments( + c, is_current_completion, menu_width, space_after=True + ) + + if show_meta: + result += self._get_menu_item_meta_fragments( + c, is_current_completion, menu_meta_width + ) + return result + + return UIContent( + get_line=get_line, + cursor_position=Point(x=0, y=index or 0), + line_count=len(completions), + ) + + return UIContent() + + def _show_meta(self, complete_state: CompletionState) -> bool: + """ + Return ``True`` if we need to show a column with meta information. + """ + return any(c.display_meta_text for c in complete_state.completions) + + def _get_menu_width(self, max_width: int, complete_state: CompletionState) -> int: + """ + Return the width of the main column. + """ + return min( + max_width, + max( + self.MIN_WIDTH, + max(get_cwidth(c.display_text) for c in complete_state.completions) + 2, + ), + ) + + def _get_menu_meta_width( + self, max_width: int, complete_state: CompletionState + ) -> int: + """ + Return the width of the meta column. + """ + + def meta_width(completion: Completion) -> int: + return get_cwidth(completion.display_meta_text) + + if self._show_meta(complete_state): + # If the amount of completions is over 200, compute the width based + # on the first 200 completions, otherwise this can be very slow. + completions = complete_state.completions + if len(completions) > 200: + completions = completions[:200] + + return min(max_width, max(meta_width(c) for c in completions) + 2) + else: + return 0 + + def _get_menu_item_meta_fragments( + self, completion: Completion, is_current_completion: bool, width: int + ) -> StyleAndTextTuples: + if is_current_completion: + style_str = "class:completion-menu.meta.completion.current" + else: + style_str = "class:completion-menu.meta.completion" + + text, tw = _trim_formatted_text(completion.display_meta, width - 2) + padding = " " * (width - 1 - tw) + + return to_formatted_text( + cast(StyleAndTextTuples, []) + [("", " ")] + text + [("", padding)], + style=style_str, + ) + + def mouse_handler(self, mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Handle mouse events: clicking and scrolling. + """ + b = get_app().current_buffer + + if mouse_event.event_type == MouseEventType.MOUSE_UP: + # Select completion. + b.go_to_completion(mouse_event.position.y) + b.complete_state = None + + elif mouse_event.event_type == MouseEventType.SCROLL_DOWN: + # Scroll up. + b.complete_next(count=3, disable_wrap_around=True) + + elif mouse_event.event_type == MouseEventType.SCROLL_UP: + # Scroll down. + b.complete_previous(count=3, disable_wrap_around=True) + + return None + + +def _get_menu_item_fragments( + completion: Completion, + is_current_completion: bool, + width: int, + space_after: bool = False, +) -> StyleAndTextTuples: + """ + Get the style/text tuples for a menu item, styled and trimmed to the given + width. + """ + if is_current_completion: + style_str = f"class:completion-menu.completion.current {completion.style} {completion.selected_style}" + else: + style_str = "class:completion-menu.completion " + completion.style + + text, tw = _trim_formatted_text( + completion.display, (width - 2 if space_after else width - 1) + ) + + padding = " " * (width - 1 - tw) + + return to_formatted_text( + cast(StyleAndTextTuples, []) + [("", " ")] + text + [("", padding)], + style=style_str, + ) + + +def _trim_formatted_text( + formatted_text: StyleAndTextTuples, max_width: int +) -> tuple[StyleAndTextTuples, int]: + """ + Trim the text to `max_width`, append dots when the text is too long. + Returns (text, width) tuple. + """ + width = fragment_list_width(formatted_text) + + # When the text is too wide, trim it. + if width > max_width: + result = [] # Text fragments. + remaining_width = max_width - 3 + + for style_and_ch in explode_text_fragments(formatted_text): + ch_width = get_cwidth(style_and_ch[1]) + + if ch_width <= remaining_width: + result.append(style_and_ch) + remaining_width -= ch_width + else: + break + + result.append(("", "...")) + + return result, max_width - remaining_width + else: + return formatted_text, width + + +class CompletionsMenu(ConditionalContainer): + # NOTE: We use a pretty big z_index by default. Menus are supposed to be + # above anything else. We also want to make sure that the content is + # visible at the point where we draw this menu. + def __init__( + self, + max_height: int | None = None, + scroll_offset: int | Callable[[], int] = 0, + extra_filter: FilterOrBool = True, + display_arrows: FilterOrBool = False, + z_index: int = 10**8, + ) -> None: + extra_filter = to_filter(extra_filter) + display_arrows = to_filter(display_arrows) + + super().__init__( + content=Window( + content=CompletionsMenuControl(), + width=Dimension(min=8), + height=Dimension(min=1, max=max_height), + scroll_offsets=ScrollOffsets(top=scroll_offset, bottom=scroll_offset), + right_margins=[ScrollbarMargin(display_arrows=display_arrows)], + dont_extend_width=True, + style="class:completion-menu", + z_index=z_index, + ), + # Show when there are completions but not at the point we are + # returning the input. + filter=extra_filter & has_completions & ~is_done, + ) + + +class MultiColumnCompletionMenuControl(UIControl): + """ + Completion menu that displays all the completions in several columns. + When there are more completions than space for them to be displayed, an + arrow is shown on the left or right side. + + `min_rows` indicates how many rows will be available in any possible case. + When this is larger than one, it will try to use less columns and more + rows until this value is reached. + Be careful passing in a too big value, if less than the given amount of + rows are available, more columns would have been required, but + `preferred_width` doesn't know about that and reports a too small value. + This results in less completions displayed and additional scrolling. + (It's a limitation of how the layout engine currently works: first the + widths are calculated, then the heights.) + + :param suggested_max_column_width: The suggested max width of a column. + The column can still be bigger than this, but if there is place for two + columns of this width, we will display two columns. This to avoid that + if there is one very wide completion, that it doesn't significantly + reduce the amount of columns. + """ + + _required_margin = 3 # One extra padding on the right + space for arrows. + + def __init__(self, min_rows: int = 3, suggested_max_column_width: int = 30) -> None: + assert min_rows >= 1 + + self.min_rows = min_rows + self.suggested_max_column_width = suggested_max_column_width + self.scroll = 0 + + # Cache for column width computations. This computation is not cheap, + # so we don't want to do it over and over again while the user + # navigates through the completions. + # (map `completion_state` to `(completion_count, width)`. We remember + # the count, because a completer can add new completions to the + # `CompletionState` while loading.) + self._column_width_for_completion_state: WeakKeyDictionary[ + CompletionState, tuple[int, int] + ] = WeakKeyDictionary() + + # Info of last rendering. + self._rendered_rows = 0 + self._rendered_columns = 0 + self._total_columns = 0 + self._render_pos_to_completion: dict[tuple[int, int], Completion] = {} + self._render_left_arrow = False + self._render_right_arrow = False + self._render_width = 0 + + def reset(self) -> None: + self.scroll = 0 + + def has_focus(self) -> bool: + return False + + def preferred_width(self, max_available_width: int) -> int | None: + """ + Preferred width: prefer to use at least min_rows, but otherwise as much + as possible horizontally. + """ + complete_state = get_app().current_buffer.complete_state + if complete_state is None: + return 0 + + column_width = self._get_column_width(complete_state) + result = int( + column_width + * math.ceil(len(complete_state.completions) / float(self.min_rows)) + ) + + # When the desired width is still more than the maximum available, + # reduce by removing columns until we are less than the available + # width. + while ( + result > column_width + and result > max_available_width - self._required_margin + ): + result -= column_width + return result + self._required_margin + + def preferred_height( + self, + width: int, + max_available_height: int, + wrap_lines: bool, + get_line_prefix: GetLinePrefixCallable | None, + ) -> int | None: + """ + Preferred height: as much as needed in order to display all the completions. + """ + complete_state = get_app().current_buffer.complete_state + if complete_state is None: + return 0 + + column_width = self._get_column_width(complete_state) + column_count = max(1, (width - self._required_margin) // column_width) + + return int(math.ceil(len(complete_state.completions) / float(column_count))) + + def create_content(self, width: int, height: int) -> UIContent: + """ + Create a UIContent object for this menu. + """ + complete_state = get_app().current_buffer.complete_state + if complete_state is None: + return UIContent() + + column_width = self._get_column_width(complete_state) + self._render_pos_to_completion = {} + + _T = TypeVar("_T") + + def grouper( + n: int, iterable: Iterable[_T], fillvalue: _T | None = None + ) -> Iterable[Sequence[_T | None]]: + "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return zip_longest(fillvalue=fillvalue, *args) + + def is_current_completion(completion: Completion) -> bool: + "Returns True when this completion is the currently selected one." + return ( + complete_state is not None + and complete_state.complete_index is not None + and c == complete_state.current_completion + ) + + # Space required outside of the regular columns, for displaying the + # left and right arrow. + HORIZONTAL_MARGIN_REQUIRED = 3 + + # There should be at least one column, but it cannot be wider than + # the available width. + column_width = min(width - HORIZONTAL_MARGIN_REQUIRED, column_width) + + # However, when the columns tend to be very wide, because there are + # some very wide entries, shrink it anyway. + if column_width > self.suggested_max_column_width: + # `column_width` can still be bigger that `suggested_max_column_width`, + # but if there is place for two columns, we divide by two. + column_width //= column_width // self.suggested_max_column_width + + visible_columns = max(1, (width - self._required_margin) // column_width) + + columns_ = list(grouper(height, complete_state.completions)) + rows_ = list(zip(*columns_)) + + # Make sure the current completion is always visible: update scroll offset. + selected_column = (complete_state.complete_index or 0) // height + self.scroll = min( + selected_column, max(self.scroll, selected_column - visible_columns + 1) + ) + + render_left_arrow = self.scroll > 0 + render_right_arrow = self.scroll < len(rows_[0]) - visible_columns + + # Write completions to screen. + fragments_for_line = [] + + for row_index, row in enumerate(rows_): + fragments: StyleAndTextTuples = [] + middle_row = row_index == len(rows_) // 2 + + # Draw left arrow if we have hidden completions on the left. + if render_left_arrow: + fragments.append(("class:scrollbar", "<" if middle_row else " ")) + elif render_right_arrow: + # Reserve one column empty space. (If there is a right + # arrow right now, there can be a left arrow as well.) + fragments.append(("", " ")) + + # Draw row content. + for column_index, c in enumerate(row[self.scroll :][:visible_columns]): + if c is not None: + fragments += _get_menu_item_fragments( + c, is_current_completion(c), column_width, space_after=False + ) + + # Remember render position for mouse click handler. + for x in range(column_width): + self._render_pos_to_completion[ + (column_index * column_width + x, row_index) + ] = c + else: + fragments.append(("class:completion", " " * column_width)) + + # Draw trailing padding for this row. + # (_get_menu_item_fragments only returns padding on the left.) + if render_left_arrow or render_right_arrow: + fragments.append(("class:completion", " ")) + + # Draw right arrow if we have hidden completions on the right. + if render_right_arrow: + fragments.append(("class:scrollbar", ">" if middle_row else " ")) + elif render_left_arrow: + fragments.append(("class:completion", " ")) + + # Add line. + fragments_for_line.append( + to_formatted_text(fragments, style="class:completion-menu") + ) + + self._rendered_rows = height + self._rendered_columns = visible_columns + self._total_columns = len(columns_) + self._render_left_arrow = render_left_arrow + self._render_right_arrow = render_right_arrow + self._render_width = ( + column_width * visible_columns + render_left_arrow + render_right_arrow + 1 + ) + + def get_line(i: int) -> StyleAndTextTuples: + return fragments_for_line[i] + + return UIContent(get_line=get_line, line_count=len(rows_)) + + def _get_column_width(self, completion_state: CompletionState) -> int: + """ + Return the width of each column. + """ + try: + count, width = self._column_width_for_completion_state[completion_state] + if count != len(completion_state.completions): + # Number of completions changed, recompute. + raise KeyError + return width + except KeyError: + result = ( + max(get_cwidth(c.display_text) for c in completion_state.completions) + + 1 + ) + self._column_width_for_completion_state[completion_state] = ( + len(completion_state.completions), + result, + ) + return result + + def mouse_handler(self, mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + Handle scroll and click events. + """ + b = get_app().current_buffer + + def scroll_left() -> None: + b.complete_previous(count=self._rendered_rows, disable_wrap_around=True) + self.scroll = max(0, self.scroll - 1) + + def scroll_right() -> None: + b.complete_next(count=self._rendered_rows, disable_wrap_around=True) + self.scroll = min( + self._total_columns - self._rendered_columns, self.scroll + 1 + ) + + if mouse_event.event_type == MouseEventType.SCROLL_DOWN: + scroll_right() + + elif mouse_event.event_type == MouseEventType.SCROLL_UP: + scroll_left() + + elif mouse_event.event_type == MouseEventType.MOUSE_UP: + x = mouse_event.position.x + y = mouse_event.position.y + + # Mouse click on left arrow. + if x == 0: + if self._render_left_arrow: + scroll_left() + + # Mouse click on right arrow. + elif x == self._render_width - 1: + if self._render_right_arrow: + scroll_right() + + # Mouse click on completion. + else: + completion = self._render_pos_to_completion.get((x, y)) + if completion: + b.apply_completion(completion) + + return None + + def get_key_bindings(self) -> KeyBindings: + """ + Expose key bindings that handle the left/right arrow keys when the menu + is displayed. + """ + from prompt_toolkit.key_binding.key_bindings import KeyBindings + + kb = KeyBindings() + + @Condition + def filter() -> bool: + "Only handle key bindings if this menu is visible." + app = get_app() + complete_state = app.current_buffer.complete_state + + # There need to be completions, and one needs to be selected. + if complete_state is None or complete_state.complete_index is None: + return False + + # This menu needs to be visible. + return any(window.content == self for window in app.layout.visible_windows) + + def move(right: bool = False) -> None: + buff = get_app().current_buffer + complete_state = buff.complete_state + + if complete_state is not None and complete_state.complete_index is not None: + # Calculate new complete index. + new_index = complete_state.complete_index + if right: + new_index += self._rendered_rows + else: + new_index -= self._rendered_rows + + if 0 <= new_index < len(complete_state.completions): + buff.go_to_completion(new_index) + + # NOTE: the is_global is required because the completion menu will + # never be focussed. + + @kb.add("left", is_global=True, filter=filter) + def _left(event: E) -> None: + move() + + @kb.add("right", is_global=True, filter=filter) + def _right(event: E) -> None: + move(True) + + return kb + + +class MultiColumnCompletionsMenu(HSplit): + """ + Container that displays the completions in several columns. + When `show_meta` (a :class:`~prompt_toolkit.filters.Filter`) evaluates + to True, it shows the meta information at the bottom. + """ + + def __init__( + self, + min_rows: int = 3, + suggested_max_column_width: int = 30, + show_meta: FilterOrBool = True, + extra_filter: FilterOrBool = True, + z_index: int = 10**8, + ) -> None: + show_meta = to_filter(show_meta) + extra_filter = to_filter(extra_filter) + + # Display filter: show when there are completions but not at the point + # we are returning the input. + full_filter = extra_filter & has_completions & ~is_done + + @Condition + def any_completion_has_meta() -> bool: + complete_state = get_app().current_buffer.complete_state + return complete_state is not None and any( + c.display_meta for c in complete_state.completions + ) + + # Create child windows. + # NOTE: We don't set style='class:completion-menu' to the + # `MultiColumnCompletionMenuControl`, because this is used in a + # Float that is made transparent, and the size of the control + # doesn't always correspond exactly with the size of the + # generated content. + completions_window = ConditionalContainer( + content=Window( + content=MultiColumnCompletionMenuControl( + min_rows=min_rows, + suggested_max_column_width=suggested_max_column_width, + ), + width=Dimension(min=8), + height=Dimension(min=1), + ), + filter=full_filter, + ) + + meta_window = ConditionalContainer( + content=Window(content=_SelectedCompletionMetaControl()), + filter=full_filter & show_meta & any_completion_has_meta, + ) + + # Initialize split. + super().__init__([completions_window, meta_window], z_index=z_index) + + +class _SelectedCompletionMetaControl(UIControl): + """ + Control that shows the meta information of the selected completion. + """ + + def preferred_width(self, max_available_width: int) -> int | None: + """ + Report the width of the longest meta text as the preferred width of this control. + + It could be that we use less width, but this way, we're sure that the + layout doesn't change when we select another completion (E.g. that + completions are suddenly shown in more or fewer columns.) + """ + app = get_app() + if app.current_buffer.complete_state: + state = app.current_buffer.complete_state + + if len(state.completions) >= 30: + # When there are many completions, calling `get_cwidth` for + # every `display_meta_text` is too expensive. In this case, + # just return the max available width. There will be enough + # columns anyway so that the whole screen is filled with + # completions and `create_content` will then take up as much + # space as needed. + return max_available_width + + return 2 + max( + get_cwidth(c.display_meta_text) for c in state.completions[:100] + ) + else: + return 0 + + def preferred_height( + self, + width: int, + max_available_height: int, + wrap_lines: bool, + get_line_prefix: GetLinePrefixCallable | None, + ) -> int | None: + return 1 + + def create_content(self, width: int, height: int) -> UIContent: + fragments = self._get_text_fragments() + + def get_line(i: int) -> StyleAndTextTuples: + return fragments + + return UIContent(get_line=get_line, line_count=1 if fragments else 0) + + def _get_text_fragments(self) -> StyleAndTextTuples: + style = "class:completion-menu.multi-column-meta" + state = get_app().current_buffer.complete_state + + if ( + state + and state.current_completion + and state.current_completion.display_meta_text + ): + return to_formatted_text( + cast(StyleAndTextTuples, [("", " ")]) + + state.current_completion.display_meta + + [("", " ")], + style=style, + ) + + return [] diff --git a/lib/prompt_toolkit/layout/mouse_handlers.py b/lib/prompt_toolkit/layout/mouse_handlers.py new file mode 100644 index 0000000..52deac1 --- /dev/null +++ b/lib/prompt_toolkit/layout/mouse_handlers.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING, Callable + +from prompt_toolkit.mouse_events import MouseEvent + +if TYPE_CHECKING: + from prompt_toolkit.key_binding.key_bindings import NotImplementedOrNone + +__all__ = [ + "MouseHandler", + "MouseHandlers", +] + + +MouseHandler = Callable[[MouseEvent], "NotImplementedOrNone"] + + +class MouseHandlers: + """ + Two dimensional raster of callbacks for mouse events. + """ + + def __init__(self) -> None: + def dummy_callback(mouse_event: MouseEvent) -> NotImplementedOrNone: + """ + :param mouse_event: `MouseEvent` instance. + """ + return NotImplemented + + # NOTE: Previously, the data structure was a dictionary mapping (x,y) + # to the handlers. This however would be more inefficient when copying + # over the mouse handlers of the visible region in the scrollable pane. + + # Map y (row) to x (column) to handlers. + self.mouse_handlers: defaultdict[int, defaultdict[int, MouseHandler]] = ( + defaultdict(lambda: defaultdict(lambda: dummy_callback)) + ) + + def set_mouse_handler_for_range( + self, + x_min: int, + x_max: int, + y_min: int, + y_max: int, + handler: Callable[[MouseEvent], NotImplementedOrNone], + ) -> None: + """ + Set mouse handler for a region. + """ + for y in range(y_min, y_max): + row = self.mouse_handlers[y] + + for x in range(x_min, x_max): + row[x] = handler diff --git a/lib/prompt_toolkit/layout/processors.py b/lib/prompt_toolkit/layout/processors.py new file mode 100644 index 0000000..666e79c --- /dev/null +++ b/lib/prompt_toolkit/layout/processors.py @@ -0,0 +1,1016 @@ +""" +Processors are little transformation blocks that transform the fragments list +from a buffer before the BufferControl will render it to the screen. + +They can insert fragments before or after, or highlight fragments by replacing the +fragment types. +""" + +from __future__ import annotations + +import re +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Callable, Hashable, cast + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.cache import SimpleCache +from prompt_toolkit.document import Document +from prompt_toolkit.filters import FilterOrBool, to_filter, vi_insert_multiple_mode +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + to_formatted_text, +) +from prompt_toolkit.formatted_text.utils import fragment_list_len, fragment_list_to_text +from prompt_toolkit.search import SearchDirection +from prompt_toolkit.utils import to_int, to_str + +from .utils import explode_text_fragments + +if TYPE_CHECKING: + from .controls import BufferControl, UIContent + +__all__ = [ + "Processor", + "TransformationInput", + "Transformation", + "DummyProcessor", + "HighlightSearchProcessor", + "HighlightIncrementalSearchProcessor", + "HighlightSelectionProcessor", + "PasswordProcessor", + "HighlightMatchingBracketProcessor", + "DisplayMultipleCursors", + "BeforeInput", + "ShowArg", + "AfterInput", + "AppendAutoSuggestion", + "ConditionalProcessor", + "ShowLeadingWhiteSpaceProcessor", + "ShowTrailingWhiteSpaceProcessor", + "TabsProcessor", + "ReverseSearchProcessor", + "DynamicProcessor", + "merge_processors", +] + + +class Processor(metaclass=ABCMeta): + """ + Manipulate the fragments for a given line in a + :class:`~prompt_toolkit.layout.controls.BufferControl`. + """ + + @abstractmethod + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + """ + Apply transformation. Returns a :class:`.Transformation` instance. + + :param transformation_input: :class:`.TransformationInput` object. + """ + return Transformation(transformation_input.fragments) + + +SourceToDisplay = Callable[[int], int] +DisplayToSource = Callable[[int], int] + + +class TransformationInput: + """ + :param buffer_control: :class:`.BufferControl` instance. + :param lineno: The number of the line to which we apply the processor. + :param source_to_display: A function that returns the position in the + `fragments` for any position in the source string. (This takes + previous processors into account.) + :param fragments: List of fragments that we can transform. (Received from the + previous processor.) + :param get_line: Optional ; a callable that returns the fragments of another + line in the current buffer; This can be used to create processors capable + of affecting transforms across multiple lines. + """ + + def __init__( + self, + buffer_control: BufferControl, + document: Document, + lineno: int, + source_to_display: SourceToDisplay, + fragments: StyleAndTextTuples, + width: int, + height: int, + get_line: Callable[[int], StyleAndTextTuples] | None = None, + ) -> None: + self.buffer_control = buffer_control + self.document = document + self.lineno = lineno + self.source_to_display = source_to_display + self.fragments = fragments + self.width = width + self.height = height + self.get_line = get_line + + def unpack( + self, + ) -> tuple[ + BufferControl, Document, int, SourceToDisplay, StyleAndTextTuples, int, int + ]: + return ( + self.buffer_control, + self.document, + self.lineno, + self.source_to_display, + self.fragments, + self.width, + self.height, + ) + + +class Transformation: + """ + Transformation result, as returned by :meth:`.Processor.apply_transformation`. + + Important: Always make sure that the length of `document.text` is equal to + the length of all the text in `fragments`! + + :param fragments: The transformed fragments. To be displayed, or to pass to + the next processor. + :param source_to_display: Cursor position transformation from original + string to transformed string. + :param display_to_source: Cursor position transformed from source string to + original string. + """ + + def __init__( + self, + fragments: StyleAndTextTuples, + source_to_display: SourceToDisplay | None = None, + display_to_source: DisplayToSource | None = None, + ) -> None: + self.fragments = fragments + self.source_to_display = source_to_display or (lambda i: i) + self.display_to_source = display_to_source or (lambda i: i) + + +class DummyProcessor(Processor): + """ + A `Processor` that doesn't do anything. + """ + + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + return Transformation(transformation_input.fragments) + + +class HighlightSearchProcessor(Processor): + """ + Processor that highlights search matches in the document. + Note that this doesn't support multiline search matches yet. + + The style classes 'search' and 'search.current' will be applied to the + content. + """ + + _classname = "search" + _classname_current = "search.current" + + def _get_search_text(self, buffer_control: BufferControl) -> str: + """ + The text we are searching for. + """ + return buffer_control.search_state.text + + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + ( + buffer_control, + document, + lineno, + source_to_display, + fragments, + _, + _, + ) = transformation_input.unpack() + + search_text = self._get_search_text(buffer_control) + searchmatch_fragment = f" class:{self._classname} " + searchmatch_current_fragment = f" class:{self._classname_current} " + + if search_text and not get_app().is_done: + # For each search match, replace the style string. + line_text = fragment_list_to_text(fragments) + fragments = explode_text_fragments(fragments) + + if buffer_control.search_state.ignore_case(): + flags = re.IGNORECASE + else: + flags = re.RegexFlag(0) + + # Get cursor column. + cursor_column: int | None + if document.cursor_position_row == lineno: + cursor_column = source_to_display(document.cursor_position_col) + else: + cursor_column = None + + for match in re.finditer(re.escape(search_text), line_text, flags=flags): + if cursor_column is not None: + on_cursor = match.start() <= cursor_column < match.end() + else: + on_cursor = False + + for i in range(match.start(), match.end()): + old_fragment, text, *_ = fragments[i] + if on_cursor: + fragments[i] = ( + old_fragment + searchmatch_current_fragment, + fragments[i][1], + ) + else: + fragments[i] = ( + old_fragment + searchmatch_fragment, + fragments[i][1], + ) + + return Transformation(fragments) + + +class HighlightIncrementalSearchProcessor(HighlightSearchProcessor): + """ + Highlight the search terms that are used for highlighting the incremental + search. The style class 'incsearch' will be applied to the content. + + Important: this requires the `preview_search=True` flag to be set for the + `BufferControl`. Otherwise, the cursor position won't be set to the search + match while searching, and nothing happens. + """ + + _classname = "incsearch" + _classname_current = "incsearch.current" + + def _get_search_text(self, buffer_control: BufferControl) -> str: + """ + The text we are searching for. + """ + # When the search buffer has focus, take that text. + search_buffer = buffer_control.search_buffer + if search_buffer is not None and search_buffer.text: + return search_buffer.text + return "" + + +class HighlightSelectionProcessor(Processor): + """ + Processor that highlights the selection in the document. + """ + + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + ( + buffer_control, + document, + lineno, + source_to_display, + fragments, + _, + _, + ) = transformation_input.unpack() + + selected_fragment = " class:selected " + + # In case of selection, highlight all matches. + selection_at_line = document.selection_range_at_line(lineno) + + if selection_at_line: + from_, to = selection_at_line + from_ = source_to_display(from_) + to = source_to_display(to) + + fragments = explode_text_fragments(fragments) + + if from_ == 0 and to == 0 and len(fragments) == 0: + # When this is an empty line, insert a space in order to + # visualize the selection. + return Transformation([(selected_fragment, " ")]) + else: + for i in range(from_, to): + if i < len(fragments): + old_fragment, old_text, *_ = fragments[i] + fragments[i] = (old_fragment + selected_fragment, old_text) + elif i == len(fragments): + fragments.append((selected_fragment, " ")) + + return Transformation(fragments) + + +class PasswordProcessor(Processor): + """ + Processor that masks the input. (For passwords.) + + :param char: (string) Character to be used. "*" by default. + """ + + def __init__(self, char: str = "*") -> None: + self.char = char + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + fragments: StyleAndTextTuples = cast( + StyleAndTextTuples, + [ + (style, self.char * len(text), *handler) + for style, text, *handler in ti.fragments + ], + ) + + return Transformation(fragments) + + +class HighlightMatchingBracketProcessor(Processor): + """ + When the cursor is on or right after a bracket, it highlights the matching + bracket. + + :param max_cursor_distance: Only highlight matching brackets when the + cursor is within this distance. (From inside a `Processor`, we can't + know which lines will be visible on the screen. But we also don't want + to scan the whole document for matching brackets on each key press, so + we limit to this value.) + """ + + _closing_braces = "])}>" + + def __init__( + self, chars: str = "[](){}<>", max_cursor_distance: int = 1000 + ) -> None: + self.chars = chars + self.max_cursor_distance = max_cursor_distance + + self._positions_cache: SimpleCache[Hashable, list[tuple[int, int]]] = ( + SimpleCache(maxsize=8) + ) + + def _get_positions_to_highlight(self, document: Document) -> list[tuple[int, int]]: + """ + Return a list of (row, col) tuples that need to be highlighted. + """ + pos: int | None + + # Try for the character under the cursor. + if document.current_char and document.current_char in self.chars: + pos = document.find_matching_bracket_position( + start_pos=document.cursor_position - self.max_cursor_distance, + end_pos=document.cursor_position + self.max_cursor_distance, + ) + + # Try for the character before the cursor. + elif ( + document.char_before_cursor + and document.char_before_cursor in self._closing_braces + and document.char_before_cursor in self.chars + ): + document = Document(document.text, document.cursor_position - 1) + + pos = document.find_matching_bracket_position( + start_pos=document.cursor_position - self.max_cursor_distance, + end_pos=document.cursor_position + self.max_cursor_distance, + ) + else: + pos = None + + # Return a list of (row, col) tuples that need to be highlighted. + if pos: + pos += document.cursor_position # pos is relative. + row, col = document.translate_index_to_position(pos) + return [ + (row, col), + (document.cursor_position_row, document.cursor_position_col), + ] + else: + return [] + + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + ( + buffer_control, + document, + lineno, + source_to_display, + fragments, + _, + _, + ) = transformation_input.unpack() + + # When the application is in the 'done' state, don't highlight. + if get_app().is_done: + return Transformation(fragments) + + # Get the highlight positions. + key = (get_app().render_counter, document.text, document.cursor_position) + positions = self._positions_cache.get( + key, lambda: self._get_positions_to_highlight(document) + ) + + # Apply if positions were found at this line. + if positions: + for row, col in positions: + if row == lineno: + col = source_to_display(col) + fragments = explode_text_fragments(fragments) + style, text, *_ = fragments[col] + + if col == document.cursor_position_col: + style += " class:matching-bracket.cursor " + else: + style += " class:matching-bracket.other " + + fragments[col] = (style, text) + + return Transformation(fragments) + + +class DisplayMultipleCursors(Processor): + """ + When we're in Vi block insert mode, display all the cursors. + """ + + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + ( + buffer_control, + document, + lineno, + source_to_display, + fragments, + _, + _, + ) = transformation_input.unpack() + + buff = buffer_control.buffer + + if vi_insert_multiple_mode(): + cursor_positions = buff.multiple_cursor_positions + fragments = explode_text_fragments(fragments) + + # If any cursor appears on the current line, highlight that. + start_pos = document.translate_row_col_to_index(lineno, 0) + end_pos = start_pos + len(document.lines[lineno]) + + fragment_suffix = " class:multiple-cursors" + + for p in cursor_positions: + if start_pos <= p <= end_pos: + column = source_to_display(p - start_pos) + + # Replace fragment. + try: + style, text, *_ = fragments[column] + except IndexError: + # Cursor needs to be displayed after the current text. + fragments.append((fragment_suffix, " ")) + else: + style += fragment_suffix + fragments[column] = (style, text) + + return Transformation(fragments) + else: + return Transformation(fragments) + + +class BeforeInput(Processor): + """ + Insert text before the input. + + :param text: This can be either plain text or formatted text + (or a callable that returns any of those). + :param style: style to be applied to this prompt/prefix. + """ + + def __init__(self, text: AnyFormattedText, style: str = "") -> None: + self.text = text + self.style = style + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + source_to_display: SourceToDisplay | None + display_to_source: DisplayToSource | None + + if ti.lineno == 0: + # Get fragments. + fragments_before = to_formatted_text(self.text, self.style) + fragments = fragments_before + ti.fragments + + shift_position = fragment_list_len(fragments_before) + source_to_display = lambda i: i + shift_position + display_to_source = lambda i: i - shift_position + else: + fragments = ti.fragments + source_to_display = None + display_to_source = None + + return Transformation( + fragments, + source_to_display=source_to_display, + display_to_source=display_to_source, + ) + + def __repr__(self) -> str: + return f"BeforeInput({self.text!r}, {self.style!r})" + + +class ShowArg(BeforeInput): + """ + Display the 'arg' in front of the input. + + This was used by the `PromptSession`, but now it uses the + `Window.get_line_prefix` function instead. + """ + + def __init__(self) -> None: + super().__init__(self._get_text_fragments) + + def _get_text_fragments(self) -> StyleAndTextTuples: + app = get_app() + if app.key_processor.arg is None: + return [] + else: + arg = app.key_processor.arg + + return [ + ("class:prompt.arg", "(arg: "), + ("class:prompt.arg.text", str(arg)), + ("class:prompt.arg", ") "), + ] + + def __repr__(self) -> str: + return "ShowArg()" + + +class AfterInput(Processor): + """ + Insert text after the input. + + :param text: This can be either plain text or formatted text + (or a callable that returns any of those). + :param style: style to be applied to this prompt/prefix. + """ + + def __init__(self, text: AnyFormattedText, style: str = "") -> None: + self.text = text + self.style = style + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + # Insert fragments after the last line. + if ti.lineno == ti.document.line_count - 1: + # Get fragments. + fragments_after = to_formatted_text(self.text, self.style) + return Transformation(fragments=ti.fragments + fragments_after) + else: + return Transformation(fragments=ti.fragments) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.text!r}, style={self.style!r})" + + +class AppendAutoSuggestion(Processor): + """ + Append the auto suggestion to the input. + (The user can then press the right arrow the insert the suggestion.) + """ + + def __init__(self, style: str = "class:auto-suggestion") -> None: + self.style = style + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + # Insert fragments after the last line. + if ti.lineno == ti.document.line_count - 1: + buffer = ti.buffer_control.buffer + + if buffer.suggestion and ti.document.is_cursor_at_the_end: + suggestion = buffer.suggestion.text + else: + suggestion = "" + + return Transformation(fragments=ti.fragments + [(self.style, suggestion)]) + else: + return Transformation(fragments=ti.fragments) + + +class ShowLeadingWhiteSpaceProcessor(Processor): + """ + Make leading whitespace visible. + + :param get_char: Callable that returns one character. + """ + + def __init__( + self, + get_char: Callable[[], str] | None = None, + style: str = "class:leading-whitespace", + ) -> None: + def default_get_char() -> str: + if "\xb7".encode(get_app().output.encoding(), "replace") == b"?": + return "." + else: + return "\xb7" + + self.style = style + self.get_char = get_char or default_get_char + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + fragments = ti.fragments + + # Walk through all te fragments. + if fragments and fragment_list_to_text(fragments).startswith(" "): + t = (self.style, self.get_char()) + fragments = explode_text_fragments(fragments) + + for i in range(len(fragments)): + if fragments[i][1] == " ": + fragments[i] = t + else: + break + + return Transformation(fragments) + + +class ShowTrailingWhiteSpaceProcessor(Processor): + """ + Make trailing whitespace visible. + + :param get_char: Callable that returns one character. + """ + + def __init__( + self, + get_char: Callable[[], str] | None = None, + style: str = "class:training-whitespace", + ) -> None: + def default_get_char() -> str: + if "\xb7".encode(get_app().output.encoding(), "replace") == b"?": + return "." + else: + return "\xb7" + + self.style = style + self.get_char = get_char or default_get_char + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + fragments = ti.fragments + + if fragments and fragments[-1][1].endswith(" "): + t = (self.style, self.get_char()) + fragments = explode_text_fragments(fragments) + + # Walk backwards through all te fragments and replace whitespace. + for i in range(len(fragments) - 1, -1, -1): + char = fragments[i][1] + if char == " ": + fragments[i] = t + else: + break + + return Transformation(fragments) + + +class TabsProcessor(Processor): + """ + Render tabs as spaces (instead of ^I) or make them visible (for instance, + by replacing them with dots.) + + :param tabstop: Horizontal space taken by a tab. (`int` or callable that + returns an `int`). + :param char1: Character or callable that returns a character (text of + length one). This one is used for the first space taken by the tab. + :param char2: Like `char1`, but for the rest of the space. + """ + + def __init__( + self, + tabstop: int | Callable[[], int] = 4, + char1: str | Callable[[], str] = "|", + char2: str | Callable[[], str] = "\u2508", + style: str = "class:tab", + ) -> None: + self.char1 = char1 + self.char2 = char2 + self.tabstop = tabstop + self.style = style + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + tabstop = to_int(self.tabstop) + style = self.style + + # Create separator for tabs. + separator1 = to_str(self.char1) + separator2 = to_str(self.char2) + + # Transform fragments. + fragments = explode_text_fragments(ti.fragments) + + position_mappings = {} + result_fragments: StyleAndTextTuples = [] + pos = 0 + + for i, fragment_and_text in enumerate(fragments): + position_mappings[i] = pos + + if fragment_and_text[1] == "\t": + # Calculate how many characters we have to insert. + count = tabstop - (pos % tabstop) + if count == 0: + count = tabstop + + # Insert tab. + result_fragments.append((style, separator1)) + result_fragments.append((style, separator2 * (count - 1))) + pos += count + else: + result_fragments.append(fragment_and_text) + pos += 1 + + position_mappings[len(fragments)] = pos + # Add `pos+1` to mapping, because the cursor can be right after the + # line as well. + position_mappings[len(fragments) + 1] = pos + 1 + + def source_to_display(from_position: int) -> int: + "Maps original cursor position to the new one." + return position_mappings[from_position] + + def display_to_source(display_pos: int) -> int: + "Maps display cursor position to the original one." + position_mappings_reversed = {v: k for k, v in position_mappings.items()} + + while display_pos >= 0: + try: + return position_mappings_reversed[display_pos] + except KeyError: + display_pos -= 1 + return 0 + + return Transformation( + result_fragments, + source_to_display=source_to_display, + display_to_source=display_to_source, + ) + + +class ReverseSearchProcessor(Processor): + """ + Process to display the "(reverse-i-search)`...`:..." stuff around + the search buffer. + + Note: This processor is meant to be applied to the BufferControl that + contains the search buffer, it's not meant for the original input. + """ + + _excluded_input_processors: list[type[Processor]] = [ + HighlightSearchProcessor, + HighlightSelectionProcessor, + BeforeInput, + AfterInput, + ] + + def _get_main_buffer(self, buffer_control: BufferControl) -> BufferControl | None: + from prompt_toolkit.layout.controls import BufferControl + + prev_control = get_app().layout.search_target_buffer_control + if ( + isinstance(prev_control, BufferControl) + and prev_control.search_buffer_control == buffer_control + ): + return prev_control + return None + + def _content( + self, main_control: BufferControl, ti: TransformationInput + ) -> UIContent: + from prompt_toolkit.layout.controls import BufferControl + + # Emulate the BufferControl through which we are searching. + # For this we filter out some of the input processors. + excluded_processors = tuple(self._excluded_input_processors) + + def filter_processor(item: Processor) -> Processor | None: + """Filter processors from the main control that we want to disable + here. This returns either an accepted processor or None.""" + # For a `_MergedProcessor`, check each individual processor, recursively. + if isinstance(item, _MergedProcessor): + accepted_processors = [filter_processor(p) for p in item.processors] + return merge_processors( + [p for p in accepted_processors if p is not None] + ) + + # For a `ConditionalProcessor`, check the body. + elif isinstance(item, ConditionalProcessor): + p = filter_processor(item.processor) + if p: + return ConditionalProcessor(p, item.filter) + + # Otherwise, check the processor itself. + else: + if not isinstance(item, excluded_processors): + return item + + return None + + filtered_processor = filter_processor( + merge_processors(main_control.input_processors or []) + ) + highlight_processor = HighlightIncrementalSearchProcessor() + + if filtered_processor: + new_processors = [filtered_processor, highlight_processor] + else: + new_processors = [highlight_processor] + + from .controls import SearchBufferControl + + assert isinstance(ti.buffer_control, SearchBufferControl) + + buffer_control = BufferControl( + buffer=main_control.buffer, + input_processors=new_processors, + include_default_input_processors=False, + lexer=main_control.lexer, + preview_search=True, + search_buffer_control=ti.buffer_control, + ) + + return buffer_control.create_content(ti.width, ti.height, preview_search=True) + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + from .controls import SearchBufferControl + + assert isinstance(ti.buffer_control, SearchBufferControl), ( + "`ReverseSearchProcessor` should be applied to a `SearchBufferControl` only." + ) + + source_to_display: SourceToDisplay | None + display_to_source: DisplayToSource | None + + main_control = self._get_main_buffer(ti.buffer_control) + + if ti.lineno == 0 and main_control: + content = self._content(main_control, ti) + + # Get the line from the original document for this search. + line_fragments = content.get_line(content.cursor_position.y) + + if main_control.search_state.direction == SearchDirection.FORWARD: + direction_text = "i-search" + else: + direction_text = "reverse-i-search" + + fragments_before: StyleAndTextTuples = [ + ("class:prompt.search", "("), + ("class:prompt.search", direction_text), + ("class:prompt.search", ")`"), + ] + + fragments = ( + fragments_before + + [ + ("class:prompt.search.text", fragment_list_to_text(ti.fragments)), + ("", "': "), + ] + + line_fragments + ) + + shift_position = fragment_list_len(fragments_before) + source_to_display = lambda i: i + shift_position + display_to_source = lambda i: i - shift_position + else: + source_to_display = None + display_to_source = None + fragments = ti.fragments + + return Transformation( + fragments, + source_to_display=source_to_display, + display_to_source=display_to_source, + ) + + +class ConditionalProcessor(Processor): + """ + Processor that applies another processor, according to a certain condition. + Example:: + + # Create a function that returns whether or not the processor should + # currently be applied. + def highlight_enabled(): + return true_or_false + + # Wrapped it in a `ConditionalProcessor` for usage in a `BufferControl`. + BufferControl(input_processors=[ + ConditionalProcessor(HighlightSearchProcessor(), + Condition(highlight_enabled))]) + + :param processor: :class:`.Processor` instance. + :param filter: :class:`~prompt_toolkit.filters.Filter` instance. + """ + + def __init__(self, processor: Processor, filter: FilterOrBool) -> None: + self.processor = processor + self.filter = to_filter(filter) + + def apply_transformation( + self, transformation_input: TransformationInput + ) -> Transformation: + # Run processor when enabled. + if self.filter(): + return self.processor.apply_transformation(transformation_input) + else: + return Transformation(transformation_input.fragments) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(processor={self.processor!r}, filter={self.filter!r})" + + +class DynamicProcessor(Processor): + """ + Processor class that dynamically returns any Processor. + + :param get_processor: Callable that returns a :class:`.Processor` instance. + """ + + def __init__(self, get_processor: Callable[[], Processor | None]) -> None: + self.get_processor = get_processor + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + processor = self.get_processor() or DummyProcessor() + return processor.apply_transformation(ti) + + +def merge_processors(processors: list[Processor]) -> Processor: + """ + Merge multiple `Processor` objects into one. + """ + if len(processors) == 0: + return DummyProcessor() + + if len(processors) == 1: + return processors[0] # Nothing to merge. + + return _MergedProcessor(processors) + + +class _MergedProcessor(Processor): + """ + Processor that groups multiple other `Processor` objects, but exposes an + API as if it is one `Processor`. + """ + + def __init__(self, processors: list[Processor]): + self.processors = processors + + def apply_transformation(self, ti: TransformationInput) -> Transformation: + source_to_display_functions = [ti.source_to_display] + display_to_source_functions = [] + fragments = ti.fragments + + def source_to_display(i: int) -> int: + """Translate x position from the buffer to the x position in the + processor fragments list.""" + for f in source_to_display_functions: + i = f(i) + return i + + for p in self.processors: + transformation = p.apply_transformation( + TransformationInput( + ti.buffer_control, + ti.document, + ti.lineno, + source_to_display, + fragments, + ti.width, + ti.height, + ti.get_line, + ) + ) + fragments = transformation.fragments + display_to_source_functions.append(transformation.display_to_source) + source_to_display_functions.append(transformation.source_to_display) + + def display_to_source(i: int) -> int: + for f in reversed(display_to_source_functions): + i = f(i) + return i + + # In the case of a nested _MergedProcessor, each processor wants to + # receive a 'source_to_display' function (as part of the + # TransformationInput) that has everything in the chain before + # included, because it can be called as part of the + # `apply_transformation` function. However, this first + # `source_to_display` should not be part of the output that we are + # returning. (This is the most consistent with `display_to_source`.) + del source_to_display_functions[:1] + + return Transformation(fragments, source_to_display, display_to_source) diff --git a/lib/prompt_toolkit/layout/screen.py b/lib/prompt_toolkit/layout/screen.py new file mode 100644 index 0000000..475f540 --- /dev/null +++ b/lib/prompt_toolkit/layout/screen.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING, Callable + +from prompt_toolkit.cache import FastDictCache +from prompt_toolkit.data_structures import Point +from prompt_toolkit.utils import get_cwidth + +if TYPE_CHECKING: + from .containers import Window + + +__all__ = [ + "Screen", + "Char", +] + + +class Char: + """ + Represent a single character in a :class:`.Screen`. + + This should be considered immutable. + + :param char: A single character (can be a double-width character). + :param style: A style string. (Can contain classnames.) + """ + + __slots__ = ("char", "style", "width") + + # If we end up having one of these special control sequences in the input string, + # we should display them as follows: + # Usually this happens after a "quoted insert". + display_mappings: dict[str, str] = { + "\x00": "^@", # Control space + "\x01": "^A", + "\x02": "^B", + "\x03": "^C", + "\x04": "^D", + "\x05": "^E", + "\x06": "^F", + "\x07": "^G", + "\x08": "^H", + "\x09": "^I", + "\x0a": "^J", + "\x0b": "^K", + "\x0c": "^L", + "\x0d": "^M", + "\x0e": "^N", + "\x0f": "^O", + "\x10": "^P", + "\x11": "^Q", + "\x12": "^R", + "\x13": "^S", + "\x14": "^T", + "\x15": "^U", + "\x16": "^V", + "\x17": "^W", + "\x18": "^X", + "\x19": "^Y", + "\x1a": "^Z", + "\x1b": "^[", # Escape + "\x1c": "^\\", + "\x1d": "^]", + "\x1e": "^^", + "\x1f": "^_", + "\x7f": "^?", # ASCII Delete (backspace). + # Special characters. All visualized like Vim does. + "\x80": "<80>", + "\x81": "<81>", + "\x82": "<82>", + "\x83": "<83>", + "\x84": "<84>", + "\x85": "<85>", + "\x86": "<86>", + "\x87": "<87>", + "\x88": "<88>", + "\x89": "<89>", + "\x8a": "<8a>", + "\x8b": "<8b>", + "\x8c": "<8c>", + "\x8d": "<8d>", + "\x8e": "<8e>", + "\x8f": "<8f>", + "\x90": "<90>", + "\x91": "<91>", + "\x92": "<92>", + "\x93": "<93>", + "\x94": "<94>", + "\x95": "<95>", + "\x96": "<96>", + "\x97": "<97>", + "\x98": "<98>", + "\x99": "<99>", + "\x9a": "<9a>", + "\x9b": "<9b>", + "\x9c": "<9c>", + "\x9d": "<9d>", + "\x9e": "<9e>", + "\x9f": "<9f>", + # For the non-breaking space: visualize like Emacs does by default. + # (Print a space, but attach the 'nbsp' class that applies the + # underline style.) + "\xa0": " ", + } + + def __init__(self, char: str = " ", style: str = "") -> None: + # If this character has to be displayed otherwise, take that one. + if char in self.display_mappings: + if char == "\xa0": + style += " class:nbsp " # Will be underlined. + else: + style += " class:control-character " + + char = self.display_mappings[char] + + self.char = char + self.style = style + + # Calculate width. (We always need this, so better to store it directly + # as a member for performance.) + self.width = get_cwidth(char) + + # In theory, `other` can be any type of object, but because of performance + # we don't want to do an `isinstance` check every time. We assume "other" + # is always a "Char". + def _equal(self, other: Char) -> bool: + return self.char == other.char and self.style == other.style + + def _not_equal(self, other: Char) -> bool: + # Not equal: We don't do `not char.__eq__` here, because of the + # performance of calling yet another function. + return self.char != other.char or self.style != other.style + + if not TYPE_CHECKING: + __eq__ = _equal + __ne__ = _not_equal + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.char!r}, {self.style!r})" + + +_CHAR_CACHE: FastDictCache[tuple[str, str], Char] = FastDictCache( + Char, size=1000 * 1000 +) +Transparent = "[transparent]" + + +class Screen: + """ + Two dimensional buffer of :class:`.Char` instances. + """ + + def __init__( + self, + default_char: Char | None = None, + initial_width: int = 0, + initial_height: int = 0, + ) -> None: + if default_char is None: + default_char2 = _CHAR_CACHE[" ", Transparent] + else: + default_char2 = default_char + + self.data_buffer: defaultdict[int, defaultdict[int, Char]] = defaultdict( + lambda: defaultdict(lambda: default_char2) + ) + + #: Escape sequences to be injected. + self.zero_width_escapes: defaultdict[int, defaultdict[int, str]] = defaultdict( + lambda: defaultdict(str) + ) + + #: Position of the cursor. + self.cursor_positions: dict[ + Window, Point + ] = {} # Map `Window` objects to `Point` objects. + + #: Visibility of the cursor. + self.show_cursor = True + + #: (Optional) Where to position the menu. E.g. at the start of a completion. + #: (We can't use the cursor position, because we don't want the + #: completion menu to change its position when we browse through all the + #: completions.) + self.menu_positions: dict[ + Window, Point + ] = {} # Map `Window` objects to `Point` objects. + + #: Currently used width/height of the screen. This will increase when + #: data is written to the screen. + self.width = initial_width or 0 + self.height = initial_height or 0 + + # Windows that have been drawn. (Each `Window` class will add itself to + # this list.) + self.visible_windows_to_write_positions: dict[Window, WritePosition] = {} + + # List of (z_index, draw_func) + self._draw_float_functions: list[tuple[int, Callable[[], None]]] = [] + + @property + def visible_windows(self) -> list[Window]: + return list(self.visible_windows_to_write_positions.keys()) + + def set_cursor_position(self, window: Window, position: Point) -> None: + """ + Set the cursor position for a given window. + """ + self.cursor_positions[window] = position + + def set_menu_position(self, window: Window, position: Point) -> None: + """ + Set the cursor position for a given window. + """ + self.menu_positions[window] = position + + def get_cursor_position(self, window: Window) -> Point: + """ + Get the cursor position for a given window. + Returns a `Point`. + """ + try: + return self.cursor_positions[window] + except KeyError: + return Point(x=0, y=0) + + def get_menu_position(self, window: Window) -> Point: + """ + Get the menu position for a given window. + (This falls back to the cursor position if no menu position was set.) + """ + try: + return self.menu_positions[window] + except KeyError: + try: + return self.cursor_positions[window] + except KeyError: + return Point(x=0, y=0) + + def draw_with_z_index(self, z_index: int, draw_func: Callable[[], None]) -> None: + """ + Add a draw-function for a `Window` which has a >= 0 z_index. + This will be postponed until `draw_all_floats` is called. + """ + self._draw_float_functions.append((z_index, draw_func)) + + def draw_all_floats(self) -> None: + """ + Draw all float functions in order of z-index. + """ + # We keep looping because some draw functions could add new functions + # to this list. See `FloatContainer`. + while self._draw_float_functions: + # Sort the floats that we have so far by z_index. + functions = sorted(self._draw_float_functions, key=lambda item: item[0]) + + # Draw only one at a time, then sort everything again. Now floats + # might have been added. + self._draw_float_functions = functions[1:] + functions[0][1]() + + def append_style_to_content(self, style_str: str) -> None: + """ + For all the characters in the screen. + Set the style string to the given `style_str`. + """ + b = self.data_buffer + char_cache = _CHAR_CACHE + + append_style = " " + style_str + + for y, row in b.items(): + for x, char in row.items(): + row[x] = char_cache[char.char, char.style + append_style] + + def fill_area( + self, write_position: WritePosition, style: str = "", after: bool = False + ) -> None: + """ + Fill the content of this area, using the given `style`. + The style is prepended before whatever was here before. + """ + if not style.strip(): + return + + xmin = write_position.xpos + xmax = write_position.xpos + write_position.width + char_cache = _CHAR_CACHE + data_buffer = self.data_buffer + + if after: + append_style = " " + style + prepend_style = "" + else: + append_style = "" + prepend_style = style + " " + + for y in range( + write_position.ypos, write_position.ypos + write_position.height + ): + row = data_buffer[y] + for x in range(xmin, xmax): + cell = row[x] + row[x] = char_cache[ + cell.char, prepend_style + cell.style + append_style + ] + + +class WritePosition: + def __init__(self, xpos: int, ypos: int, width: int, height: int) -> None: + assert height >= 0 + assert width >= 0 + # xpos and ypos can be negative. (A float can be partially visible.) + + self.xpos = xpos + self.ypos = ypos + self.width = width + self.height = height + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(x={self.xpos!r}, y={self.ypos!r}, width={self.width!r}, height={self.height!r})" diff --git a/lib/prompt_toolkit/layout/scrollable_pane.py b/lib/prompt_toolkit/layout/scrollable_pane.py new file mode 100644 index 0000000..e38fd76 --- /dev/null +++ b/lib/prompt_toolkit/layout/scrollable_pane.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +from prompt_toolkit.data_structures import Point +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.key_binding import KeyBindingsBase +from prompt_toolkit.mouse_events import MouseEvent + +from .containers import Container, ScrollOffsets +from .dimension import AnyDimension, Dimension, sum_layout_dimensions, to_dimension +from .mouse_handlers import MouseHandler, MouseHandlers +from .screen import Char, Screen, WritePosition + +__all__ = ["ScrollablePane"] + +# Never go beyond this height, because performance will degrade. +MAX_AVAILABLE_HEIGHT = 10_000 + + +class ScrollablePane(Container): + """ + Container widget that exposes a larger virtual screen to its content and + displays it in a vertical scrollbale region. + + Typically this is wrapped in a large `HSplit` container. Make sure in that + case to not specify a `height` dimension of the `HSplit`, so that it will + scale according to the content. + + .. note:: + + If you want to display a completion menu for widgets in this + `ScrollablePane`, then it's still a good practice to use a + `FloatContainer` with a `CompletionsMenu` in a `Float` at the top-level + of the layout hierarchy, rather then nesting a `FloatContainer` in this + `ScrollablePane`. (Otherwise, it's possible that the completion menu + is clipped.) + + :param content: The content container. + :param scrolloffset: Try to keep the cursor within this distance from the + top/bottom (left/right offset is not used). + :param keep_cursor_visible: When `True`, automatically scroll the pane so + that the cursor (of the focused window) is always visible. + :param keep_focused_window_visible: When `True`, automatically scroll the + pane so that the focused window is visible, or as much visible as + possible if it doesn't completely fit the screen. + :param max_available_height: Always constraint the height to this amount + for performance reasons. + :param width: When given, use this width instead of looking at the children. + :param height: When given, use this height instead of looking at the children. + :param show_scrollbar: When `True` display a scrollbar on the right. + """ + + def __init__( + self, + content: Container, + scroll_offsets: ScrollOffsets | None = None, + keep_cursor_visible: FilterOrBool = True, + keep_focused_window_visible: FilterOrBool = True, + max_available_height: int = MAX_AVAILABLE_HEIGHT, + width: AnyDimension = None, + height: AnyDimension = None, + show_scrollbar: FilterOrBool = True, + display_arrows: FilterOrBool = True, + up_arrow_symbol: str = "^", + down_arrow_symbol: str = "v", + ) -> None: + self.content = content + self.scroll_offsets = scroll_offsets or ScrollOffsets(top=1, bottom=1) + self.keep_cursor_visible = to_filter(keep_cursor_visible) + self.keep_focused_window_visible = to_filter(keep_focused_window_visible) + self.max_available_height = max_available_height + self.width = width + self.height = height + self.show_scrollbar = to_filter(show_scrollbar) + self.display_arrows = to_filter(display_arrows) + self.up_arrow_symbol = up_arrow_symbol + self.down_arrow_symbol = down_arrow_symbol + + self.vertical_scroll = 0 + + def __repr__(self) -> str: + return f"ScrollablePane({self.content!r})" + + def reset(self) -> None: + self.content.reset() + + def preferred_width(self, max_available_width: int) -> Dimension: + if self.width is not None: + return to_dimension(self.width) + + # We're only scrolling vertical. So the preferred width is equal to + # that of the content. + content_width = self.content.preferred_width(max_available_width) + + # If a scrollbar needs to be displayed, add +1 to the content width. + if self.show_scrollbar(): + return sum_layout_dimensions([Dimension.exact(1), content_width]) + + return content_width + + def preferred_height(self, width: int, max_available_height: int) -> Dimension: + if self.height is not None: + return to_dimension(self.height) + + # Prefer a height large enough so that it fits all the content. If not, + # we'll make the pane scrollable. + if self.show_scrollbar(): + # If `show_scrollbar` is set. Always reserve space for the scrollbar. + width -= 1 + + dimension = self.content.preferred_height(width, self.max_available_height) + + # Only take 'preferred' into account. Min/max can be anything. + return Dimension(min=0, preferred=dimension.preferred) + + def write_to_screen( + self, + screen: Screen, + mouse_handlers: MouseHandlers, + write_position: WritePosition, + parent_style: str, + erase_bg: bool, + z_index: int | None, + ) -> None: + """ + Render scrollable pane content. + + This works by rendering on an off-screen canvas, and copying over the + visible region. + """ + show_scrollbar = self.show_scrollbar() + + if show_scrollbar: + virtual_width = write_position.width - 1 + else: + virtual_width = write_position.width + + # Compute preferred height again. + virtual_height = self.content.preferred_height( + virtual_width, self.max_available_height + ).preferred + + # Ensure virtual height is at least the available height. + virtual_height = max(virtual_height, write_position.height) + virtual_height = min(virtual_height, self.max_available_height) + + # First, write the content to a virtual screen, then copy over the + # visible part to the real screen. + temp_screen = Screen(default_char=Char(char=" ", style=parent_style)) + temp_screen.show_cursor = screen.show_cursor + temp_write_position = WritePosition( + xpos=0, ypos=0, width=virtual_width, height=virtual_height + ) + + temp_mouse_handlers = MouseHandlers() + + self.content.write_to_screen( + temp_screen, + temp_mouse_handlers, + temp_write_position, + parent_style, + erase_bg, + z_index, + ) + temp_screen.draw_all_floats() + + # If anything in the virtual screen is focused, move vertical scroll to + from prompt_toolkit.application import get_app + + focused_window = get_app().layout.current_window + + try: + visible_win_write_pos = temp_screen.visible_windows_to_write_positions[ + focused_window + ] + except KeyError: + pass # No window focused here. Don't scroll. + else: + # Make sure this window is visible. + self._make_window_visible( + write_position.height, + virtual_height, + visible_win_write_pos, + temp_screen.cursor_positions.get(focused_window), + ) + + # Copy over virtual screen and zero width escapes to real screen. + self._copy_over_screen(screen, temp_screen, write_position, virtual_width) + + # Copy over mouse handlers. + self._copy_over_mouse_handlers( + mouse_handlers, temp_mouse_handlers, write_position, virtual_width + ) + + # Set screen.width/height. + ypos = write_position.ypos + xpos = write_position.xpos + + screen.width = max(screen.width, xpos + virtual_width) + screen.height = max(screen.height, ypos + write_position.height) + + # Copy over window write positions. + self._copy_over_write_positions(screen, temp_screen, write_position) + + if temp_screen.show_cursor: + screen.show_cursor = True + + # Copy over cursor positions, if they are visible. + for window, point in temp_screen.cursor_positions.items(): + if ( + 0 <= point.x < write_position.width + and self.vertical_scroll + <= point.y + < write_position.height + self.vertical_scroll + ): + screen.cursor_positions[window] = Point( + x=point.x + xpos, y=point.y + ypos - self.vertical_scroll + ) + + # Copy over menu positions, but clip them to the visible area. + for window, point in temp_screen.menu_positions.items(): + screen.menu_positions[window] = self._clip_point_to_visible_area( + Point(x=point.x + xpos, y=point.y + ypos - self.vertical_scroll), + write_position, + ) + + # Draw scrollbar. + if show_scrollbar: + self._draw_scrollbar( + write_position, + virtual_height, + screen, + ) + + def _clip_point_to_visible_area( + self, point: Point, write_position: WritePosition + ) -> Point: + """ + Ensure that the cursor and menu positions always are always reported + """ + if point.x < write_position.xpos: + point = point._replace(x=write_position.xpos) + if point.y < write_position.ypos: + point = point._replace(y=write_position.ypos) + if point.x >= write_position.xpos + write_position.width: + point = point._replace(x=write_position.xpos + write_position.width - 1) + if point.y >= write_position.ypos + write_position.height: + point = point._replace(y=write_position.ypos + write_position.height - 1) + + return point + + def _copy_over_screen( + self, + screen: Screen, + temp_screen: Screen, + write_position: WritePosition, + virtual_width: int, + ) -> None: + """ + Copy over visible screen content and "zero width escape sequences". + """ + ypos = write_position.ypos + xpos = write_position.xpos + + for y in range(write_position.height): + temp_row = temp_screen.data_buffer[y + self.vertical_scroll] + row = screen.data_buffer[y + ypos] + temp_zero_width_escapes = temp_screen.zero_width_escapes[ + y + self.vertical_scroll + ] + zero_width_escapes = screen.zero_width_escapes[y + ypos] + + for x in range(virtual_width): + row[x + xpos] = temp_row[x] + + if x in temp_zero_width_escapes: + zero_width_escapes[x + xpos] = temp_zero_width_escapes[x] + + def _copy_over_mouse_handlers( + self, + mouse_handlers: MouseHandlers, + temp_mouse_handlers: MouseHandlers, + write_position: WritePosition, + virtual_width: int, + ) -> None: + """ + Copy over mouse handlers from virtual screen to real screen. + + Note: we take `virtual_width` because we don't want to copy over mouse + handlers that we possibly have behind the scrollbar. + """ + ypos = write_position.ypos + xpos = write_position.xpos + + # Cache mouse handlers when wrapping them. Very often the same mouse + # handler is registered for many positions. + mouse_handler_wrappers: dict[MouseHandler, MouseHandler] = {} + + def wrap_mouse_handler(handler: MouseHandler) -> MouseHandler: + "Wrap mouse handler. Translate coordinates in `MouseEvent`." + if handler not in mouse_handler_wrappers: + + def new_handler(event: MouseEvent) -> None: + new_event = MouseEvent( + position=Point( + x=event.position.x - xpos, + y=event.position.y + self.vertical_scroll - ypos, + ), + event_type=event.event_type, + button=event.button, + modifiers=event.modifiers, + ) + handler(new_event) + + mouse_handler_wrappers[handler] = new_handler + return mouse_handler_wrappers[handler] + + # Copy handlers. + mouse_handlers_dict = mouse_handlers.mouse_handlers + temp_mouse_handlers_dict = temp_mouse_handlers.mouse_handlers + + for y in range(write_position.height): + if y in temp_mouse_handlers_dict: + temp_mouse_row = temp_mouse_handlers_dict[y + self.vertical_scroll] + mouse_row = mouse_handlers_dict[y + ypos] + for x in range(virtual_width): + if x in temp_mouse_row: + mouse_row[x + xpos] = wrap_mouse_handler(temp_mouse_row[x]) + + def _copy_over_write_positions( + self, screen: Screen, temp_screen: Screen, write_position: WritePosition + ) -> None: + """ + Copy over window write positions. + """ + ypos = write_position.ypos + xpos = write_position.xpos + + for win, write_pos in temp_screen.visible_windows_to_write_positions.items(): + screen.visible_windows_to_write_positions[win] = WritePosition( + xpos=write_pos.xpos + xpos, + ypos=write_pos.ypos + ypos - self.vertical_scroll, + # TODO: if the window is only partly visible, then truncate width/height. + # This could be important if we have nested ScrollablePanes. + height=write_pos.height, + width=write_pos.width, + ) + + def is_modal(self) -> bool: + return self.content.is_modal() + + def get_key_bindings(self) -> KeyBindingsBase | None: + return self.content.get_key_bindings() + + def get_children(self) -> list[Container]: + return [self.content] + + def _make_window_visible( + self, + visible_height: int, + virtual_height: int, + visible_win_write_pos: WritePosition, + cursor_position: Point | None, + ) -> None: + """ + Scroll the scrollable pane, so that this window becomes visible. + + :param visible_height: Height of this `ScrollablePane` that is rendered. + :param virtual_height: Height of the virtual, temp screen. + :param visible_win_write_pos: `WritePosition` of the nested window on the + temp screen. + :param cursor_position: The location of the cursor position of this + window on the temp screen. + """ + # Start with maximum allowed scroll range, and then reduce according to + # the focused window and cursor position. + min_scroll = 0 + max_scroll = virtual_height - visible_height + + if self.keep_cursor_visible(): + # Reduce min/max scroll according to the cursor in the focused window. + if cursor_position is not None: + offsets = self.scroll_offsets + cpos_min_scroll = ( + cursor_position.y - visible_height + 1 + offsets.bottom + ) + cpos_max_scroll = cursor_position.y - offsets.top + min_scroll = max(min_scroll, cpos_min_scroll) + max_scroll = max(0, min(max_scroll, cpos_max_scroll)) + + if self.keep_focused_window_visible(): + # Reduce min/max scroll according to focused window position. + # If the window is small enough, bot the top and bottom of the window + # should be visible. + if visible_win_write_pos.height <= visible_height: + window_min_scroll = ( + visible_win_write_pos.ypos + + visible_win_write_pos.height + - visible_height + ) + window_max_scroll = visible_win_write_pos.ypos + else: + # Window does not fit on the screen. Make sure at least the whole + # screen is occupied with this window, and nothing else is shown. + window_min_scroll = visible_win_write_pos.ypos + window_max_scroll = ( + visible_win_write_pos.ypos + + visible_win_write_pos.height + - visible_height + ) + + min_scroll = max(min_scroll, window_min_scroll) + max_scroll = min(max_scroll, window_max_scroll) + + if min_scroll > max_scroll: + min_scroll = max_scroll # Should not happen. + + # Finally, properly clip the vertical scroll. + if self.vertical_scroll > max_scroll: + self.vertical_scroll = max_scroll + if self.vertical_scroll < min_scroll: + self.vertical_scroll = min_scroll + + def _draw_scrollbar( + self, write_position: WritePosition, content_height: int, screen: Screen + ) -> None: + """ + Draw the scrollbar on the screen. + + Note: There is some code duplication with the `ScrollbarMargin` + implementation. + """ + + window_height = write_position.height + display_arrows = self.display_arrows() + + if display_arrows: + window_height -= 2 + + try: + fraction_visible = write_position.height / float(content_height) + fraction_above = self.vertical_scroll / float(content_height) + + scrollbar_height = int( + min(window_height, max(1, window_height * fraction_visible)) + ) + scrollbar_top = int(window_height * fraction_above) + except ZeroDivisionError: + return + else: + + def is_scroll_button(row: int) -> bool: + "True if we should display a button on this row." + return scrollbar_top <= row <= scrollbar_top + scrollbar_height + + xpos = write_position.xpos + write_position.width - 1 + ypos = write_position.ypos + data_buffer = screen.data_buffer + + # Up arrow. + if display_arrows: + data_buffer[ypos][xpos] = Char( + self.up_arrow_symbol, "class:scrollbar.arrow" + ) + ypos += 1 + + # Scrollbar body. + scrollbar_background = "class:scrollbar.background" + scrollbar_background_start = "class:scrollbar.background,scrollbar.start" + scrollbar_button = "class:scrollbar.button" + scrollbar_button_end = "class:scrollbar.button,scrollbar.end" + + for i in range(window_height): + style = "" + if is_scroll_button(i): + if not is_scroll_button(i + 1): + # Give the last cell a different style, because we want + # to underline this. + style = scrollbar_button_end + else: + style = scrollbar_button + else: + if is_scroll_button(i + 1): + style = scrollbar_background_start + else: + style = scrollbar_background + + data_buffer[ypos][xpos] = Char(" ", style) + ypos += 1 + + # Down arrow + if display_arrows: + data_buffer[ypos][xpos] = Char( + self.down_arrow_symbol, "class:scrollbar.arrow" + ) diff --git a/lib/prompt_toolkit/layout/utils.py b/lib/prompt_toolkit/layout/utils.py new file mode 100644 index 0000000..373fe52 --- /dev/null +++ b/lib/prompt_toolkit/layout/utils.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Iterable, List, TypeVar, cast, overload + +from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple + +if TYPE_CHECKING: + from typing_extensions import SupportsIndex + +__all__ = [ + "explode_text_fragments", +] + +_T = TypeVar("_T", bound=OneStyleAndTextTuple) + + +class _ExplodedList(List[_T]): + """ + Wrapper around a list, that marks it as 'exploded'. + + As soon as items are added or the list is extended, the new items are + automatically exploded as well. + """ + + exploded = True + + def append(self, item: _T) -> None: + self.extend([item]) + + def extend(self, lst: Iterable[_T]) -> None: + super().extend(explode_text_fragments(lst)) + + def insert(self, index: SupportsIndex, item: _T) -> None: + raise NotImplementedError # TODO + + # TODO: When creating a copy() or [:], return also an _ExplodedList. + + @overload + def __setitem__(self, index: SupportsIndex, value: _T) -> None: ... + + @overload + def __setitem__(self, index: slice, value: Iterable[_T]) -> None: ... + + def __setitem__( + self, index: SupportsIndex | slice, value: _T | Iterable[_T] + ) -> None: + """ + Ensure that when `(style_str, 'long string')` is set, the string will be + exploded. + """ + if not isinstance(index, slice): + int_index = index.__index__() + index = slice(int_index, int_index + 1) + if isinstance(value, tuple): # In case of `OneStyleAndTextTuple`. + value = cast("List[_T]", [value]) + + super().__setitem__(index, explode_text_fragments(value)) + + +def explode_text_fragments(fragments: Iterable[_T]) -> _ExplodedList[_T]: + """ + Turn a list of (style_str, text) tuples into another list where each string is + exactly one character. + + It should be fine to call this function several times. Calling this on a + list that is already exploded, is a null operation. + + :param fragments: List of (style, text) tuples. + """ + # When the fragments is already exploded, don't explode again. + if isinstance(fragments, _ExplodedList): + return fragments + + result: list[_T] = [] + + for style, string, *rest in fragments: + for c in string: + result.append((style, c, *rest)) # type: ignore + + return _ExplodedList(result) diff --git a/lib/prompt_toolkit/lexers/__init__.py b/lib/prompt_toolkit/lexers/__init__.py new file mode 100644 index 0000000..8f72d07 --- /dev/null +++ b/lib/prompt_toolkit/lexers/__init__.py @@ -0,0 +1,21 @@ +""" +Lexer interface and implementations. +Used for syntax highlighting. +""" + +from __future__ import annotations + +from .base import DynamicLexer, Lexer, SimpleLexer +from .pygments import PygmentsLexer, RegexSync, SyncFromStart, SyntaxSync + +__all__ = [ + # Base. + "Lexer", + "SimpleLexer", + "DynamicLexer", + # Pygments. + "PygmentsLexer", + "RegexSync", + "SyncFromStart", + "SyntaxSync", +] diff --git a/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..d0902b8 Binary files /dev/null and b/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..f6c4fed Binary files /dev/null and b/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc b/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc new file mode 100644 index 0000000..8ab02f6 Binary files /dev/null and b/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/lexers/base.py b/lib/prompt_toolkit/lexers/base.py new file mode 100644 index 0000000..c61e2b9 --- /dev/null +++ b/lib/prompt_toolkit/lexers/base.py @@ -0,0 +1,85 @@ +""" +Base classes for prompt_toolkit lexers. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import Callable, Hashable + +from prompt_toolkit.document import Document +from prompt_toolkit.formatted_text.base import StyleAndTextTuples + +__all__ = [ + "Lexer", + "SimpleLexer", + "DynamicLexer", +] + + +class Lexer(metaclass=ABCMeta): + """ + Base class for all lexers. + """ + + @abstractmethod + def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: + """ + Takes a :class:`~prompt_toolkit.document.Document` and returns a + callable that takes a line number and returns a list of + ``(style_str, text)`` tuples for that line. + + XXX: Note that in the past, this was supposed to return a list + of ``(Token, text)`` tuples, just like a Pygments lexer. + """ + + def invalidation_hash(self) -> Hashable: + """ + When this changes, `lex_document` could give a different output. + (Only used for `DynamicLexer`.) + """ + return id(self) + + +class SimpleLexer(Lexer): + """ + Lexer that doesn't do any tokenizing and returns the whole input as one + token. + + :param style: The style string for this lexer. + """ + + def __init__(self, style: str = "") -> None: + self.style = style + + def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: + lines = document.lines + + def get_line(lineno: int) -> StyleAndTextTuples: + "Return the tokens for the given line." + try: + return [(self.style, lines[lineno])] + except IndexError: + return [] + + return get_line + + +class DynamicLexer(Lexer): + """ + Lexer class that can dynamically returns any Lexer. + + :param get_lexer: Callable that returns a :class:`.Lexer` instance. + """ + + def __init__(self, get_lexer: Callable[[], Lexer | None]) -> None: + self.get_lexer = get_lexer + self._dummy = SimpleLexer() + + def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: + lexer = self.get_lexer() or self._dummy + return lexer.lex_document(document) + + def invalidation_hash(self) -> Hashable: + lexer = self.get_lexer() or self._dummy + return id(lexer) diff --git a/lib/prompt_toolkit/lexers/pygments.py b/lib/prompt_toolkit/lexers/pygments.py new file mode 100644 index 0000000..d5a39c4 --- /dev/null +++ b/lib/prompt_toolkit/lexers/pygments.py @@ -0,0 +1,328 @@ +""" +Adaptor classes for using Pygments lexers within prompt_toolkit. + +This includes syntax synchronization code, so that we don't have to start +lexing at the beginning of a document, when displaying a very large text. +""" + +from __future__ import annotations + +import re +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Callable, Dict, Generator, Iterable, Tuple + +from prompt_toolkit.document import Document +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.formatted_text.base import StyleAndTextTuples +from prompt_toolkit.formatted_text.utils import split_lines +from prompt_toolkit.styles.pygments import pygments_token_to_classname + +from .base import Lexer, SimpleLexer + +if TYPE_CHECKING: + from pygments.lexer import Lexer as PygmentsLexerCls + +__all__ = [ + "PygmentsLexer", + "SyntaxSync", + "SyncFromStart", + "RegexSync", +] + + +class SyntaxSync(metaclass=ABCMeta): + """ + Syntax synchronizer. This is a tool that finds a start position for the + lexer. This is especially important when editing big documents; we don't + want to start the highlighting by running the lexer from the beginning of + the file. That is very slow when editing. + """ + + @abstractmethod + def get_sync_start_position( + self, document: Document, lineno: int + ) -> tuple[int, int]: + """ + Return the position from where we can start lexing as a (row, column) + tuple. + + :param document: `Document` instance that contains all the lines. + :param lineno: The line that we want to highlight. (We need to return + this line, or an earlier position.) + """ + + +class SyncFromStart(SyntaxSync): + """ + Always start the syntax highlighting from the beginning. + """ + + def get_sync_start_position( + self, document: Document, lineno: int + ) -> tuple[int, int]: + return 0, 0 + + +class RegexSync(SyntaxSync): + """ + Synchronize by starting at a line that matches the given regex pattern. + """ + + # Never go more than this amount of lines backwards for synchronization. + # That would be too CPU intensive. + MAX_BACKWARDS = 500 + + # Start lexing at the start, if we are in the first 'n' lines and no + # synchronization position was found. + FROM_START_IF_NO_SYNC_POS_FOUND = 100 + + def __init__(self, pattern: str) -> None: + self._compiled_pattern = re.compile(pattern) + + def get_sync_start_position( + self, document: Document, lineno: int + ) -> tuple[int, int]: + """ + Scan backwards, and find a possible position to start. + """ + pattern = self._compiled_pattern + lines = document.lines + + # Scan upwards, until we find a point where we can start the syntax + # synchronization. + for i in range(lineno, max(-1, lineno - self.MAX_BACKWARDS), -1): + match = pattern.match(lines[i]) + if match: + return i, match.start() + + # No synchronization point found. If we aren't that far from the + # beginning, start at the very beginning, otherwise, just try to start + # at the current line. + if lineno < self.FROM_START_IF_NO_SYNC_POS_FOUND: + return 0, 0 + else: + return lineno, 0 + + @classmethod + def from_pygments_lexer_cls(cls, lexer_cls: PygmentsLexerCls) -> RegexSync: + """ + Create a :class:`.RegexSync` instance for this Pygments lexer class. + """ + patterns = { + # For Python, start highlighting at any class/def block. + "Python": r"^\s*(class|def)\s+", + "Python 3": r"^\s*(class|def)\s+", + # For HTML, start at any open/close tag definition. + "HTML": r"<[/a-zA-Z]", + # For javascript, start at a function. + "JavaScript": r"\bfunction\b", + # TODO: Add definitions for other languages. + # By default, we start at every possible line. + } + p = patterns.get(lexer_cls.name, "^") + return cls(p) + + +class _TokenCache(Dict[Tuple[str, ...], str]): + """ + Cache that converts Pygments tokens into `prompt_toolkit` style objects. + + ``Token.A.B.C`` will be converted into: + ``class:pygments,pygments.A,pygments.A.B,pygments.A.B.C`` + """ + + def __missing__(self, key: tuple[str, ...]) -> str: + result = "class:" + pygments_token_to_classname(key) + self[key] = result + return result + + +_token_cache = _TokenCache() + + +class PygmentsLexer(Lexer): + """ + Lexer that calls a pygments lexer. + + Example:: + + from pygments.lexers.html import HtmlLexer + lexer = PygmentsLexer(HtmlLexer) + + Note: Don't forget to also load a Pygments compatible style. E.g.:: + + from prompt_toolkit.styles.from_pygments import style_from_pygments_cls + from pygments.styles import get_style_by_name + style = style_from_pygments_cls(get_style_by_name('monokai')) + + :param pygments_lexer_cls: A `Lexer` from Pygments. + :param sync_from_start: Start lexing at the start of the document. This + will always give the best results, but it will be slow for bigger + documents. (When the last part of the document is display, then the + whole document will be lexed by Pygments on every key stroke.) It is + recommended to disable this for inputs that are expected to be more + than 1,000 lines. + :param syntax_sync: `SyntaxSync` object. + """ + + # Minimum amount of lines to go backwards when starting the parser. + # This is important when the lines are retrieved in reverse order, or when + # scrolling upwards. (Due to the complexity of calculating the vertical + # scroll offset in the `Window` class, lines are not always retrieved in + # order.) + MIN_LINES_BACKWARDS = 50 + + # When a parser was started this amount of lines back, read the parser + # until we get the current line. Otherwise, start a new parser. + # (This should probably be bigger than MIN_LINES_BACKWARDS.) + REUSE_GENERATOR_MAX_DISTANCE = 100 + + def __init__( + self, + pygments_lexer_cls: type[PygmentsLexerCls], + sync_from_start: FilterOrBool = True, + syntax_sync: SyntaxSync | None = None, + ) -> None: + self.pygments_lexer_cls = pygments_lexer_cls + self.sync_from_start = to_filter(sync_from_start) + + # Instantiate the Pygments lexer. + self.pygments_lexer = pygments_lexer_cls( + stripnl=False, stripall=False, ensurenl=False + ) + + # Create syntax sync instance. + self.syntax_sync = syntax_sync or RegexSync.from_pygments_lexer_cls( + pygments_lexer_cls + ) + + @classmethod + def from_filename( + cls, filename: str, sync_from_start: FilterOrBool = True + ) -> Lexer: + """ + Create a `Lexer` from a filename. + """ + # Inline imports: the Pygments dependency is optional! + from pygments.lexers import get_lexer_for_filename + from pygments.util import ClassNotFound + + try: + pygments_lexer = get_lexer_for_filename(filename) + except ClassNotFound: + return SimpleLexer() + else: + return cls(pygments_lexer.__class__, sync_from_start=sync_from_start) + + def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: + """ + Create a lexer function that takes a line number and returns the list + of (style_str, text) tuples as the Pygments lexer returns for that line. + """ + LineGenerator = Generator[Tuple[int, StyleAndTextTuples], None, None] + + # Cache of already lexed lines. + cache: dict[int, StyleAndTextTuples] = {} + + # Pygments generators that are currently lexing. + # Map lexer generator to the line number. + line_generators: dict[LineGenerator, int] = {} + + def get_syntax_sync() -> SyntaxSync: + "The Syntax synchronization object that we currently use." + if self.sync_from_start(): + return SyncFromStart() + else: + return self.syntax_sync + + def find_closest_generator(i: int) -> LineGenerator | None: + "Return a generator close to line 'i', or None if none was found." + for generator, lineno in line_generators.items(): + if lineno < i and i - lineno < self.REUSE_GENERATOR_MAX_DISTANCE: + return generator + return None + + def create_line_generator(start_lineno: int, column: int = 0) -> LineGenerator: + """ + Create a generator that yields the lexed lines. + Each iteration it yields a (line_number, [(style_str, text), ...]) tuple. + """ + + def get_text_fragments() -> Iterable[tuple[str, str]]: + text = "\n".join(document.lines[start_lineno:])[column:] + + # We call `get_text_fragments_unprocessed`, because `get_tokens` will + # still replace \r\n and \r by \n. (We don't want that, + # Pygments should return exactly the same amount of text, as we + # have given as input.) + for _, t, v in self.pygments_lexer.get_tokens_unprocessed(text): + # Turn Pygments `Token` object into prompt_toolkit style + # strings. + yield _token_cache[t], v + + yield from enumerate(split_lines(list(get_text_fragments())), start_lineno) + + def get_generator(i: int) -> LineGenerator: + """ + Find an already started generator that is close, or create a new one. + """ + # Find closest line generator. + generator = find_closest_generator(i) + if generator: + return generator + + # No generator found. Determine starting point for the syntax + # synchronization first. + + # Go at least x lines back. (Make scrolling upwards more + # efficient.) + i = max(0, i - self.MIN_LINES_BACKWARDS) + + if i == 0: + row = 0 + column = 0 + else: + row, column = get_syntax_sync().get_sync_start_position(document, i) + + # Find generator close to this point, or otherwise create a new one. + generator = find_closest_generator(i) + if generator: + return generator + else: + generator = create_line_generator(row, column) + + # If the column is not 0, ignore the first line. (Which is + # incomplete. This happens when the synchronization algorithm tells + # us to start parsing in the middle of a line.) + if column: + next(generator) + row += 1 + + line_generators[generator] = row + return generator + + def get_line(i: int) -> StyleAndTextTuples: + "Return the tokens for a given line number." + try: + return cache[i] + except KeyError: + generator = get_generator(i) + + # Exhaust the generator, until we find the requested line. + for num, line in generator: + cache[num] = line + if num == i: + line_generators[generator] = i + + # Remove the next item from the cache. + # (It could happen that it's already there, because of + # another generator that started filling these lines, + # but we want to synchronize these lines with the + # current lexer's state.) + if num + 1 in cache: + del cache[num + 1] + + return cache[num] + return [] + + return get_line diff --git a/lib/prompt_toolkit/log.py b/lib/prompt_toolkit/log.py new file mode 100644 index 0000000..2853579 --- /dev/null +++ b/lib/prompt_toolkit/log.py @@ -0,0 +1,13 @@ +""" +Logging configuration. +""" + +from __future__ import annotations + +import logging + +__all__ = [ + "logger", +] + +logger = logging.getLogger(__package__) diff --git a/lib/prompt_toolkit/mouse_events.py b/lib/prompt_toolkit/mouse_events.py new file mode 100644 index 0000000..f244f8e --- /dev/null +++ b/lib/prompt_toolkit/mouse_events.py @@ -0,0 +1,85 @@ +""" +Mouse events. + + +How it works +------------ + +The renderer has a 2 dimensional grid of mouse event handlers. +(`prompt_toolkit.layout.MouseHandlers`.) When the layout is rendered, the +`Window` class will make sure that this grid will also be filled with +callbacks. For vt100 terminals, mouse events are received through stdin, just +like any other key press. There is a handler among the key bindings that +catches these events and forwards them to such a mouse event handler. It passes +through the `Window` class where the coordinates are translated from absolute +coordinates to coordinates relative to the user control, and there +`UIControl.mouse_handler` is called. +""" + +from __future__ import annotations + +from enum import Enum + +from .data_structures import Point + +__all__ = ["MouseEventType", "MouseButton", "MouseModifier", "MouseEvent"] + + +class MouseEventType(Enum): + # Mouse up: This same event type is fired for all three events: left mouse + # up, right mouse up, or middle mouse up + MOUSE_UP = "MOUSE_UP" + + # Mouse down: This implicitly refers to the left mouse down (this event is + # not fired upon pressing the middle or right mouse buttons). + MOUSE_DOWN = "MOUSE_DOWN" + + SCROLL_UP = "SCROLL_UP" + SCROLL_DOWN = "SCROLL_DOWN" + + # Triggered when the left mouse button is held down, and the mouse moves + MOUSE_MOVE = "MOUSE_MOVE" + + +class MouseButton(Enum): + LEFT = "LEFT" + MIDDLE = "MIDDLE" + RIGHT = "RIGHT" + + # When we're scrolling, or just moving the mouse and not pressing a button. + NONE = "NONE" + + # This is for when we don't know which mouse button was pressed, but we do + # know that one has been pressed during this mouse event (as opposed to + # scrolling, for example) + UNKNOWN = "UNKNOWN" + + +class MouseModifier(Enum): + SHIFT = "SHIFT" + ALT = "ALT" + CONTROL = "CONTROL" + + +class MouseEvent: + """ + Mouse event, sent to `UIControl.mouse_handler`. + + :param position: `Point` instance. + :param event_type: `MouseEventType`. + """ + + def __init__( + self, + position: Point, + event_type: MouseEventType, + button: MouseButton, + modifiers: frozenset[MouseModifier], + ) -> None: + self.position = position + self.event_type = event_type + self.button = button + self.modifiers = modifiers + + def __repr__(self) -> str: + return f"MouseEvent({self.position!r},{self.event_type!r},{self.button!r},{self.modifiers!r})" diff --git a/lib/prompt_toolkit/output/__init__.py b/lib/prompt_toolkit/output/__init__.py new file mode 100644 index 0000000..6b4c5f3 --- /dev/null +++ b/lib/prompt_toolkit/output/__init__.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from .base import DummyOutput, Output +from .color_depth import ColorDepth +from .defaults import create_output + +__all__ = [ + # Base. + "Output", + "DummyOutput", + # Color depth. + "ColorDepth", + # Defaults. + "create_output", +] diff --git a/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..f964a87 Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..7c01bce Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc new file mode 100644 index 0000000..fc8f3ad Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc new file mode 100644 index 0000000..1488ea4 Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc new file mode 100644 index 0000000..98300fa Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc new file mode 100644 index 0000000..2baf2ae Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc new file mode 100644 index 0000000..2006558 Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc new file mode 100644 index 0000000..c3d5159 Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc new file mode 100644 index 0000000..094dfcc Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc b/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc new file mode 100644 index 0000000..687d6f2 Binary files /dev/null and b/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/output/base.py b/lib/prompt_toolkit/output/base.py new file mode 100644 index 0000000..6ba09fd --- /dev/null +++ b/lib/prompt_toolkit/output/base.py @@ -0,0 +1,332 @@ +""" +Interface for an output. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import TextIO + +from prompt_toolkit.cursor_shapes import CursorShape +from prompt_toolkit.data_structures import Size +from prompt_toolkit.styles import Attrs + +from .color_depth import ColorDepth + +__all__ = [ + "Output", + "DummyOutput", +] + + +class Output(metaclass=ABCMeta): + """ + Base class defining the output interface for a + :class:`~prompt_toolkit.renderer.Renderer`. + + Actual implementations are + :class:`~prompt_toolkit.output.vt100.Vt100_Output` and + :class:`~prompt_toolkit.output.win32.Win32Output`. + """ + + stdout: TextIO | None = None + + @abstractmethod + def fileno(self) -> int: + "Return the file descriptor to which we can write for the output." + + @abstractmethod + def encoding(self) -> str: + """ + Return the encoding for this output, e.g. 'utf-8'. + (This is used mainly to know which characters are supported by the + output the data, so that the UI can provide alternatives, when + required.) + """ + + @abstractmethod + def write(self, data: str) -> None: + "Write text (Terminal escape sequences will be removed/escaped.)" + + @abstractmethod + def write_raw(self, data: str) -> None: + "Write text." + + @abstractmethod + def set_title(self, title: str) -> None: + "Set terminal title." + + @abstractmethod + def clear_title(self) -> None: + "Clear title again. (or restore previous title.)" + + @abstractmethod + def flush(self) -> None: + "Write to output stream and flush." + + @abstractmethod + def erase_screen(self) -> None: + """ + Erases the screen with the background color and moves the cursor to + home. + """ + + @abstractmethod + def enter_alternate_screen(self) -> None: + "Go to the alternate screen buffer. (For full screen applications)." + + @abstractmethod + def quit_alternate_screen(self) -> None: + "Leave the alternate screen buffer." + + @abstractmethod + def enable_mouse_support(self) -> None: + "Enable mouse." + + @abstractmethod + def disable_mouse_support(self) -> None: + "Disable mouse." + + @abstractmethod + def erase_end_of_line(self) -> None: + """ + Erases from the current cursor position to the end of the current line. + """ + + @abstractmethod + def erase_down(self) -> None: + """ + Erases the screen from the current line down to the bottom of the + screen. + """ + + @abstractmethod + def reset_attributes(self) -> None: + "Reset color and styling attributes." + + @abstractmethod + def set_attributes(self, attrs: Attrs, color_depth: ColorDepth) -> None: + "Set new color and styling attributes." + + @abstractmethod + def disable_autowrap(self) -> None: + "Disable auto line wrapping." + + @abstractmethod + def enable_autowrap(self) -> None: + "Enable auto line wrapping." + + @abstractmethod + def cursor_goto(self, row: int = 0, column: int = 0) -> None: + "Move cursor position." + + @abstractmethod + def cursor_up(self, amount: int) -> None: + "Move cursor `amount` place up." + + @abstractmethod + def cursor_down(self, amount: int) -> None: + "Move cursor `amount` place down." + + @abstractmethod + def cursor_forward(self, amount: int) -> None: + "Move cursor `amount` place forward." + + @abstractmethod + def cursor_backward(self, amount: int) -> None: + "Move cursor `amount` place backward." + + @abstractmethod + def hide_cursor(self) -> None: + "Hide cursor." + + @abstractmethod + def show_cursor(self) -> None: + "Show cursor." + + @abstractmethod + def set_cursor_shape(self, cursor_shape: CursorShape) -> None: + "Set cursor shape to block, beam or underline." + + @abstractmethod + def reset_cursor_shape(self) -> None: + "Reset cursor shape." + + def ask_for_cpr(self) -> None: + """ + Asks for a cursor position report (CPR). + (VT100 only.) + """ + + @property + def responds_to_cpr(self) -> bool: + """ + `True` if the `Application` can expect to receive a CPR response after + calling `ask_for_cpr` (this will come back through the corresponding + `Input`). + + This is used to determine the amount of available rows we have below + the cursor position. In the first place, we have this so that the drop + down autocompletion menus are sized according to the available space. + + On Windows, we don't need this, there we have + `get_rows_below_cursor_position`. + """ + return False + + @abstractmethod + def get_size(self) -> Size: + "Return the size of the output window." + + def bell(self) -> None: + "Sound bell." + + def enable_bracketed_paste(self) -> None: + "For vt100 only." + + def disable_bracketed_paste(self) -> None: + "For vt100 only." + + def reset_cursor_key_mode(self) -> None: + """ + For vt100 only. + Put the terminal in normal cursor mode (instead of application mode). + + See: https://vt100.net/docs/vt100-ug/chapter3.html + """ + + def scroll_buffer_to_prompt(self) -> None: + "For Win32 only." + + def get_rows_below_cursor_position(self) -> int: + "For Windows only." + raise NotImplementedError + + @abstractmethod + def get_default_color_depth(self) -> ColorDepth: + """ + Get default color depth for this output. + + This value will be used if no color depth was explicitly passed to the + `Application`. + + .. note:: + + If the `$PROMPT_TOOLKIT_COLOR_DEPTH` environment variable has been + set, then `outputs.defaults.create_output` will pass this value to + the implementation as the default_color_depth, which is returned + here. (This is not used when the output corresponds to a + prompt_toolkit SSH/Telnet session.) + """ + + +class DummyOutput(Output): + """ + For testing. An output class that doesn't render anything. + """ + + def fileno(self) -> int: + "There is no sensible default for fileno()." + raise NotImplementedError + + def encoding(self) -> str: + return "utf-8" + + def write(self, data: str) -> None: + pass + + def write_raw(self, data: str) -> None: + pass + + def set_title(self, title: str) -> None: + pass + + def clear_title(self) -> None: + pass + + def flush(self) -> None: + pass + + def erase_screen(self) -> None: + pass + + def enter_alternate_screen(self) -> None: + pass + + def quit_alternate_screen(self) -> None: + pass + + def enable_mouse_support(self) -> None: + pass + + def disable_mouse_support(self) -> None: + pass + + def erase_end_of_line(self) -> None: + pass + + def erase_down(self) -> None: + pass + + def reset_attributes(self) -> None: + pass + + def set_attributes(self, attrs: Attrs, color_depth: ColorDepth) -> None: + pass + + def disable_autowrap(self) -> None: + pass + + def enable_autowrap(self) -> None: + pass + + def cursor_goto(self, row: int = 0, column: int = 0) -> None: + pass + + def cursor_up(self, amount: int) -> None: + pass + + def cursor_down(self, amount: int) -> None: + pass + + def cursor_forward(self, amount: int) -> None: + pass + + def cursor_backward(self, amount: int) -> None: + pass + + def hide_cursor(self) -> None: + pass + + def show_cursor(self) -> None: + pass + + def set_cursor_shape(self, cursor_shape: CursorShape) -> None: + pass + + def reset_cursor_shape(self) -> None: + pass + + def ask_for_cpr(self) -> None: + pass + + def bell(self) -> None: + pass + + def enable_bracketed_paste(self) -> None: + pass + + def disable_bracketed_paste(self) -> None: + pass + + def scroll_buffer_to_prompt(self) -> None: + pass + + def get_size(self) -> Size: + return Size(rows=40, columns=80) + + def get_rows_below_cursor_position(self) -> int: + return 40 + + def get_default_color_depth(self) -> ColorDepth: + return ColorDepth.DEPTH_1_BIT diff --git a/lib/prompt_toolkit/output/color_depth.py b/lib/prompt_toolkit/output/color_depth.py new file mode 100644 index 0000000..f66d2be --- /dev/null +++ b/lib/prompt_toolkit/output/color_depth.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os +from enum import Enum + +__all__ = [ + "ColorDepth", +] + + +class ColorDepth(str, Enum): + """ + Possible color depth values for the output. + """ + + value: str + + #: One color only. + DEPTH_1_BIT = "DEPTH_1_BIT" + + #: ANSI Colors. + DEPTH_4_BIT = "DEPTH_4_BIT" + + #: The default. + DEPTH_8_BIT = "DEPTH_8_BIT" + + #: 24 bit True color. + DEPTH_24_BIT = "DEPTH_24_BIT" + + # Aliases. + MONOCHROME = DEPTH_1_BIT + ANSI_COLORS_ONLY = DEPTH_4_BIT + DEFAULT = DEPTH_8_BIT + TRUE_COLOR = DEPTH_24_BIT + + @classmethod + def from_env(cls) -> ColorDepth | None: + """ + Return the color depth if the $PROMPT_TOOLKIT_COLOR_DEPTH environment + variable has been set. + + This is a way to enforce a certain color depth in all prompt_toolkit + applications. + """ + # Disable color if a `NO_COLOR` environment variable is set. + # See: https://no-color.org/ + if os.environ.get("NO_COLOR"): + return cls.DEPTH_1_BIT + + # Check the `PROMPT_TOOLKIT_COLOR_DEPTH` environment variable. + all_values = [i.value for i in ColorDepth] + if os.environ.get("PROMPT_TOOLKIT_COLOR_DEPTH") in all_values: + return cls(os.environ["PROMPT_TOOLKIT_COLOR_DEPTH"]) + + return None + + @classmethod + def default(cls) -> ColorDepth: + """ + Return the default color depth for the default output. + """ + from .defaults import create_output + + return create_output().get_default_color_depth() diff --git a/lib/prompt_toolkit/output/conemu.py b/lib/prompt_toolkit/output/conemu.py new file mode 100644 index 0000000..6369944 --- /dev/null +++ b/lib/prompt_toolkit/output/conemu.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import sys + +assert sys.platform == "win32" + +from typing import Any, TextIO + +from prompt_toolkit.data_structures import Size + +from .base import Output +from .color_depth import ColorDepth +from .vt100 import Vt100_Output +from .win32 import Win32Output + +__all__ = [ + "ConEmuOutput", +] + + +class ConEmuOutput: + """ + ConEmu (Windows) output abstraction. + + ConEmu is a Windows console application, but it also supports ANSI escape + sequences. This output class is actually a proxy to both `Win32Output` and + `Vt100_Output`. It uses `Win32Output` for console sizing and scrolling, but + all cursor movements and scrolling happens through the `Vt100_Output`. + + This way, we can have 256 colors in ConEmu and Cmder. Rendering will be + even a little faster as well. + + http://conemu.github.io/ + http://gooseberrycreative.com/cmder/ + """ + + def __init__( + self, stdout: TextIO, default_color_depth: ColorDepth | None = None + ) -> None: + self.win32_output = Win32Output(stdout, default_color_depth=default_color_depth) + self.vt100_output = Vt100_Output( + stdout, lambda: Size(0, 0), default_color_depth=default_color_depth + ) + + @property + def responds_to_cpr(self) -> bool: + return False # We don't need this on Windows. + + def __getattr__(self, name: str) -> Any: + if name in ( + "get_size", + "get_rows_below_cursor_position", + "enable_mouse_support", + "disable_mouse_support", + "scroll_buffer_to_prompt", + "get_win32_screen_buffer_info", + "enable_bracketed_paste", + "disable_bracketed_paste", + ): + return getattr(self.win32_output, name) + else: + return getattr(self.vt100_output, name) + + +Output.register(ConEmuOutput) diff --git a/lib/prompt_toolkit/output/defaults.py b/lib/prompt_toolkit/output/defaults.py new file mode 100644 index 0000000..6b06ed4 --- /dev/null +++ b/lib/prompt_toolkit/output/defaults.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, TextIO, cast + +from prompt_toolkit.utils import ( + get_bell_environment_variable, + get_term_environment_variable, + is_conemu_ansi, +) + +from .base import DummyOutput, Output +from .color_depth import ColorDepth +from .plain_text import PlainTextOutput + +if TYPE_CHECKING: + from prompt_toolkit.patch_stdout import StdoutProxy + + +__all__ = [ + "create_output", +] + + +def create_output( + stdout: TextIO | StdoutProxy | None = None, always_prefer_tty: bool = False +) -> Output: + """ + Return an :class:`~prompt_toolkit.output.Output` instance for the command + line. + + :param stdout: The stdout object + :param always_prefer_tty: When set, look for `sys.stderr` if `sys.stdout` + is not a TTY. Useful if `sys.stdout` is redirected to a file, but we + still want user input and output on the terminal. + + By default, this is `False`. If `sys.stdout` is not a terminal (maybe + it's redirected to a file), then a `PlainTextOutput` will be returned. + That way, tools like `print_formatted_text` will write plain text into + that file. + """ + # Consider TERM, PROMPT_TOOLKIT_BELL, and PROMPT_TOOLKIT_COLOR_DEPTH + # environment variables. Notice that PROMPT_TOOLKIT_COLOR_DEPTH value is + # the default that's used if the Application doesn't override it. + term_from_env = get_term_environment_variable() + bell_from_env = get_bell_environment_variable() + color_depth_from_env = ColorDepth.from_env() + + if stdout is None: + # By default, render to stdout. If the output is piped somewhere else, + # render to stderr. + stdout = sys.stdout + + if always_prefer_tty: + for io in [sys.stdout, sys.stderr]: + if io is not None and io.isatty(): + # (This is `None` when using `pythonw.exe` on Windows.) + stdout = io + break + + # If the patch_stdout context manager has been used, then sys.stdout is + # replaced by this proxy. For prompt_toolkit applications, we want to use + # the real stdout. + from prompt_toolkit.patch_stdout import StdoutProxy + + while isinstance(stdout, StdoutProxy): + stdout = stdout.original_stdout + + # If the output is still `None`, use a DummyOutput. + # This happens for instance on Windows, when running the application under + # `pythonw.exe`. In that case, there won't be a terminal Window, and + # stdin/stdout/stderr are `None`. + if stdout is None: + return DummyOutput() + + if sys.platform == "win32": + from .conemu import ConEmuOutput + from .win32 import Win32Output + from .windows10 import Windows10_Output, is_win_vt100_enabled + + if is_win_vt100_enabled(): + return cast( + Output, + Windows10_Output(stdout, default_color_depth=color_depth_from_env), + ) + if is_conemu_ansi(): + return cast( + Output, ConEmuOutput(stdout, default_color_depth=color_depth_from_env) + ) + else: + return Win32Output(stdout, default_color_depth=color_depth_from_env) + else: + from .vt100 import Vt100_Output + + # Stdout is not a TTY? Render as plain text. + # This is mostly useful if stdout is redirected to a file, and + # `print_formatted_text` is used. + if not stdout.isatty(): + return PlainTextOutput(stdout) + + return Vt100_Output.from_pty( + stdout, + term=term_from_env, + default_color_depth=color_depth_from_env, + enable_bell=bell_from_env, + ) diff --git a/lib/prompt_toolkit/output/flush_stdout.py b/lib/prompt_toolkit/output/flush_stdout.py new file mode 100644 index 0000000..daf58ef --- /dev/null +++ b/lib/prompt_toolkit/output/flush_stdout.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import errno +import os +import sys +from contextlib import contextmanager +from typing import IO, Iterator, TextIO + +__all__ = ["flush_stdout"] + + +def flush_stdout(stdout: TextIO, data: str) -> None: + # If the IO object has an `encoding` and `buffer` attribute, it means that + # we can access the underlying BinaryIO object and write into it in binary + # mode. This is preferred if possible. + # NOTE: When used in a Jupyter notebook, don't write binary. + # `ipykernel.iostream.OutStream` has an `encoding` attribute, but not + # a `buffer` attribute, so we can't write binary in it. + has_binary_io = hasattr(stdout, "encoding") and hasattr(stdout, "buffer") + + try: + # Ensure that `stdout` is made blocking when writing into it. + # Otherwise, when uvloop is activated (which makes stdout + # non-blocking), and we write big amounts of text, then we get a + # `BlockingIOError` here. + with _blocking_io(stdout): + # (We try to encode ourself, because that way we can replace + # characters that don't exist in the character set, avoiding + # UnicodeEncodeError crashes. E.g. u'\xb7' does not appear in 'ascii'.) + # My Arch Linux installation of july 2015 reported 'ANSI_X3.4-1968' + # for sys.stdout.encoding in xterm. + if has_binary_io: + stdout.buffer.write(data.encode(stdout.encoding or "utf-8", "replace")) + else: + stdout.write(data) + + stdout.flush() + except OSError as e: + if e.args and e.args[0] == errno.EINTR: + # Interrupted system call. Can happen in case of a window + # resize signal. (Just ignore. The resize handler will render + # again anyway.) + pass + elif e.args and e.args[0] == 0: + # This can happen when there is a lot of output and the user + # sends a KeyboardInterrupt by pressing Control-C. E.g. in + # a Python REPL when we execute "while True: print('test')". + # (The `ptpython` REPL uses this `Output` class instead of + # `stdout` directly -- in order to be network transparent.) + # So, just ignore. + pass + else: + raise + + +@contextmanager +def _blocking_io(io: IO[str]) -> Iterator[None]: + """ + Ensure that the FD for `io` is set to blocking in here. + """ + if sys.platform == "win32": + # On Windows, the `os` module doesn't have a `get/set_blocking` + # function. + yield + return + + try: + fd = io.fileno() + blocking = os.get_blocking(fd) + except: # noqa + # Failed somewhere. + # `get_blocking` can raise `OSError`. + # The io object can raise `AttributeError` when no `fileno()` method is + # present if we're not a real file object. + blocking = True # Assume we're good, and don't do anything. + + try: + # Make blocking if we weren't blocking yet. + if not blocking: + os.set_blocking(fd, True) + + yield + + finally: + # Restore original blocking mode. + if not blocking: + os.set_blocking(fd, blocking) diff --git a/lib/prompt_toolkit/output/plain_text.py b/lib/prompt_toolkit/output/plain_text.py new file mode 100644 index 0000000..4b24ad9 --- /dev/null +++ b/lib/prompt_toolkit/output/plain_text.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from typing import TextIO + +from prompt_toolkit.cursor_shapes import CursorShape +from prompt_toolkit.data_structures import Size +from prompt_toolkit.styles import Attrs + +from .base import Output +from .color_depth import ColorDepth +from .flush_stdout import flush_stdout + +__all__ = ["PlainTextOutput"] + + +class PlainTextOutput(Output): + """ + Output that won't include any ANSI escape sequences. + + Useful when stdout is not a terminal. Maybe stdout is redirected to a file. + In this case, if `print_formatted_text` is used, for instance, we don't + want to include formatting. + + (The code is mostly identical to `Vt100_Output`, but without the + formatting.) + """ + + def __init__(self, stdout: TextIO) -> None: + assert all(hasattr(stdout, a) for a in ("write", "flush")) + + self.stdout: TextIO = stdout + self._buffer: list[str] = [] + + def fileno(self) -> int: + "There is no sensible default for fileno()." + return self.stdout.fileno() + + def encoding(self) -> str: + return "utf-8" + + def write(self, data: str) -> None: + self._buffer.append(data) + + def write_raw(self, data: str) -> None: + self._buffer.append(data) + + def set_title(self, title: str) -> None: + pass + + def clear_title(self) -> None: + pass + + def flush(self) -> None: + if not self._buffer: + return + + data = "".join(self._buffer) + self._buffer = [] + flush_stdout(self.stdout, data) + + def erase_screen(self) -> None: + pass + + def enter_alternate_screen(self) -> None: + pass + + def quit_alternate_screen(self) -> None: + pass + + def enable_mouse_support(self) -> None: + pass + + def disable_mouse_support(self) -> None: + pass + + def erase_end_of_line(self) -> None: + pass + + def erase_down(self) -> None: + pass + + def reset_attributes(self) -> None: + pass + + def set_attributes(self, attrs: Attrs, color_depth: ColorDepth) -> None: + pass + + def disable_autowrap(self) -> None: + pass + + def enable_autowrap(self) -> None: + pass + + def cursor_goto(self, row: int = 0, column: int = 0) -> None: + pass + + def cursor_up(self, amount: int) -> None: + pass + + def cursor_down(self, amount: int) -> None: + self._buffer.append("\n") + + def cursor_forward(self, amount: int) -> None: + self._buffer.append(" " * amount) + + def cursor_backward(self, amount: int) -> None: + pass + + def hide_cursor(self) -> None: + pass + + def show_cursor(self) -> None: + pass + + def set_cursor_shape(self, cursor_shape: CursorShape) -> None: + pass + + def reset_cursor_shape(self) -> None: + pass + + def ask_for_cpr(self) -> None: + pass + + def bell(self) -> None: + pass + + def enable_bracketed_paste(self) -> None: + pass + + def disable_bracketed_paste(self) -> None: + pass + + def scroll_buffer_to_prompt(self) -> None: + pass + + def get_size(self) -> Size: + return Size(rows=40, columns=80) + + def get_rows_below_cursor_position(self) -> int: + return 8 + + def get_default_color_depth(self) -> ColorDepth: + return ColorDepth.DEPTH_1_BIT diff --git a/lib/prompt_toolkit/output/vt100.py b/lib/prompt_toolkit/output/vt100.py new file mode 100644 index 0000000..57826b9 --- /dev/null +++ b/lib/prompt_toolkit/output/vt100.py @@ -0,0 +1,760 @@ +""" +Output for vt100 terminals. + +A lot of thanks, regarding outputting of colors, goes to the Pygments project: +(We don't rely on Pygments anymore, because many things are very custom, and +everything has been highly optimized.) +http://pygments.org/ +""" + +from __future__ import annotations + +import io +import os +import sys +from typing import Callable, Dict, Hashable, Iterable, Sequence, TextIO, Tuple + +from prompt_toolkit.cursor_shapes import CursorShape +from prompt_toolkit.data_structures import Size +from prompt_toolkit.output import Output +from prompt_toolkit.styles import ANSI_COLOR_NAMES, Attrs +from prompt_toolkit.utils import is_dumb_terminal + +from .color_depth import ColorDepth +from .flush_stdout import flush_stdout + +__all__ = [ + "Vt100_Output", +] + + +FG_ANSI_COLORS = { + "ansidefault": 39, + # Low intensity. + "ansiblack": 30, + "ansired": 31, + "ansigreen": 32, + "ansiyellow": 33, + "ansiblue": 34, + "ansimagenta": 35, + "ansicyan": 36, + "ansigray": 37, + # High intensity. + "ansibrightblack": 90, + "ansibrightred": 91, + "ansibrightgreen": 92, + "ansibrightyellow": 93, + "ansibrightblue": 94, + "ansibrightmagenta": 95, + "ansibrightcyan": 96, + "ansiwhite": 97, +} + +BG_ANSI_COLORS = { + "ansidefault": 49, + # Low intensity. + "ansiblack": 40, + "ansired": 41, + "ansigreen": 42, + "ansiyellow": 43, + "ansiblue": 44, + "ansimagenta": 45, + "ansicyan": 46, + "ansigray": 47, + # High intensity. + "ansibrightblack": 100, + "ansibrightred": 101, + "ansibrightgreen": 102, + "ansibrightyellow": 103, + "ansibrightblue": 104, + "ansibrightmagenta": 105, + "ansibrightcyan": 106, + "ansiwhite": 107, +} + + +ANSI_COLORS_TO_RGB = { + "ansidefault": ( + 0x00, + 0x00, + 0x00, + ), # Don't use, 'default' doesn't really have a value. + "ansiblack": (0x00, 0x00, 0x00), + "ansigray": (0xE5, 0xE5, 0xE5), + "ansibrightblack": (0x7F, 0x7F, 0x7F), + "ansiwhite": (0xFF, 0xFF, 0xFF), + # Low intensity. + "ansired": (0xCD, 0x00, 0x00), + "ansigreen": (0x00, 0xCD, 0x00), + "ansiyellow": (0xCD, 0xCD, 0x00), + "ansiblue": (0x00, 0x00, 0xCD), + "ansimagenta": (0xCD, 0x00, 0xCD), + "ansicyan": (0x00, 0xCD, 0xCD), + # High intensity. + "ansibrightred": (0xFF, 0x00, 0x00), + "ansibrightgreen": (0x00, 0xFF, 0x00), + "ansibrightyellow": (0xFF, 0xFF, 0x00), + "ansibrightblue": (0x00, 0x00, 0xFF), + "ansibrightmagenta": (0xFF, 0x00, 0xFF), + "ansibrightcyan": (0x00, 0xFF, 0xFF), +} + + +assert set(FG_ANSI_COLORS) == set(ANSI_COLOR_NAMES) +assert set(BG_ANSI_COLORS) == set(ANSI_COLOR_NAMES) +assert set(ANSI_COLORS_TO_RGB) == set(ANSI_COLOR_NAMES) + + +def _get_closest_ansi_color(r: int, g: int, b: int, exclude: Sequence[str] = ()) -> str: + """ + Find closest ANSI color. Return it by name. + + :param r: Red (Between 0 and 255.) + :param g: Green (Between 0 and 255.) + :param b: Blue (Between 0 and 255.) + :param exclude: A tuple of color names to exclude. (E.g. ``('ansired', )``.) + """ + exclude = list(exclude) + + # When we have a bit of saturation, avoid the gray-like colors, otherwise, + # too often the distance to the gray color is less. + saturation = abs(r - g) + abs(g - b) + abs(b - r) # Between 0..510 + + if saturation > 30: + exclude.extend(["ansilightgray", "ansidarkgray", "ansiwhite", "ansiblack"]) + + # Take the closest color. + # (Thanks to Pygments for this part.) + distance = 257 * 257 * 3 # "infinity" (>distance from #000000 to #ffffff) + match = "ansidefault" + + for name, (r2, g2, b2) in ANSI_COLORS_TO_RGB.items(): + if name != "ansidefault" and name not in exclude: + d = (r - r2) ** 2 + (g - g2) ** 2 + (b - b2) ** 2 + + if d < distance: + match = name + distance = d + + return match + + +_ColorCodeAndName = Tuple[int, str] + + +class _16ColorCache: + """ + Cache which maps (r, g, b) tuples to 16 ansi colors. + + :param bg: Cache for background colors, instead of foreground. + """ + + def __init__(self, bg: bool = False) -> None: + self.bg = bg + self._cache: dict[Hashable, _ColorCodeAndName] = {} + + def get_code( + self, value: tuple[int, int, int], exclude: Sequence[str] = () + ) -> _ColorCodeAndName: + """ + Return a (ansi_code, ansi_name) tuple. (E.g. ``(44, 'ansiblue')``.) for + a given (r,g,b) value. + """ + key: Hashable = (value, tuple(exclude)) + cache = self._cache + + if key not in cache: + cache[key] = self._get(value, exclude) + + return cache[key] + + def _get( + self, value: tuple[int, int, int], exclude: Sequence[str] = () + ) -> _ColorCodeAndName: + r, g, b = value + match = _get_closest_ansi_color(r, g, b, exclude=exclude) + + # Turn color name into code. + if self.bg: + code = BG_ANSI_COLORS[match] + else: + code = FG_ANSI_COLORS[match] + + return code, match + + +class _256ColorCache(Dict[Tuple[int, int, int], int]): + """ + Cache which maps (r, g, b) tuples to 256 colors. + """ + + def __init__(self) -> None: + # Build color table. + colors: list[tuple[int, int, int]] = [] + + # colors 0..15: 16 basic colors + colors.append((0x00, 0x00, 0x00)) # 0 + colors.append((0xCD, 0x00, 0x00)) # 1 + colors.append((0x00, 0xCD, 0x00)) # 2 + colors.append((0xCD, 0xCD, 0x00)) # 3 + colors.append((0x00, 0x00, 0xEE)) # 4 + colors.append((0xCD, 0x00, 0xCD)) # 5 + colors.append((0x00, 0xCD, 0xCD)) # 6 + colors.append((0xE5, 0xE5, 0xE5)) # 7 + colors.append((0x7F, 0x7F, 0x7F)) # 8 + colors.append((0xFF, 0x00, 0x00)) # 9 + colors.append((0x00, 0xFF, 0x00)) # 10 + colors.append((0xFF, 0xFF, 0x00)) # 11 + colors.append((0x5C, 0x5C, 0xFF)) # 12 + colors.append((0xFF, 0x00, 0xFF)) # 13 + colors.append((0x00, 0xFF, 0xFF)) # 14 + colors.append((0xFF, 0xFF, 0xFF)) # 15 + + # colors 16..232: the 6x6x6 color cube + valuerange = (0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF) + + for i in range(217): + r = valuerange[(i // 36) % 6] + g = valuerange[(i // 6) % 6] + b = valuerange[i % 6] + colors.append((r, g, b)) + + # colors 233..253: grayscale + for i in range(1, 22): + v = 8 + i * 10 + colors.append((v, v, v)) + + self.colors = colors + + def __missing__(self, value: tuple[int, int, int]) -> int: + r, g, b = value + + # Find closest color. + # (Thanks to Pygments for this!) + distance = 257 * 257 * 3 # "infinity" (>distance from #000000 to #ffffff) + match = 0 + + for i, (r2, g2, b2) in enumerate(self.colors): + if i >= 16: # XXX: We ignore the 16 ANSI colors when mapping RGB + # to the 256 colors, because these highly depend on + # the color scheme of the terminal. + d = (r - r2) ** 2 + (g - g2) ** 2 + (b - b2) ** 2 + + if d < distance: + match = i + distance = d + + # Turn color name into code. + self[value] = match + return match + + +_16_fg_colors = _16ColorCache(bg=False) +_16_bg_colors = _16ColorCache(bg=True) +_256_colors = _256ColorCache() + + +class _EscapeCodeCache(Dict[Attrs, str]): + """ + Cache for VT100 escape codes. It maps + (fgcolor, bgcolor, bold, underline, strike, italic, blink, reverse, hidden, dim) tuples to VT100 + escape sequences. + + :param true_color: When True, use 24bit colors instead of 256 colors. + """ + + def __init__(self, color_depth: ColorDepth) -> None: + self.color_depth = color_depth + + def __missing__(self, attrs: Attrs) -> str: + ( + fgcolor, + bgcolor, + bold, + underline, + strike, + italic, + blink, + reverse, + hidden, + dim, + ) = attrs + parts: list[str] = [] + + parts.extend(self._colors_to_code(fgcolor or "", bgcolor or "")) + + if bold: + parts.append("1") + if dim: + parts.append("2") + if italic: + parts.append("3") + if blink: + parts.append("5") + if underline: + parts.append("4") + if reverse: + parts.append("7") + if hidden: + parts.append("8") + if strike: + parts.append("9") + + if parts: + result = "\x1b[0;" + ";".join(parts) + "m" + else: + result = "\x1b[0m" + + self[attrs] = result + return result + + def _color_name_to_rgb(self, color: str) -> tuple[int, int, int]: + "Turn 'ffffff', into (0xff, 0xff, 0xff)." + try: + rgb = int(color, 16) + except ValueError: + raise + else: + r = (rgb >> 16) & 0xFF + g = (rgb >> 8) & 0xFF + b = rgb & 0xFF + return r, g, b + + def _colors_to_code(self, fg_color: str, bg_color: str) -> Iterable[str]: + """ + Return a tuple with the vt100 values that represent this color. + """ + # When requesting ANSI colors only, and both fg/bg color were converted + # to ANSI, ensure that the foreground and background color are not the + # same. (Unless they were explicitly defined to be the same color.) + fg_ansi = "" + + def get(color: str, bg: bool) -> list[int]: + nonlocal fg_ansi + + table = BG_ANSI_COLORS if bg else FG_ANSI_COLORS + + if not color or self.color_depth == ColorDepth.DEPTH_1_BIT: + return [] + + # 16 ANSI colors. (Given by name.) + elif color in table: + return [table[color]] + + # RGB colors. (Defined as 'ffffff'.) + else: + try: + rgb = self._color_name_to_rgb(color) + except ValueError: + return [] + + # When only 16 colors are supported, use that. + if self.color_depth == ColorDepth.DEPTH_4_BIT: + if bg: # Background. + if fg_color != bg_color: + exclude = [fg_ansi] + else: + exclude = [] + code, name = _16_bg_colors.get_code(rgb, exclude=exclude) + return [code] + else: # Foreground. + code, name = _16_fg_colors.get_code(rgb) + fg_ansi = name + return [code] + + # True colors. (Only when this feature is enabled.) + elif self.color_depth == ColorDepth.DEPTH_24_BIT: + r, g, b = rgb + return [(48 if bg else 38), 2, r, g, b] + + # 256 RGB colors. + else: + return [(48 if bg else 38), 5, _256_colors[rgb]] + + result: list[int] = [] + result.extend(get(fg_color, False)) + result.extend(get(bg_color, True)) + + return map(str, result) + + +def _get_size(fileno: int) -> tuple[int, int]: + """ + Get the size of this pseudo terminal. + + :param fileno: stdout.fileno() + :returns: A (rows, cols) tuple. + """ + size = os.get_terminal_size(fileno) + return size.lines, size.columns + + +class Vt100_Output(Output): + """ + :param get_size: A callable which returns the `Size` of the output terminal. + :param stdout: Any object with has a `write` and `flush` method + an 'encoding' property. + :param term: The terminal environment variable. (xterm, xterm-256color, linux, ...) + :param enable_cpr: When `True` (the default), send "cursor position + request" escape sequences to the output in order to detect the cursor + position. That way, we can properly determine how much space there is + available for the UI (especially for drop down menus) to render. The + `Renderer` will still try to figure out whether the current terminal + does respond to CPR escapes. When `False`, never attempt to send CPR + requests. + """ + + # For the error messages. Only display "Output is not a terminal" once per + # file descriptor. + _fds_not_a_terminal: set[int] = set() + + def __init__( + self, + stdout: TextIO, + get_size: Callable[[], Size], + term: str | None = None, + default_color_depth: ColorDepth | None = None, + enable_bell: bool = True, + enable_cpr: bool = True, + ) -> None: + assert all(hasattr(stdout, a) for a in ("write", "flush")) + + self._buffer: list[str] = [] + self.stdout: TextIO = stdout + self.default_color_depth = default_color_depth + self._get_size = get_size + self.term = term + self.enable_bell = enable_bell + self.enable_cpr = enable_cpr + + # Cache for escape codes. + self._escape_code_caches: dict[ColorDepth, _EscapeCodeCache] = { + ColorDepth.DEPTH_1_BIT: _EscapeCodeCache(ColorDepth.DEPTH_1_BIT), + ColorDepth.DEPTH_4_BIT: _EscapeCodeCache(ColorDepth.DEPTH_4_BIT), + ColorDepth.DEPTH_8_BIT: _EscapeCodeCache(ColorDepth.DEPTH_8_BIT), + ColorDepth.DEPTH_24_BIT: _EscapeCodeCache(ColorDepth.DEPTH_24_BIT), + } + + # Keep track of whether the cursor shape was ever changed. + # (We don't restore the cursor shape if it was never changed - by + # default, we don't change them.) + self._cursor_shape_changed = False + + # Don't hide/show the cursor when this was already done. + # (`None` means that we don't know whether the cursor is visible or + # not.) + self._cursor_visible: bool | None = None + + @classmethod + def from_pty( + cls, + stdout: TextIO, + term: str | None = None, + default_color_depth: ColorDepth | None = None, + enable_bell: bool = True, + ) -> Vt100_Output: + """ + Create an Output class from a pseudo terminal. + (This will take the dimensions by reading the pseudo + terminal attributes.) + """ + fd: int | None + # Normally, this requires a real TTY device, but people instantiate + # this class often during unit tests as well. For convenience, we print + # an error message, use standard dimensions, and go on. + try: + fd = stdout.fileno() + except io.UnsupportedOperation: + fd = None + + if not stdout.isatty() and (fd is None or fd not in cls._fds_not_a_terminal): + msg = "Warning: Output is not a terminal (fd=%r).\n" + sys.stderr.write(msg % fd) + sys.stderr.flush() + if fd is not None: + cls._fds_not_a_terminal.add(fd) + + def get_size() -> Size: + # If terminal (incorrectly) reports its size as 0, pick a + # reasonable default. See + # https://github.com/ipython/ipython/issues/10071 + rows, columns = (None, None) + + # It is possible that `stdout` is no longer a TTY device at this + # point. In that case we get an `OSError` in the ioctl call in + # `get_size`. See: + # https://github.com/prompt-toolkit/python-prompt-toolkit/pull/1021 + try: + rows, columns = _get_size(stdout.fileno()) + except OSError: + pass + return Size(rows=rows or 24, columns=columns or 80) + + return cls( + stdout, + get_size, + term=term, + default_color_depth=default_color_depth, + enable_bell=enable_bell, + ) + + def get_size(self) -> Size: + return self._get_size() + + def fileno(self) -> int: + "Return file descriptor." + return self.stdout.fileno() + + def encoding(self) -> str: + "Return encoding used for stdout." + return self.stdout.encoding + + def write_raw(self, data: str) -> None: + """ + Write raw data to output. + """ + self._buffer.append(data) + + def write(self, data: str) -> None: + """ + Write text to output. + (Removes vt100 escape codes. -- used for safely writing text.) + """ + self._buffer.append(data.replace("\x1b", "?")) + + def set_title(self, title: str) -> None: + """ + Set terminal title. + """ + if self.term not in ( + "linux", + "eterm-color", + ): # Not supported by the Linux console. + self.write_raw( + "\x1b]2;{}\x07".format(title.replace("\x1b", "").replace("\x07", "")) + ) + + def clear_title(self) -> None: + self.set_title("") + + def erase_screen(self) -> None: + """ + Erases the screen with the background color and moves the cursor to + home. + """ + self.write_raw("\x1b[2J") + + def enter_alternate_screen(self) -> None: + self.write_raw("\x1b[?1049h\x1b[H") + + def quit_alternate_screen(self) -> None: + self.write_raw("\x1b[?1049l") + + def enable_mouse_support(self) -> None: + self.write_raw("\x1b[?1000h") + + # Enable mouse-drag support. + self.write_raw("\x1b[?1003h") + + # Enable urxvt Mouse mode. (For terminals that understand this.) + self.write_raw("\x1b[?1015h") + + # Also enable Xterm SGR mouse mode. (For terminals that understand this.) + self.write_raw("\x1b[?1006h") + + # Note: E.g. lxterminal understands 1000h, but not the urxvt or sgr + # extensions. + + def disable_mouse_support(self) -> None: + self.write_raw("\x1b[?1000l") + self.write_raw("\x1b[?1015l") + self.write_raw("\x1b[?1006l") + self.write_raw("\x1b[?1003l") + + def erase_end_of_line(self) -> None: + """ + Erases from the current cursor position to the end of the current line. + """ + self.write_raw("\x1b[K") + + def erase_down(self) -> None: + """ + Erases the screen from the current line down to the bottom of the + screen. + """ + self.write_raw("\x1b[J") + + def reset_attributes(self) -> None: + self.write_raw("\x1b[0m") + + def set_attributes(self, attrs: Attrs, color_depth: ColorDepth) -> None: + """ + Create new style and output. + + :param attrs: `Attrs` instance. + """ + # Get current depth. + escape_code_cache = self._escape_code_caches[color_depth] + + # Write escape character. + self.write_raw(escape_code_cache[attrs]) + + def disable_autowrap(self) -> None: + self.write_raw("\x1b[?7l") + + def enable_autowrap(self) -> None: + self.write_raw("\x1b[?7h") + + def enable_bracketed_paste(self) -> None: + self.write_raw("\x1b[?2004h") + + def disable_bracketed_paste(self) -> None: + self.write_raw("\x1b[?2004l") + + def reset_cursor_key_mode(self) -> None: + """ + For vt100 only. + Put the terminal in cursor mode (instead of application mode). + """ + # Put the terminal in cursor mode. (Instead of application mode.) + self.write_raw("\x1b[?1l") + + def cursor_goto(self, row: int = 0, column: int = 0) -> None: + """ + Move cursor position. + """ + self.write_raw("\x1b[%i;%iH" % (row, column)) + + def cursor_up(self, amount: int) -> None: + if amount == 0: + pass + elif amount == 1: + self.write_raw("\x1b[A") + else: + self.write_raw("\x1b[%iA" % amount) + + def cursor_down(self, amount: int) -> None: + if amount == 0: + pass + elif amount == 1: + # Note: Not the same as '\n', '\n' can cause the window content to + # scroll. + self.write_raw("\x1b[B") + else: + self.write_raw("\x1b[%iB" % amount) + + def cursor_forward(self, amount: int) -> None: + if amount == 0: + pass + elif amount == 1: + self.write_raw("\x1b[C") + else: + self.write_raw("\x1b[%iC" % amount) + + def cursor_backward(self, amount: int) -> None: + if amount == 0: + pass + elif amount == 1: + self.write_raw("\b") # '\x1b[D' + else: + self.write_raw("\x1b[%iD" % amount) + + def hide_cursor(self) -> None: + if self._cursor_visible in (True, None): + self._cursor_visible = False + self.write_raw("\x1b[?25l") + + def show_cursor(self) -> None: + if self._cursor_visible in (False, None): + self._cursor_visible = True + self.write_raw("\x1b[?12l\x1b[?25h") # Stop blinking cursor and show. + + def set_cursor_shape(self, cursor_shape: CursorShape) -> None: + if cursor_shape == CursorShape._NEVER_CHANGE: + return + + self._cursor_shape_changed = True + self.write_raw( + { + CursorShape.BLOCK: "\x1b[2 q", + CursorShape.BEAM: "\x1b[6 q", + CursorShape.UNDERLINE: "\x1b[4 q", + CursorShape.BLINKING_BLOCK: "\x1b[1 q", + CursorShape.BLINKING_BEAM: "\x1b[5 q", + CursorShape.BLINKING_UNDERLINE: "\x1b[3 q", + }.get(cursor_shape, "") + ) + + def reset_cursor_shape(self) -> None: + "Reset cursor shape." + # (Only reset cursor shape, if we ever changed it.) + if self._cursor_shape_changed: + self._cursor_shape_changed = False + + # Reset cursor shape. + self.write_raw("\x1b[0 q") + + def flush(self) -> None: + """ + Write to output stream and flush. + """ + if not self._buffer: + return + + data = "".join(self._buffer) + self._buffer = [] + + flush_stdout(self.stdout, data) + + def ask_for_cpr(self) -> None: + """ + Asks for a cursor position report (CPR). + """ + self.write_raw("\x1b[6n") + self.flush() + + @property + def responds_to_cpr(self) -> bool: + if not self.enable_cpr: + return False + + # When the input is a tty, we assume that CPR is supported. + # It's not when the input is piped from Pexpect. + if os.environ.get("PROMPT_TOOLKIT_NO_CPR", "") == "1": + return False + + if is_dumb_terminal(self.term): + return False + try: + return self.stdout.isatty() + except ValueError: + return False # ValueError: I/O operation on closed file + + def bell(self) -> None: + "Sound bell." + if self.enable_bell: + self.write_raw("\a") + self.flush() + + def get_default_color_depth(self) -> ColorDepth: + """ + Return the default color depth for a vt100 terminal, according to the + our term value. + + We prefer 256 colors almost always, because this is what most terminals + support these days, and is a good default. + """ + if self.default_color_depth is not None: + return self.default_color_depth + + term = self.term + + if term is None: + return ColorDepth.DEFAULT + + if is_dumb_terminal(term): + return ColorDepth.DEPTH_1_BIT + + if term in ("linux", "eterm-color"): + return ColorDepth.DEPTH_4_BIT + + return ColorDepth.DEFAULT diff --git a/lib/prompt_toolkit/output/win32.py b/lib/prompt_toolkit/output/win32.py new file mode 100644 index 0000000..1c0cc4e --- /dev/null +++ b/lib/prompt_toolkit/output/win32.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import sys + +assert sys.platform == "win32" + +import os +from ctypes import ArgumentError, byref, c_char, c_long, c_uint, c_ulong, pointer +from ctypes.wintypes import DWORD, HANDLE +from typing import Callable, TextIO, TypeVar + +from prompt_toolkit.cursor_shapes import CursorShape +from prompt_toolkit.data_structures import Size +from prompt_toolkit.styles import ANSI_COLOR_NAMES, Attrs +from prompt_toolkit.utils import get_cwidth +from prompt_toolkit.win32_types import ( + CONSOLE_SCREEN_BUFFER_INFO, + COORD, + SMALL_RECT, + STD_INPUT_HANDLE, + STD_OUTPUT_HANDLE, +) + +from ..utils import SPHINX_AUTODOC_RUNNING +from .base import Output +from .color_depth import ColorDepth + +# Do not import win32-specific stuff when generating documentation. +# Otherwise RTD would be unable to generate docs for this module. +if not SPHINX_AUTODOC_RUNNING: + from ctypes import windll + + +__all__ = [ + "Win32Output", +] + + +def _coord_byval(coord: COORD) -> c_long: + """ + Turns a COORD object into a c_long. + This will cause it to be passed by value instead of by reference. (That is what I think at least.) + + When running ``ptipython`` is run (only with IPython), we often got the following error:: + + Error in 'SetConsoleCursorPosition'. + ArgumentError("argument 2: : wrong type",) + argument 2: : wrong type + + It was solved by turning ``COORD`` parameters into a ``c_long`` like this. + + More info: http://msdn.microsoft.com/en-us/library/windows/desktop/ms686025(v=vs.85).aspx + """ + return c_long(coord.Y * 0x10000 | coord.X & 0xFFFF) + + +#: If True: write the output of the renderer also to the following file. This +#: is very useful for debugging. (e.g.: to see that we don't write more bytes +#: than required.) +_DEBUG_RENDER_OUTPUT = False +_DEBUG_RENDER_OUTPUT_FILENAME = r"prompt-toolkit-windows-output.log" + + +class NoConsoleScreenBufferError(Exception): + """ + Raised when the application is not running inside a Windows Console, but + the user tries to instantiate Win32Output. + """ + + def __init__(self) -> None: + # Are we running in 'xterm' on Windows, like git-bash for instance? + xterm = "xterm" in os.environ.get("TERM", "") + + if xterm: + message = ( + "Found {}, while expecting a Windows console. " + 'Maybe try to run this program using "winpty" ' + "or run it in cmd.exe instead. Or otherwise, " + "in case of Cygwin, use the Python executable " + "that is compiled for Cygwin.".format(os.environ["TERM"]) + ) + else: + message = "No Windows console found. Are you running cmd.exe?" + super().__init__(message) + + +_T = TypeVar("_T") + + +class Win32Output(Output): + """ + I/O abstraction for rendering to Windows consoles. + (cmd.exe and similar.) + """ + + def __init__( + self, + stdout: TextIO, + use_complete_width: bool = False, + default_color_depth: ColorDepth | None = None, + ) -> None: + self.use_complete_width = use_complete_width + self.default_color_depth = default_color_depth + + self._buffer: list[str] = [] + self.stdout: TextIO = stdout + self.hconsole = HANDLE(windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)) + + self._in_alternate_screen = False + self._hidden = False + + self.color_lookup_table = ColorLookupTable() + + # Remember the default console colors. + info = self.get_win32_screen_buffer_info() + self.default_attrs = info.wAttributes if info else 15 + + if _DEBUG_RENDER_OUTPUT: + self.LOG = open(_DEBUG_RENDER_OUTPUT_FILENAME, "ab") + + def fileno(self) -> int: + "Return file descriptor." + return self.stdout.fileno() + + def encoding(self) -> str: + "Return encoding used for stdout." + return self.stdout.encoding + + def write(self, data: str) -> None: + if self._hidden: + data = " " * get_cwidth(data) + + self._buffer.append(data) + + def write_raw(self, data: str) -> None: + "For win32, there is no difference between write and write_raw." + self.write(data) + + def get_size(self) -> Size: + info = self.get_win32_screen_buffer_info() + + # We take the width of the *visible* region as the size. Not the width + # of the complete screen buffer. (Unless use_complete_width has been + # set.) + if self.use_complete_width: + width = info.dwSize.X + else: + width = info.srWindow.Right - info.srWindow.Left + + height = info.srWindow.Bottom - info.srWindow.Top + 1 + + # We avoid the right margin, windows will wrap otherwise. + maxwidth = info.dwSize.X - 1 + width = min(maxwidth, width) + + # Create `Size` object. + return Size(rows=height, columns=width) + + def _winapi(self, func: Callable[..., _T], *a: object, **kw: object) -> _T: + """ + Flush and call win API function. + """ + self.flush() + + if _DEBUG_RENDER_OUTPUT: + self.LOG.write((f"{func.__name__!r}").encode() + b"\n") + self.LOG.write( + b" " + ", ".join([f"{i!r}" for i in a]).encode("utf-8") + b"\n" + ) + self.LOG.write( + b" " + + ", ".join([f"{type(i)!r}" for i in a]).encode("utf-8") + + b"\n" + ) + self.LOG.flush() + + try: + return func(*a, **kw) + except ArgumentError as e: + if _DEBUG_RENDER_OUTPUT: + self.LOG.write((f" Error in {func.__name__!r} {e!r} {e}\n").encode()) + + raise + + def get_win32_screen_buffer_info(self) -> CONSOLE_SCREEN_BUFFER_INFO: + """ + Return Screen buffer info. + """ + # NOTE: We don't call the `GetConsoleScreenBufferInfo` API through + # `self._winapi`. Doing so causes Python to crash on certain 64bit + # Python versions. (Reproduced with 64bit Python 2.7.6, on Windows + # 10). It is not clear why. Possibly, it has to do with passing + # these objects as an argument, or through *args. + + # The Python documentation contains the following - possibly related - warning: + # ctypes does not support passing unions or structures with + # bit-fields to functions by value. While this may work on 32-bit + # x86, it's not guaranteed by the library to work in the general + # case. Unions and structures with bit-fields should always be + # passed to functions by pointer. + + # Also see: + # - https://github.com/ipython/ipython/issues/10070 + # - https://github.com/jonathanslenders/python-prompt-toolkit/issues/406 + # - https://github.com/jonathanslenders/python-prompt-toolkit/issues/86 + + self.flush() + sbinfo = CONSOLE_SCREEN_BUFFER_INFO() + success = windll.kernel32.GetConsoleScreenBufferInfo( + self.hconsole, byref(sbinfo) + ) + + # success = self._winapi(windll.kernel32.GetConsoleScreenBufferInfo, + # self.hconsole, byref(sbinfo)) + + if success: + return sbinfo + else: + raise NoConsoleScreenBufferError + + def set_title(self, title: str) -> None: + """ + Set terminal title. + """ + self._winapi(windll.kernel32.SetConsoleTitleW, title) + + def clear_title(self) -> None: + self._winapi(windll.kernel32.SetConsoleTitleW, "") + + def erase_screen(self) -> None: + start = COORD(0, 0) + sbinfo = self.get_win32_screen_buffer_info() + length = sbinfo.dwSize.X * sbinfo.dwSize.Y + + self.cursor_goto(row=0, column=0) + self._erase(start, length) + + def erase_down(self) -> None: + sbinfo = self.get_win32_screen_buffer_info() + size = sbinfo.dwSize + + start = sbinfo.dwCursorPosition + length = (size.X - size.X) + size.X * (size.Y - sbinfo.dwCursorPosition.Y) + + self._erase(start, length) + + def erase_end_of_line(self) -> None: + """""" + sbinfo = self.get_win32_screen_buffer_info() + start = sbinfo.dwCursorPosition + length = sbinfo.dwSize.X - sbinfo.dwCursorPosition.X + + self._erase(start, length) + + def _erase(self, start: COORD, length: int) -> None: + chars_written = c_ulong() + + self._winapi( + windll.kernel32.FillConsoleOutputCharacterA, + self.hconsole, + c_char(b" "), + DWORD(length), + _coord_byval(start), + byref(chars_written), + ) + + # Reset attributes. + sbinfo = self.get_win32_screen_buffer_info() + self._winapi( + windll.kernel32.FillConsoleOutputAttribute, + self.hconsole, + sbinfo.wAttributes, + length, + _coord_byval(start), + byref(chars_written), + ) + + def reset_attributes(self) -> None: + "Reset the console foreground/background color." + self._winapi( + windll.kernel32.SetConsoleTextAttribute, self.hconsole, self.default_attrs + ) + self._hidden = False + + def set_attributes(self, attrs: Attrs, color_depth: ColorDepth) -> None: + ( + fgcolor, + bgcolor, + bold, + underline, + strike, + italic, + blink, + reverse, + hidden, + dim, + ) = attrs + self._hidden = bool(hidden) + + # Start from the default attributes. + win_attrs: int = self.default_attrs + + if color_depth != ColorDepth.DEPTH_1_BIT: + # Override the last four bits: foreground color. + if fgcolor: + win_attrs = win_attrs & ~0xF + win_attrs |= self.color_lookup_table.lookup_fg_color(fgcolor) + + # Override the next four bits: background color. + if bgcolor: + win_attrs = win_attrs & ~0xF0 + win_attrs |= self.color_lookup_table.lookup_bg_color(bgcolor) + + # Reverse: swap these four bits groups. + if reverse: + win_attrs = ( + (win_attrs & ~0xFF) + | ((win_attrs & 0xF) << 4) + | ((win_attrs & 0xF0) >> 4) + ) + + self._winapi(windll.kernel32.SetConsoleTextAttribute, self.hconsole, win_attrs) + + def disable_autowrap(self) -> None: + # Not supported by Windows. + pass + + def enable_autowrap(self) -> None: + # Not supported by Windows. + pass + + def cursor_goto(self, row: int = 0, column: int = 0) -> None: + pos = COORD(X=column, Y=row) + self._winapi( + windll.kernel32.SetConsoleCursorPosition, self.hconsole, _coord_byval(pos) + ) + + def cursor_up(self, amount: int) -> None: + sr = self.get_win32_screen_buffer_info().dwCursorPosition + pos = COORD(X=sr.X, Y=sr.Y - amount) + self._winapi( + windll.kernel32.SetConsoleCursorPosition, self.hconsole, _coord_byval(pos) + ) + + def cursor_down(self, amount: int) -> None: + self.cursor_up(-amount) + + def cursor_forward(self, amount: int) -> None: + sr = self.get_win32_screen_buffer_info().dwCursorPosition + # assert sr.X + amount >= 0, 'Negative cursor position: x=%r amount=%r' % (sr.X, amount) + + pos = COORD(X=max(0, sr.X + amount), Y=sr.Y) + self._winapi( + windll.kernel32.SetConsoleCursorPosition, self.hconsole, _coord_byval(pos) + ) + + def cursor_backward(self, amount: int) -> None: + self.cursor_forward(-amount) + + def flush(self) -> None: + """ + Write to output stream and flush. + """ + if not self._buffer: + # Only flush stdout buffer. (It could be that Python still has + # something in its buffer. -- We want to be sure to print that in + # the correct color.) + self.stdout.flush() + return + + data = "".join(self._buffer) + + if _DEBUG_RENDER_OUTPUT: + self.LOG.write((f"{data!r}").encode() + b"\n") + self.LOG.flush() + + # Print characters one by one. This appears to be the best solution + # in order to avoid traces of vertical lines when the completion + # menu disappears. + for b in data: + written = DWORD() + + retval = windll.kernel32.WriteConsoleW( + self.hconsole, b, 1, byref(written), None + ) + assert retval != 0 + + self._buffer = [] + + def get_rows_below_cursor_position(self) -> int: + info = self.get_win32_screen_buffer_info() + return info.srWindow.Bottom - info.dwCursorPosition.Y + 1 + + def scroll_buffer_to_prompt(self) -> None: + """ + To be called before drawing the prompt. This should scroll the console + to left, with the cursor at the bottom (if possible). + """ + # Get current window size + info = self.get_win32_screen_buffer_info() + sr = info.srWindow + cursor_pos = info.dwCursorPosition + + result = SMALL_RECT() + + # Scroll to the left. + result.Left = 0 + result.Right = sr.Right - sr.Left + + # Scroll vertical + win_height = sr.Bottom - sr.Top + if 0 < sr.Bottom - cursor_pos.Y < win_height - 1: + # no vertical scroll if cursor already on the screen + result.Bottom = sr.Bottom + else: + result.Bottom = max(win_height, cursor_pos.Y) + result.Top = result.Bottom - win_height + + # Scroll API + self._winapi( + windll.kernel32.SetConsoleWindowInfo, self.hconsole, True, byref(result) + ) + + def enter_alternate_screen(self) -> None: + """ + Go to alternate screen buffer. + """ + if not self._in_alternate_screen: + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + + # Create a new console buffer and activate that one. + handle = HANDLE( + self._winapi( + windll.kernel32.CreateConsoleScreenBuffer, + GENERIC_READ | GENERIC_WRITE, + DWORD(0), + None, + DWORD(1), + None, + ) + ) + + self._winapi(windll.kernel32.SetConsoleActiveScreenBuffer, handle) + self.hconsole = handle + self._in_alternate_screen = True + + def quit_alternate_screen(self) -> None: + """ + Make stdout again the active buffer. + """ + if self._in_alternate_screen: + stdout = HANDLE( + self._winapi(windll.kernel32.GetStdHandle, STD_OUTPUT_HANDLE) + ) + self._winapi(windll.kernel32.SetConsoleActiveScreenBuffer, stdout) + self._winapi(windll.kernel32.CloseHandle, self.hconsole) + self.hconsole = stdout + self._in_alternate_screen = False + + def enable_mouse_support(self) -> None: + ENABLE_MOUSE_INPUT = 0x10 + + # This `ENABLE_QUICK_EDIT_MODE` flag needs to be cleared for mouse + # support to work, but it's possible that it was already cleared + # before. + ENABLE_QUICK_EDIT_MODE = 0x0040 + + handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + + original_mode = DWORD() + self._winapi(windll.kernel32.GetConsoleMode, handle, pointer(original_mode)) + self._winapi( + windll.kernel32.SetConsoleMode, + handle, + (original_mode.value | ENABLE_MOUSE_INPUT) & ~ENABLE_QUICK_EDIT_MODE, + ) + + def disable_mouse_support(self) -> None: + ENABLE_MOUSE_INPUT = 0x10 + handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)) + + original_mode = DWORD() + self._winapi(windll.kernel32.GetConsoleMode, handle, pointer(original_mode)) + self._winapi( + windll.kernel32.SetConsoleMode, + handle, + original_mode.value & ~ENABLE_MOUSE_INPUT, + ) + + def hide_cursor(self) -> None: + pass + + def show_cursor(self) -> None: + pass + + def set_cursor_shape(self, cursor_shape: CursorShape) -> None: + pass + + def reset_cursor_shape(self) -> None: + pass + + @classmethod + def win32_refresh_window(cls) -> None: + """ + Call win32 API to refresh the whole Window. + + This is sometimes necessary when the application paints background + for completion menus. When the menu disappears, it leaves traces due + to a bug in the Windows Console. Sending a repaint request solves it. + """ + # Get console handle + handle = HANDLE(windll.kernel32.GetConsoleWindow()) + + RDW_INVALIDATE = 0x0001 + windll.user32.RedrawWindow(handle, None, None, c_uint(RDW_INVALIDATE)) + + def get_default_color_depth(self) -> ColorDepth: + """ + Return the default color depth for a windows terminal. + + Contrary to the Vt100 implementation, this doesn't depend on a $TERM + variable. + """ + if self.default_color_depth is not None: + return self.default_color_depth + + return ColorDepth.DEPTH_4_BIT + + +class FOREGROUND_COLOR: + BLACK = 0x0000 + BLUE = 0x0001 + GREEN = 0x0002 + CYAN = 0x0003 + RED = 0x0004 + MAGENTA = 0x0005 + YELLOW = 0x0006 + GRAY = 0x0007 + INTENSITY = 0x0008 # Foreground color is intensified. + + +class BACKGROUND_COLOR: + BLACK = 0x0000 + BLUE = 0x0010 + GREEN = 0x0020 + CYAN = 0x0030 + RED = 0x0040 + MAGENTA = 0x0050 + YELLOW = 0x0060 + GRAY = 0x0070 + INTENSITY = 0x0080 # Background color is intensified. + + +def _create_ansi_color_dict( + color_cls: type[FOREGROUND_COLOR] | type[BACKGROUND_COLOR], +) -> dict[str, int]: + "Create a table that maps the 16 named ansi colors to their Windows code." + return { + "ansidefault": color_cls.BLACK, + "ansiblack": color_cls.BLACK, + "ansigray": color_cls.GRAY, + "ansibrightblack": color_cls.BLACK | color_cls.INTENSITY, + "ansiwhite": color_cls.GRAY | color_cls.INTENSITY, + # Low intensity. + "ansired": color_cls.RED, + "ansigreen": color_cls.GREEN, + "ansiyellow": color_cls.YELLOW, + "ansiblue": color_cls.BLUE, + "ansimagenta": color_cls.MAGENTA, + "ansicyan": color_cls.CYAN, + # High intensity. + "ansibrightred": color_cls.RED | color_cls.INTENSITY, + "ansibrightgreen": color_cls.GREEN | color_cls.INTENSITY, + "ansibrightyellow": color_cls.YELLOW | color_cls.INTENSITY, + "ansibrightblue": color_cls.BLUE | color_cls.INTENSITY, + "ansibrightmagenta": color_cls.MAGENTA | color_cls.INTENSITY, + "ansibrightcyan": color_cls.CYAN | color_cls.INTENSITY, + } + + +FG_ANSI_COLORS = _create_ansi_color_dict(FOREGROUND_COLOR) +BG_ANSI_COLORS = _create_ansi_color_dict(BACKGROUND_COLOR) + +assert set(FG_ANSI_COLORS) == set(ANSI_COLOR_NAMES) +assert set(BG_ANSI_COLORS) == set(ANSI_COLOR_NAMES) + + +class ColorLookupTable: + """ + Inspired by pygments/formatters/terminal256.py + """ + + def __init__(self) -> None: + self._win32_colors = self._build_color_table() + + # Cache (map color string to foreground and background code). + self.best_match: dict[str, tuple[int, int]] = {} + + @staticmethod + def _build_color_table() -> list[tuple[int, int, int, int, int]]: + """ + Build an RGB-to-256 color conversion table + """ + FG = FOREGROUND_COLOR + BG = BACKGROUND_COLOR + + return [ + (0x00, 0x00, 0x00, FG.BLACK, BG.BLACK), + (0x00, 0x00, 0xAA, FG.BLUE, BG.BLUE), + (0x00, 0xAA, 0x00, FG.GREEN, BG.GREEN), + (0x00, 0xAA, 0xAA, FG.CYAN, BG.CYAN), + (0xAA, 0x00, 0x00, FG.RED, BG.RED), + (0xAA, 0x00, 0xAA, FG.MAGENTA, BG.MAGENTA), + (0xAA, 0xAA, 0x00, FG.YELLOW, BG.YELLOW), + (0x88, 0x88, 0x88, FG.GRAY, BG.GRAY), + (0x44, 0x44, 0xFF, FG.BLUE | FG.INTENSITY, BG.BLUE | BG.INTENSITY), + (0x44, 0xFF, 0x44, FG.GREEN | FG.INTENSITY, BG.GREEN | BG.INTENSITY), + (0x44, 0xFF, 0xFF, FG.CYAN | FG.INTENSITY, BG.CYAN | BG.INTENSITY), + (0xFF, 0x44, 0x44, FG.RED | FG.INTENSITY, BG.RED | BG.INTENSITY), + (0xFF, 0x44, 0xFF, FG.MAGENTA | FG.INTENSITY, BG.MAGENTA | BG.INTENSITY), + (0xFF, 0xFF, 0x44, FG.YELLOW | FG.INTENSITY, BG.YELLOW | BG.INTENSITY), + (0x44, 0x44, 0x44, FG.BLACK | FG.INTENSITY, BG.BLACK | BG.INTENSITY), + (0xFF, 0xFF, 0xFF, FG.GRAY | FG.INTENSITY, BG.GRAY | BG.INTENSITY), + ] + + def _closest_color(self, r: int, g: int, b: int) -> tuple[int, int]: + distance = 257 * 257 * 3 # "infinity" (>distance from #000000 to #ffffff) + fg_match = 0 + bg_match = 0 + + for r_, g_, b_, fg_, bg_ in self._win32_colors: + rd = r - r_ + gd = g - g_ + bd = b - b_ + + d = rd * rd + gd * gd + bd * bd + + if d < distance: + fg_match = fg_ + bg_match = bg_ + distance = d + return fg_match, bg_match + + def _color_indexes(self, color: str) -> tuple[int, int]: + indexes = self.best_match.get(color, None) + if indexes is None: + try: + rgb = int(str(color), 16) + except ValueError: + rgb = 0 + + r = (rgb >> 16) & 0xFF + g = (rgb >> 8) & 0xFF + b = rgb & 0xFF + indexes = self._closest_color(r, g, b) + self.best_match[color] = indexes + return indexes + + def lookup_fg_color(self, fg_color: str) -> int: + """ + Return the color for use in the + `windll.kernel32.SetConsoleTextAttribute` API call. + + :param fg_color: Foreground as text. E.g. 'ffffff' or 'red' + """ + # Foreground. + if fg_color in FG_ANSI_COLORS: + return FG_ANSI_COLORS[fg_color] + else: + return self._color_indexes(fg_color)[0] + + def lookup_bg_color(self, bg_color: str) -> int: + """ + Return the color for use in the + `windll.kernel32.SetConsoleTextAttribute` API call. + + :param bg_color: Background as text. E.g. 'ffffff' or 'red' + """ + # Background. + if bg_color in BG_ANSI_COLORS: + return BG_ANSI_COLORS[bg_color] + else: + return self._color_indexes(bg_color)[1] diff --git a/lib/prompt_toolkit/output/windows10.py b/lib/prompt_toolkit/output/windows10.py new file mode 100644 index 0000000..2b7e596 --- /dev/null +++ b/lib/prompt_toolkit/output/windows10.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import sys + +assert sys.platform == "win32" + +from ctypes import byref, windll +from ctypes.wintypes import DWORD, HANDLE +from typing import Any, TextIO + +from prompt_toolkit.data_structures import Size +from prompt_toolkit.win32_types import STD_OUTPUT_HANDLE + +from .base import Output +from .color_depth import ColorDepth +from .vt100 import Vt100_Output +from .win32 import Win32Output + +__all__ = [ + "Windows10_Output", +] + +# See: https://msdn.microsoft.com/pl-pl/library/windows/desktop/ms686033(v=vs.85).aspx +ENABLE_PROCESSED_INPUT = 0x0001 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 + + +class Windows10_Output: + """ + Windows 10 output abstraction. This enables and uses vt100 escape sequences. + """ + + def __init__( + self, stdout: TextIO, default_color_depth: ColorDepth | None = None + ) -> None: + self.default_color_depth = default_color_depth + self.win32_output = Win32Output(stdout, default_color_depth=default_color_depth) + self.vt100_output = Vt100_Output( + stdout, lambda: Size(0, 0), default_color_depth=default_color_depth + ) + self._hconsole = HANDLE(windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)) + + def flush(self) -> None: + """ + Write to output stream and flush. + """ + original_mode = DWORD(0) + + # Remember the previous console mode. + windll.kernel32.GetConsoleMode(self._hconsole, byref(original_mode)) + + # Enable processing of vt100 sequences. + windll.kernel32.SetConsoleMode( + self._hconsole, + DWORD(ENABLE_PROCESSED_INPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING), + ) + + try: + self.vt100_output.flush() + finally: + # Restore console mode. + windll.kernel32.SetConsoleMode(self._hconsole, original_mode) + + @property + def responds_to_cpr(self) -> bool: + return False # We don't need this on Windows. + + def __getattr__(self, name: str) -> Any: + # NOTE: Now that we use "virtual terminal input" on + # Windows, both input and output are done through + # ANSI escape sequences on Windows. This means, we + # should enable bracketed paste like on Linux, and + # enable mouse support by calling the vt100_output. + if name in ( + "get_size", + "get_rows_below_cursor_position", + "scroll_buffer_to_prompt", + "get_win32_screen_buffer_info", + # "enable_mouse_support", + # "disable_mouse_support", + # "enable_bracketed_paste", + # "disable_bracketed_paste", + ): + return getattr(self.win32_output, name) + else: + return getattr(self.vt100_output, name) + + def get_default_color_depth(self) -> ColorDepth: + """ + Return the default color depth for a windows terminal. + + Contrary to the Vt100 implementation, this doesn't depend on a $TERM + variable. + """ + if self.default_color_depth is not None: + return self.default_color_depth + + # Previously, we used `DEPTH_4_BIT`, even on Windows 10. This was + # because true color support was added after "Console Virtual Terminal + # Sequences" support was added, and there was no good way to detect + # what support was given. + # 24bit color support was added in 2016, so let's assume it's safe to + # take that as a default: + # https://devblogs.microsoft.com/commandline/24-bit-color-in-the-windows-console/ + return ColorDepth.TRUE_COLOR + + +Output.register(Windows10_Output) + + +def is_win_vt100_enabled() -> bool: + """ + Returns True when we're running Windows and VT100 escape sequences are + supported. + """ + if sys.platform != "win32": + return False + + hconsole = HANDLE(windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)) + + # Get original console mode. + original_mode = DWORD(0) + windll.kernel32.GetConsoleMode(hconsole, byref(original_mode)) + + try: + # Try to enable VT100 sequences. + result: int = windll.kernel32.SetConsoleMode( + hconsole, DWORD(ENABLE_PROCESSED_INPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING) + ) + + return result == 1 + finally: + windll.kernel32.SetConsoleMode(hconsole, original_mode) diff --git a/lib/prompt_toolkit/patch_stdout.py b/lib/prompt_toolkit/patch_stdout.py new file mode 100644 index 0000000..e1f2a7a --- /dev/null +++ b/lib/prompt_toolkit/patch_stdout.py @@ -0,0 +1,297 @@ +""" +patch_stdout +============ + +This implements a context manager that ensures that print statements within +it won't destroy the user interface. The context manager will replace +`sys.stdout` by something that draws the output above the current prompt, +rather than overwriting the UI. + +Usage:: + + with patch_stdout(application): + ... + application.run() + ... + +Multiple applications can run in the body of the context manager, one after the +other. +""" + +from __future__ import annotations + +import asyncio +import queue +import sys +import threading +import time +from contextlib import contextmanager +from typing import Generator, TextIO, cast + +from .application import get_app_session, run_in_terminal +from .output import Output + +__all__ = [ + "patch_stdout", + "StdoutProxy", +] + + +@contextmanager +def patch_stdout(raw: bool = False) -> Generator[None, None, None]: + """ + Replace `sys.stdout` by an :class:`_StdoutProxy` instance. + + Writing to this proxy will make sure that the text appears above the + prompt, and that it doesn't destroy the output from the renderer. If no + application is curring, the behavior should be identical to writing to + `sys.stdout` directly. + + Warning: If a new event loop is installed using `asyncio.set_event_loop()`, + then make sure that the context manager is applied after the event loop + is changed. Printing to stdout will be scheduled in the event loop + that's active when the context manager is created. + + :param raw: (`bool`) When True, vt100 terminal escape sequences are not + removed/escaped. + """ + with StdoutProxy(raw=raw) as proxy: + original_stdout = sys.stdout + original_stderr = sys.stderr + + # Enter. + sys.stdout = cast(TextIO, proxy) + sys.stderr = cast(TextIO, proxy) + + try: + yield + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr + + +class _Done: + "Sentinel value for stopping the stdout proxy." + + +class StdoutProxy: + """ + File-like object, which prints everything written to it, output above the + current application/prompt. This class is compatible with other file + objects and can be used as a drop-in replacement for `sys.stdout` or can + for instance be passed to `logging.StreamHandler`. + + The current application, above which we print, is determined by looking + what application currently runs in the `AppSession` that is active during + the creation of this instance. + + This class can be used as a context manager. + + In order to avoid having to repaint the prompt continuously for every + little write, a short delay of `sleep_between_writes` seconds will be added + between writes in order to bundle many smaller writes in a short timespan. + """ + + def __init__( + self, + sleep_between_writes: float = 0.2, + raw: bool = False, + ) -> None: + self.sleep_between_writes = sleep_between_writes + self.raw = raw + + self._lock = threading.RLock() + self._buffer: list[str] = [] + + # Keep track of the curret app session. + self.app_session = get_app_session() + + # See what output is active *right now*. We should do it at this point, + # before this `StdoutProxy` instance is possibly assigned to `sys.stdout`. + # Otherwise, if `patch_stdout` is used, and no `Output` instance has + # been created, then the default output creation code will see this + # proxy object as `sys.stdout`, and get in a recursive loop trying to + # access `StdoutProxy.isatty()` which will again retrieve the output. + self._output: Output = self.app_session.output + + # Flush thread + self._flush_queue: queue.Queue[str | _Done] = queue.Queue() + self._flush_thread = self._start_write_thread() + self.closed = False + + def __enter__(self) -> StdoutProxy: + return self + + def __exit__(self, *args: object) -> None: + self.close() + + def close(self) -> None: + """ + Stop `StdoutProxy` proxy. + + This will terminate the write thread, make sure everything is flushed + and wait for the write thread to finish. + """ + if not self.closed: + self._flush_queue.put(_Done()) + self._flush_thread.join() + self.closed = True + + def _start_write_thread(self) -> threading.Thread: + thread = threading.Thread( + target=self._write_thread, + name="patch-stdout-flush-thread", + daemon=True, + ) + thread.start() + return thread + + def _write_thread(self) -> None: + done = False + + while not done: + item = self._flush_queue.get() + + if isinstance(item, _Done): + break + + # Don't bother calling when we got an empty string. + if not item: + continue + + text = [] + text.append(item) + + # Read the rest of the queue if more data was queued up. + while True: + try: + item = self._flush_queue.get_nowait() + except queue.Empty: + break + else: + if isinstance(item, _Done): + done = True + else: + text.append(item) + + app_loop = self._get_app_loop() + self._write_and_flush(app_loop, "".join(text)) + + # If an application was running that requires repainting, then wait + # for a very short time, in order to bundle actual writes and avoid + # having to repaint to often. + if app_loop is not None: + time.sleep(self.sleep_between_writes) + + def _get_app_loop(self) -> asyncio.AbstractEventLoop | None: + """ + Return the event loop for the application currently running in our + `AppSession`. + """ + app = self.app_session.app + + if app is None: + return None + + return app.loop + + def _write_and_flush( + self, loop: asyncio.AbstractEventLoop | None, text: str + ) -> None: + """ + Write the given text to stdout and flush. + If an application is running, use `run_in_terminal`. + """ + + def write_and_flush() -> None: + # Ensure that autowrap is enabled before calling `write`. + # XXX: On Windows, the `Windows10_Output` enables/disables VT + # terminal processing for every flush. It turns out that this + # causes autowrap to be reset (disabled) after each flush. So, + # we have to enable it again before writing text. + self._output.enable_autowrap() + + if self.raw: + self._output.write_raw(text) + else: + self._output.write(text) + + self._output.flush() + + def write_and_flush_in_loop() -> None: + # If an application is running, use `run_in_terminal`, otherwise + # call it directly. + run_in_terminal(write_and_flush, in_executor=False) + + if loop is None: + # No loop, write immediately. + write_and_flush() + else: + # Make sure `write_and_flush` is executed *in* the event loop, not + # in another thread. + loop.call_soon_threadsafe(write_and_flush_in_loop) + + def _write(self, data: str) -> None: + """ + Note: print()-statements cause to multiple write calls. + (write('line') and write('\n')). Of course we don't want to call + `run_in_terminal` for every individual call, because that's too + expensive, and as long as the newline hasn't been written, the + text itself is again overwritten by the rendering of the input + command line. Therefor, we have a little buffer which holds the + text until a newline is written to stdout. + """ + if "\n" in data: + # When there is a newline in the data, write everything before the + # newline, including the newline itself. + before, after = data.rsplit("\n", 1) + to_write = self._buffer + [before, "\n"] + self._buffer = [after] + + text = "".join(to_write) + self._flush_queue.put(text) + else: + # Otherwise, cache in buffer. + self._buffer.append(data) + + def _flush(self) -> None: + text = "".join(self._buffer) + self._buffer = [] + self._flush_queue.put(text) + + def write(self, data: str) -> int: + with self._lock: + self._write(data) + + return len(data) # Pretend everything was written. + + def flush(self) -> None: + """ + Flush buffered output. + """ + with self._lock: + self._flush() + + @property + def original_stdout(self) -> TextIO | None: + return self._output.stdout or sys.__stdout__ + + # Attributes for compatibility with sys.__stdout__: + + def fileno(self) -> int: + return self._output.fileno() + + def isatty(self) -> bool: + stdout = self._output.stdout + if stdout is None: + return False + + return stdout.isatty() + + @property + def encoding(self) -> str: + return self._output.encoding() + + @property + def errors(self) -> str: + return "strict" diff --git a/lib/prompt_toolkit/py.typed b/lib/prompt_toolkit/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lib/prompt_toolkit/renderer.py b/lib/prompt_toolkit/renderer.py new file mode 100644 index 0000000..8d5e03c --- /dev/null +++ b/lib/prompt_toolkit/renderer.py @@ -0,0 +1,820 @@ +""" +Renders the command line on the console. +(Redraws parts of the input line that were changed.) +""" + +from __future__ import annotations + +from asyncio import FIRST_COMPLETED, Future, ensure_future, sleep, wait +from collections import deque +from enum import Enum +from typing import TYPE_CHECKING, Any, Callable, Dict, Hashable + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.cursor_shapes import CursorShape +from prompt_toolkit.data_structures import Point, Size +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.formatted_text import AnyFormattedText, to_formatted_text +from prompt_toolkit.layout.mouse_handlers import MouseHandlers +from prompt_toolkit.layout.screen import Char, Screen, WritePosition +from prompt_toolkit.output import ColorDepth, Output +from prompt_toolkit.styles import ( + Attrs, + BaseStyle, + DummyStyleTransformation, + StyleTransformation, +) + +if TYPE_CHECKING: + from prompt_toolkit.application import Application + from prompt_toolkit.layout.layout import Layout + + +__all__ = [ + "Renderer", + "print_formatted_text", +] + + +def _output_screen_diff( + app: Application[Any], + output: Output, + screen: Screen, + current_pos: Point, + color_depth: ColorDepth, + previous_screen: Screen | None, + last_style: str | None, + is_done: bool, # XXX: drop is_done + full_screen: bool, + attrs_for_style_string: _StyleStringToAttrsCache, + style_string_has_style: _StyleStringHasStyleCache, + size: Size, + previous_width: int, +) -> tuple[Point, str | None]: + """ + Render the diff between this screen and the previous screen. + + This takes two `Screen` instances. The one that represents the output like + it was during the last rendering and one that represents the current + output raster. Looking at these two `Screen` instances, this function will + render the difference by calling the appropriate methods of the `Output` + object that only paint the changes to the terminal. + + This is some performance-critical code which is heavily optimized. + Don't change things without profiling first. + + :param current_pos: Current cursor position. + :param last_style: The style string, used for drawing the last drawn + character. (Color/attributes.) + :param attrs_for_style_string: :class:`._StyleStringToAttrsCache` instance. + :param width: The width of the terminal. + :param previous_width: The width of the terminal during the last rendering. + """ + width, height = size.columns, size.rows + + #: Variable for capturing the output. + write = output.write + write_raw = output.write_raw + + # Create locals for the most used output methods. + # (Save expensive attribute lookups.) + _output_set_attributes = output.set_attributes + _output_reset_attributes = output.reset_attributes + _output_cursor_forward = output.cursor_forward + _output_cursor_up = output.cursor_up + _output_cursor_backward = output.cursor_backward + + # Hide cursor before rendering. (Avoid flickering.) + output.hide_cursor() + + def reset_attributes() -> None: + "Wrapper around Output.reset_attributes." + nonlocal last_style + _output_reset_attributes() + last_style = None # Forget last char after resetting attributes. + + def move_cursor(new: Point) -> Point: + "Move cursor to this `new` point. Returns the given Point." + current_x, current_y = current_pos.x, current_pos.y + + if new.y > current_y: + # Use newlines instead of CURSOR_DOWN, because this might add new lines. + # CURSOR_DOWN will never create new lines at the bottom. + # Also reset attributes, otherwise the newline could draw a + # background color. + reset_attributes() + write("\r\n" * (new.y - current_y)) + current_x = 0 + _output_cursor_forward(new.x) + return new + elif new.y < current_y: + _output_cursor_up(current_y - new.y) + + if current_x >= width - 1: + write("\r") + _output_cursor_forward(new.x) + elif new.x < current_x or current_x >= width - 1: + _output_cursor_backward(current_x - new.x) + elif new.x > current_x: + _output_cursor_forward(new.x - current_x) + + return new + + def output_char(char: Char) -> None: + """ + Write the output of this character. + """ + nonlocal last_style + + # If the last printed character has the same style, don't output the + # style again. + if last_style == char.style: + write(char.char) + else: + # Look up `Attr` for this style string. Only set attributes if different. + # (Two style strings can still have the same formatting.) + # Note that an empty style string can have formatting that needs to + # be applied, because of style transformations. + new_attrs = attrs_for_style_string[char.style] + if not last_style or new_attrs != attrs_for_style_string[last_style]: + _output_set_attributes(new_attrs, color_depth) + + write(char.char) + last_style = char.style + + def get_max_column_index(row: dict[int, Char]) -> int: + """ + Return max used column index, ignoring whitespace (without style) at + the end of the line. This is important for people that copy/paste + terminal output. + + There are two reasons we are sometimes seeing whitespace at the end: + - `BufferControl` adds a trailing space to each line, because it's a + possible cursor position, so that the line wrapping won't change if + the cursor position moves around. + - The `Window` adds a style class to the current line for highlighting + (cursor-line). + """ + numbers = ( + index + for index, cell in row.items() + if cell.char != " " or style_string_has_style[cell.style] + ) + return max(numbers, default=0) + + # Render for the first time: reset styling. + if not previous_screen: + reset_attributes() + + # Disable autowrap. (When entering a the alternate screen, or anytime when + # we have a prompt. - In the case of a REPL, like IPython, people can have + # background threads, and it's hard for debugging if their output is not + # wrapped.) + if not previous_screen or not full_screen: + output.disable_autowrap() + + # When the previous screen has a different size, redraw everything anyway. + # Also when we are done. (We might take up less rows, so clearing is important.) + if ( + is_done or not previous_screen or previous_width != width + ): # XXX: also consider height?? + current_pos = move_cursor(Point(x=0, y=0)) + reset_attributes() + output.erase_down() + + previous_screen = Screen() + + # Get height of the screen. + # (height changes as we loop over data_buffer, so remember the current value.) + # (Also make sure to clip the height to the size of the output.) + current_height = min(screen.height, height) + + # Loop over the rows. + row_count = min(max(screen.height, previous_screen.height), height) + + for y in range(row_count): + new_row = screen.data_buffer[y] + previous_row = previous_screen.data_buffer[y] + zero_width_escapes_row = screen.zero_width_escapes[y] + + new_max_line_len = min(width - 1, get_max_column_index(new_row)) + previous_max_line_len = min(width - 1, get_max_column_index(previous_row)) + + # Loop over the columns. + c = 0 # Column counter. + while c <= new_max_line_len: + new_char = new_row[c] + old_char = previous_row[c] + char_width = new_char.width or 1 + + # When the old and new character at this position are different, + # draw the output. (Because of the performance, we don't call + # `Char.__ne__`, but inline the same expression.) + if new_char.char != old_char.char or new_char.style != old_char.style: + current_pos = move_cursor(Point(x=c, y=y)) + + # Send injected escape sequences to output. + if c in zero_width_escapes_row: + write_raw(zero_width_escapes_row[c]) + + output_char(new_char) + current_pos = Point(x=current_pos.x + char_width, y=current_pos.y) + + c += char_width + + # If the new line is shorter, trim it. + if previous_screen and new_max_line_len < previous_max_line_len: + current_pos = move_cursor(Point(x=new_max_line_len + 1, y=y)) + reset_attributes() + output.erase_end_of_line() + + # Correctly reserve vertical space as required by the layout. + # When this is a new screen (drawn for the first time), or for some reason + # higher than the previous one. Move the cursor once to the bottom of the + # output. That way, we're sure that the terminal scrolls up, even when the + # lower lines of the canvas just contain whitespace. + + # The most obvious reason that we actually want this behavior is the avoid + # the artifact of the input scrolling when the completion menu is shown. + # (If the scrolling is actually wanted, the layout can still be build in a + # way to behave that way by setting a dynamic height.) + if current_height > previous_screen.height: + current_pos = move_cursor(Point(x=0, y=current_height - 1)) + + # Move cursor: + if is_done: + current_pos = move_cursor(Point(x=0, y=current_height)) + output.erase_down() + else: + current_pos = move_cursor(screen.get_cursor_position(app.layout.current_window)) + + if is_done or not full_screen: + output.enable_autowrap() + + # Always reset the color attributes. This is important because a background + # thread could print data to stdout and we want that to be displayed in the + # default colors. (Also, if a background color has been set, many terminals + # give weird artifacts on resize events.) + reset_attributes() + + if screen.show_cursor: + output.show_cursor() + + return current_pos, last_style + + +class HeightIsUnknownError(Exception): + "Information unavailable. Did not yet receive the CPR response." + + +class _StyleStringToAttrsCache(Dict[str, Attrs]): + """ + A cache structure that maps style strings to :class:`.Attr`. + (This is an important speed up.) + """ + + def __init__( + self, + get_attrs_for_style_str: Callable[[str], Attrs], + style_transformation: StyleTransformation, + ) -> None: + self.get_attrs_for_style_str = get_attrs_for_style_str + self.style_transformation = style_transformation + + def __missing__(self, style_str: str) -> Attrs: + attrs = self.get_attrs_for_style_str(style_str) + attrs = self.style_transformation.transform_attrs(attrs) + + self[style_str] = attrs + return attrs + + +class _StyleStringHasStyleCache(Dict[str, bool]): + """ + Cache for remember which style strings don't render the default output + style (default fg/bg, no underline and no reverse and no blink). That way + we know that we should render these cells, even when they're empty (when + they contain a space). + + Note: we don't consider bold/italic/hidden because they don't change the + output if there's no text in the cell. + """ + + def __init__(self, style_string_to_attrs: dict[str, Attrs]) -> None: + self.style_string_to_attrs = style_string_to_attrs + + def __missing__(self, style_str: str) -> bool: + attrs = self.style_string_to_attrs[style_str] + is_default = bool( + attrs.color + or attrs.bgcolor + or attrs.underline + or attrs.strike + or attrs.blink + or attrs.reverse + ) + + self[style_str] = is_default + return is_default + + +class CPR_Support(Enum): + "Enum: whether or not CPR is supported." + + SUPPORTED = "SUPPORTED" + NOT_SUPPORTED = "NOT_SUPPORTED" + UNKNOWN = "UNKNOWN" + + +class Renderer: + """ + Typical usage: + + :: + + output = Vt100_Output.from_pty(sys.stdout) + r = Renderer(style, output) + r.render(app, layout=...) + """ + + CPR_TIMEOUT = 2 # Time to wait until we consider CPR to be not supported. + + def __init__( + self, + style: BaseStyle, + output: Output, + full_screen: bool = False, + mouse_support: FilterOrBool = False, + cpr_not_supported_callback: Callable[[], None] | None = None, + ) -> None: + self.style = style + self.output = output + self.full_screen = full_screen + self.mouse_support = to_filter(mouse_support) + self.cpr_not_supported_callback = cpr_not_supported_callback + + # TODO: Move following state flags into `Vt100_Output`, similar to + # `_cursor_shape_changed` and `_cursor_visible`. But then also + # adjust the `Win32Output` to not call win32 APIs if nothing has + # to be changed. + + self._in_alternate_screen = False + self._mouse_support_enabled = False + self._bracketed_paste_enabled = False + self._cursor_key_mode_reset = False + + # Future set when we are waiting for a CPR flag. + self._waiting_for_cpr_futures: deque[Future[None]] = deque() + self.cpr_support = CPR_Support.UNKNOWN + + if not output.responds_to_cpr: + self.cpr_support = CPR_Support.NOT_SUPPORTED + + # Cache for the style. + self._attrs_for_style: _StyleStringToAttrsCache | None = None + self._style_string_has_style: _StyleStringHasStyleCache | None = None + self._last_style_hash: Hashable | None = None + self._last_transformation_hash: Hashable | None = None + self._last_color_depth: ColorDepth | None = None + + self.reset(_scroll=True) + + def reset(self, _scroll: bool = False, leave_alternate_screen: bool = True) -> None: + # Reset position + self._cursor_pos = Point(x=0, y=0) + + # Remember the last screen instance between renderers. This way, + # we can create a `diff` between two screens and only output the + # difference. It's also to remember the last height. (To show for + # instance a toolbar at the bottom position.) + self._last_screen: Screen | None = None + self._last_size: Size | None = None + self._last_style: str | None = None + self._last_cursor_shape: CursorShape | None = None + + # Default MouseHandlers. (Just empty.) + self.mouse_handlers = MouseHandlers() + + #: Space from the top of the layout, until the bottom of the terminal. + #: We don't know this until a `report_absolute_cursor_row` call. + self._min_available_height = 0 + + # In case of Windows, also make sure to scroll to the current cursor + # position. (Only when rendering the first time.) + # It does nothing for vt100 terminals. + if _scroll: + self.output.scroll_buffer_to_prompt() + + # Quit alternate screen. + if self._in_alternate_screen and leave_alternate_screen: + self.output.quit_alternate_screen() + self._in_alternate_screen = False + + # Disable mouse support. + if self._mouse_support_enabled: + self.output.disable_mouse_support() + self._mouse_support_enabled = False + + # Disable bracketed paste. + if self._bracketed_paste_enabled: + self.output.disable_bracketed_paste() + self._bracketed_paste_enabled = False + + self.output.reset_cursor_shape() + self.output.show_cursor() + + # NOTE: No need to set/reset cursor key mode here. + + # Flush output. `disable_mouse_support` needs to write to stdout. + self.output.flush() + + @property + def last_rendered_screen(self) -> Screen | None: + """ + The `Screen` class that was generated during the last rendering. + This can be `None`. + """ + return self._last_screen + + @property + def height_is_known(self) -> bool: + """ + True when the height from the cursor until the bottom of the terminal + is known. (It's often nicer to draw bottom toolbars only if the height + is known, in order to avoid flickering when the CPR response arrives.) + """ + if self.full_screen or self._min_available_height > 0: + return True + try: + self._min_available_height = self.output.get_rows_below_cursor_position() + return True + except NotImplementedError: + return False + + @property + def rows_above_layout(self) -> int: + """ + Return the number of rows visible in the terminal above the layout. + """ + if self._in_alternate_screen: + return 0 + elif self._min_available_height > 0: + total_rows = self.output.get_size().rows + last_screen_height = self._last_screen.height if self._last_screen else 0 + return total_rows - max(self._min_available_height, last_screen_height) + else: + raise HeightIsUnknownError("Rows above layout is unknown.") + + def request_absolute_cursor_position(self) -> None: + """ + Get current cursor position. + + We do this to calculate the minimum available height that we can + consume for rendering the prompt. This is the available space below te + cursor. + + For vt100: Do CPR request. (answer will arrive later.) + For win32: Do API call. (Answer comes immediately.) + """ + # Only do this request when the cursor is at the top row. (after a + # clear or reset). We will rely on that in `report_absolute_cursor_row`. + assert self._cursor_pos.y == 0 + + # In full-screen mode, always use the total height as min-available-height. + if self.full_screen: + self._min_available_height = self.output.get_size().rows + return + + # For Win32, we have an API call to get the number of rows below the + # cursor. + try: + self._min_available_height = self.output.get_rows_below_cursor_position() + return + except NotImplementedError: + pass + + # Use CPR. + if self.cpr_support == CPR_Support.NOT_SUPPORTED: + return + + def do_cpr() -> None: + # Asks for a cursor position report (CPR). + self._waiting_for_cpr_futures.append(Future()) + self.output.ask_for_cpr() + + if self.cpr_support == CPR_Support.SUPPORTED: + do_cpr() + return + + # If we don't know whether CPR is supported, only do a request if + # none is pending, and test it, using a timer. + if self.waiting_for_cpr: + return + + do_cpr() + + async def timer() -> None: + await sleep(self.CPR_TIMEOUT) + + # Not set in the meantime -> not supported. + if self.cpr_support == CPR_Support.UNKNOWN: + self.cpr_support = CPR_Support.NOT_SUPPORTED + + if self.cpr_not_supported_callback: + # Make sure to call this callback in the main thread. + self.cpr_not_supported_callback() + + get_app().create_background_task(timer()) + + def report_absolute_cursor_row(self, row: int) -> None: + """ + To be called when we know the absolute cursor position. + (As an answer of a "Cursor Position Request" response.) + """ + self.cpr_support = CPR_Support.SUPPORTED + + # Calculate the amount of rows from the cursor position until the + # bottom of the terminal. + total_rows = self.output.get_size().rows + rows_below_cursor = total_rows - row + 1 + + # Set the minimum available height. + self._min_available_height = rows_below_cursor + + # Pop and set waiting for CPR future. + try: + f = self._waiting_for_cpr_futures.popleft() + except IndexError: + pass # Received CPR response without having a CPR. + else: + f.set_result(None) + + @property + def waiting_for_cpr(self) -> bool: + """ + Waiting for CPR flag. True when we send the request, but didn't got a + response. + """ + return bool(self._waiting_for_cpr_futures) + + async def wait_for_cpr_responses(self, timeout: int = 1) -> None: + """ + Wait for a CPR response. + """ + cpr_futures = list(self._waiting_for_cpr_futures) # Make copy. + + # When there are no CPRs in the queue. Don't do anything. + if not cpr_futures or self.cpr_support == CPR_Support.NOT_SUPPORTED: + return None + + async def wait_for_responses() -> None: + for response_f in cpr_futures: + await response_f + + async def wait_for_timeout() -> None: + await sleep(timeout) + + # Got timeout, erase queue. + for response_f in cpr_futures: + response_f.cancel() + self._waiting_for_cpr_futures = deque() + + tasks = { + ensure_future(wait_for_responses()), + ensure_future(wait_for_timeout()), + } + _, pending = await wait(tasks, return_when=FIRST_COMPLETED) + for task in pending: + task.cancel() + + def render( + self, app: Application[Any], layout: Layout, is_done: bool = False + ) -> None: + """ + Render the current interface to the output. + + :param is_done: When True, put the cursor at the end of the interface. We + won't print any changes to this part. + """ + output = self.output + + # Enter alternate screen. + if self.full_screen and not self._in_alternate_screen: + self._in_alternate_screen = True + output.enter_alternate_screen() + + # Enable bracketed paste. + if not self._bracketed_paste_enabled: + self.output.enable_bracketed_paste() + self._bracketed_paste_enabled = True + + # Reset cursor key mode. + if not self._cursor_key_mode_reset: + self.output.reset_cursor_key_mode() + self._cursor_key_mode_reset = True + + # Enable/disable mouse support. + needs_mouse_support = self.mouse_support() + + if needs_mouse_support and not self._mouse_support_enabled: + output.enable_mouse_support() + self._mouse_support_enabled = True + + elif not needs_mouse_support and self._mouse_support_enabled: + output.disable_mouse_support() + self._mouse_support_enabled = False + + # Create screen and write layout to it. + size = output.get_size() + screen = Screen() + screen.show_cursor = False # Hide cursor by default, unless one of the + # containers decides to display it. + mouse_handlers = MouseHandlers() + + # Calculate height. + if self.full_screen: + height = size.rows + elif is_done: + # When we are done, we don't necessary want to fill up until the bottom. + height = layout.container.preferred_height( + size.columns, size.rows + ).preferred + else: + last_height = self._last_screen.height if self._last_screen else 0 + height = max( + self._min_available_height, + last_height, + layout.container.preferred_height(size.columns, size.rows).preferred, + ) + + height = min(height, size.rows) + + # When the size changes, don't consider the previous screen. + if self._last_size != size: + self._last_screen = None + + # When we render using another style or another color depth, do a full + # repaint. (Forget about the previous rendered screen.) + # (But note that we still use _last_screen to calculate the height.) + if ( + self.style.invalidation_hash() != self._last_style_hash + or app.style_transformation.invalidation_hash() + != self._last_transformation_hash + or app.color_depth != self._last_color_depth + ): + self._last_screen = None + self._attrs_for_style = None + self._style_string_has_style = None + + if self._attrs_for_style is None: + self._attrs_for_style = _StyleStringToAttrsCache( + self.style.get_attrs_for_style_str, app.style_transformation + ) + if self._style_string_has_style is None: + self._style_string_has_style = _StyleStringHasStyleCache( + self._attrs_for_style + ) + + self._last_style_hash = self.style.invalidation_hash() + self._last_transformation_hash = app.style_transformation.invalidation_hash() + self._last_color_depth = app.color_depth + + layout.container.write_to_screen( + screen, + mouse_handlers, + WritePosition(xpos=0, ypos=0, width=size.columns, height=height), + parent_style="", + erase_bg=False, + z_index=None, + ) + screen.draw_all_floats() + + # When grayed. Replace all styles in the new screen. + if app.exit_style: + screen.append_style_to_content(app.exit_style) + + # Process diff and write to output. + self._cursor_pos, self._last_style = _output_screen_diff( + app, + output, + screen, + self._cursor_pos, + app.color_depth, + self._last_screen, + self._last_style, + is_done, + full_screen=self.full_screen, + attrs_for_style_string=self._attrs_for_style, + style_string_has_style=self._style_string_has_style, + size=size, + previous_width=(self._last_size.columns if self._last_size else 0), + ) + self._last_screen = screen + self._last_size = size + self.mouse_handlers = mouse_handlers + + # Handle cursor shapes. + new_cursor_shape = app.cursor.get_cursor_shape(app) + if ( + self._last_cursor_shape is None + or self._last_cursor_shape != new_cursor_shape + ): + output.set_cursor_shape(new_cursor_shape) + self._last_cursor_shape = new_cursor_shape + + # Flush buffered output. + output.flush() + + # Set visible windows in layout. + app.layout.visible_windows = screen.visible_windows + + if is_done: + self.reset() + + def erase(self, leave_alternate_screen: bool = True) -> None: + """ + Hide all output and put the cursor back at the first line. This is for + instance used for running a system command (while hiding the CLI) and + later resuming the same CLI.) + + :param leave_alternate_screen: When True, and when inside an alternate + screen buffer, quit the alternate screen. + """ + output = self.output + + output.cursor_backward(self._cursor_pos.x) + output.cursor_up(self._cursor_pos.y) + output.erase_down() + output.reset_attributes() + output.enable_autowrap() + + output.flush() + + self.reset(leave_alternate_screen=leave_alternate_screen) + + def clear(self) -> None: + """ + Clear screen and go to 0,0 + """ + # Erase current output first. + self.erase() + + # Send "Erase Screen" command and go to (0, 0). + output = self.output + + output.erase_screen() + output.cursor_goto(0, 0) + output.flush() + + self.request_absolute_cursor_position() + + +def print_formatted_text( + output: Output, + formatted_text: AnyFormattedText, + style: BaseStyle, + style_transformation: StyleTransformation | None = None, + color_depth: ColorDepth | None = None, +) -> None: + """ + Print a list of (style_str, text) tuples in the given style to the output. + """ + fragments = to_formatted_text(formatted_text) + style_transformation = style_transformation or DummyStyleTransformation() + color_depth = color_depth or output.get_default_color_depth() + + # Reset first. + output.reset_attributes() + output.enable_autowrap() + last_attrs: Attrs | None = None + + # Print all (style_str, text) tuples. + attrs_for_style_string = _StyleStringToAttrsCache( + style.get_attrs_for_style_str, style_transformation + ) + + for style_str, text, *_ in fragments: + attrs = attrs_for_style_string[style_str] + + # Set style attributes if something changed. + if attrs != last_attrs: + if attrs: + output.set_attributes(attrs, color_depth) + else: + output.reset_attributes() + last_attrs = attrs + + # Print escape sequences as raw output + if "[ZeroWidthEscape]" in style_str: + output.write_raw(text) + else: + # Eliminate carriage returns + text = text.replace("\r", "") + # Insert a carriage return before every newline (important when the + # front-end is a telnet client). + text = text.replace("\n", "\r\n") + output.write(text) + + # Reset again. + output.reset_attributes() + output.flush() diff --git a/lib/prompt_toolkit/search.py b/lib/prompt_toolkit/search.py new file mode 100644 index 0000000..d1cf7ac --- /dev/null +++ b/lib/prompt_toolkit/search.py @@ -0,0 +1,226 @@ +""" +Search operations. + +For the key bindings implementation with attached filters, check +`prompt_toolkit.key_binding.bindings.search`. (Use these for new key bindings +instead of calling these function directly.) +""" + +from __future__ import annotations + +from enum import Enum +from typing import TYPE_CHECKING + +from .application.current import get_app +from .filters import FilterOrBool, is_searching, to_filter +from .key_binding.vi_state import InputMode + +if TYPE_CHECKING: + from prompt_toolkit.layout.controls import BufferControl, SearchBufferControl + from prompt_toolkit.layout.layout import Layout + +__all__ = [ + "SearchDirection", + "start_search", + "stop_search", +] + + +class SearchDirection(Enum): + FORWARD = "FORWARD" + BACKWARD = "BACKWARD" + + +class SearchState: + """ + A search 'query', associated with a search field (like a SearchToolbar). + + Every searchable `BufferControl` points to a `search_buffer_control` + (another `BufferControls`) which represents the search field. The + `SearchState` attached to that search field is used for storing the current + search query. + + It is possible to have one searchfield for multiple `BufferControls`. In + that case, they'll share the same `SearchState`. + If there are multiple `BufferControls` that display the same `Buffer`, then + they can have a different `SearchState` each (if they have a different + search control). + """ + + __slots__ = ("text", "direction", "ignore_case") + + def __init__( + self, + text: str = "", + direction: SearchDirection = SearchDirection.FORWARD, + ignore_case: FilterOrBool = False, + ) -> None: + self.text = text + self.direction = direction + self.ignore_case = to_filter(ignore_case) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.text!r}, direction={self.direction!r}, ignore_case={self.ignore_case!r})" + + def __invert__(self) -> SearchState: + """ + Create a new SearchState where backwards becomes forwards and the other + way around. + """ + if self.direction == SearchDirection.BACKWARD: + direction = SearchDirection.FORWARD + else: + direction = SearchDirection.BACKWARD + + return SearchState( + text=self.text, direction=direction, ignore_case=self.ignore_case + ) + + +def start_search( + buffer_control: BufferControl | None = None, + direction: SearchDirection = SearchDirection.FORWARD, +) -> None: + """ + Start search through the given `buffer_control` using the + `search_buffer_control`. + + :param buffer_control: Start search for this `BufferControl`. If not given, + search through the current control. + """ + from prompt_toolkit.layout.controls import BufferControl + + assert buffer_control is None or isinstance(buffer_control, BufferControl) + + layout = get_app().layout + + # When no control is given, use the current control if that's a BufferControl. + if buffer_control is None: + if not isinstance(layout.current_control, BufferControl): + return + buffer_control = layout.current_control + + # Only if this control is searchable. + search_buffer_control = buffer_control.search_buffer_control + + if search_buffer_control: + buffer_control.search_state.direction = direction + + # Make sure to focus the search BufferControl + layout.focus(search_buffer_control) + + # Remember search link. + layout.search_links[search_buffer_control] = buffer_control + + # If we're in Vi mode, make sure to go into insert mode. + get_app().vi_state.input_mode = InputMode.INSERT + + +def stop_search(buffer_control: BufferControl | None = None) -> None: + """ + Stop search through the given `buffer_control`. + """ + layout = get_app().layout + + if buffer_control is None: + buffer_control = layout.search_target_buffer_control + if buffer_control is None: + # (Should not happen, but possible when `stop_search` is called + # when we're not searching.) + return + search_buffer_control = buffer_control.search_buffer_control + else: + assert buffer_control in layout.search_links.values() + search_buffer_control = _get_reverse_search_links(layout)[buffer_control] + + # Focus the original buffer again. + layout.focus(buffer_control) + + if search_buffer_control is not None: + # Remove the search link. + del layout.search_links[search_buffer_control] + + # Reset content of search control. + search_buffer_control.buffer.reset() + + # If we're in Vi mode, go back to navigation mode. + get_app().vi_state.input_mode = InputMode.NAVIGATION + + +def do_incremental_search(direction: SearchDirection, count: int = 1) -> None: + """ + Apply search, but keep search buffer focused. + """ + assert is_searching() + + layout = get_app().layout + + # Only search if the current control is a `BufferControl`. + from prompt_toolkit.layout.controls import BufferControl + + search_control = layout.current_control + if not isinstance(search_control, BufferControl): + return + + prev_control = layout.search_target_buffer_control + if prev_control is None: + return + search_state = prev_control.search_state + + # Update search_state. + direction_changed = search_state.direction != direction + + search_state.text = search_control.buffer.text + search_state.direction = direction + + # Apply search to current buffer. + if not direction_changed: + prev_control.buffer.apply_search( + search_state, include_current_position=False, count=count + ) + + +def accept_search() -> None: + """ + Accept current search query. Focus original `BufferControl` again. + """ + layout = get_app().layout + + search_control = layout.current_control + target_buffer_control = layout.search_target_buffer_control + + from prompt_toolkit.layout.controls import BufferControl + + if not isinstance(search_control, BufferControl): + return + if target_buffer_control is None: + return + + search_state = target_buffer_control.search_state + + # Update search state. + if search_control.buffer.text: + search_state.text = search_control.buffer.text + + # Apply search. + target_buffer_control.buffer.apply_search( + search_state, include_current_position=True + ) + + # Add query to history of search line. + search_control.buffer.append_to_history() + + # Stop search and focus previous control again. + stop_search(target_buffer_control) + + +def _get_reverse_search_links( + layout: Layout, +) -> dict[BufferControl, SearchBufferControl]: + """ + Return mapping from BufferControl to SearchBufferControl. + """ + return { + buffer_control: search_buffer_control + for search_buffer_control, buffer_control in layout.search_links.items() + } diff --git a/lib/prompt_toolkit/selection.py b/lib/prompt_toolkit/selection.py new file mode 100644 index 0000000..ff88535 --- /dev/null +++ b/lib/prompt_toolkit/selection.py @@ -0,0 +1,58 @@ +""" +Data structures for the selection. +""" + +from __future__ import annotations + +from enum import Enum + +__all__ = [ + "SelectionType", + "PasteMode", + "SelectionState", +] + + +class SelectionType(Enum): + """ + Type of selection. + """ + + #: Characters. (Visual in Vi.) + CHARACTERS = "CHARACTERS" + + #: Whole lines. (Visual-Line in Vi.) + LINES = "LINES" + + #: A block selection. (Visual-Block in Vi.) + BLOCK = "BLOCK" + + +class PasteMode(Enum): + EMACS = "EMACS" # Yank like emacs. + VI_AFTER = "VI_AFTER" # When pressing 'p' in Vi. + VI_BEFORE = "VI_BEFORE" # When pressing 'P' in Vi. + + +class SelectionState: + """ + State of the current selection. + + :param original_cursor_position: int + :param type: :class:`~.SelectionType` + """ + + def __init__( + self, + original_cursor_position: int = 0, + type: SelectionType = SelectionType.CHARACTERS, + ) -> None: + self.original_cursor_position = original_cursor_position + self.type = type + self.shift_mode = False + + def enter_shift_mode(self) -> None: + self.shift_mode = True + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(original_cursor_position={self.original_cursor_position!r}, type={self.type!r})" diff --git a/lib/prompt_toolkit/shortcuts/__init__.py b/lib/prompt_toolkit/shortcuts/__init__.py new file mode 100644 index 0000000..d93d3b6 --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/__init__.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from .choice_input import choice +from .dialogs import ( + button_dialog, + checkboxlist_dialog, + input_dialog, + message_dialog, + progress_dialog, + radiolist_dialog, + yes_no_dialog, +) +from .progress_bar import ProgressBar, ProgressBarCounter +from .prompt import ( + CompleteStyle, + PromptSession, + confirm, + create_confirm_session, + prompt, +) +from .utils import clear, clear_title, print_container, print_formatted_text, set_title + +__all__ = [ + # Dialogs. + "input_dialog", + "message_dialog", + "progress_dialog", + "checkboxlist_dialog", + "radiolist_dialog", + "yes_no_dialog", + "button_dialog", + # Prompts. + "PromptSession", + "prompt", + "confirm", + "create_confirm_session", + "CompleteStyle", + # Progress bars. + "ProgressBar", + "ProgressBarCounter", + # Choice selection. + "choice", + # Utils. + "clear", + "clear_title", + "print_container", + "print_formatted_text", + "set_title", +] diff --git a/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..72c77c3 Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc new file mode 100644 index 0000000..97794f6 Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc new file mode 100644 index 0000000..152f08e Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc new file mode 100644 index 0000000..8a96c9a Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..4eae68d Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/choice_input.py b/lib/prompt_toolkit/shortcuts/choice_input.py new file mode 100644 index 0000000..95ffd0b --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/choice_input.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +from typing import Generic, Sequence, TypeVar + +from prompt_toolkit.application import Application +from prompt_toolkit.filters import ( + Condition, + FilterOrBool, + is_done, + renderer_height_is_known, + to_filter, +) +from prompt_toolkit.formatted_text import AnyFormattedText +from prompt_toolkit.key_binding.key_bindings import ( + DynamicKeyBindings, + KeyBindings, + KeyBindingsBase, + merge_key_bindings, +) +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.layout import ( + AnyContainer, + ConditionalContainer, + HSplit, + Layout, + Window, +) +from prompt_toolkit.layout.controls import FormattedTextControl +from prompt_toolkit.layout.dimension import Dimension +from prompt_toolkit.styles import BaseStyle, Style +from prompt_toolkit.utils import suspend_to_background_supported +from prompt_toolkit.widgets import Box, Frame, Label, RadioList + +__all__ = [ + "ChoiceInput", + "choice", +] + +_T = TypeVar("_T") +E = KeyPressEvent + + +def create_default_choice_input_style() -> BaseStyle: + return Style.from_dict( + { + "frame.border": "#884444", + "selected-option": "bold", + } + ) + + +class ChoiceInput(Generic[_T]): + """ + Input selection prompt. Ask the user to choose among a set of options. + + Example usage:: + + input_selection = ChoiceInput( + message="Please select a dish:", + options=[ + ("pizza", "Pizza with mushrooms"), + ("salad", "Salad with tomatoes"), + ("sushi", "Sushi"), + ], + default="pizza", + ) + result = input_selection.prompt() + + :param message: Plain text or formatted text to be shown before the options. + :param options: Sequence of ``(value, label)`` tuples. The labels can be + formatted text. + :param default: Default value. If none is given, the first option is + considered the default. + :param mouse_support: Enable mouse support. + :param style: :class:`.Style` instance for the color scheme. + :param symbol: Symbol to be displayed in front of the selected choice. + :param bottom_toolbar: Formatted text or callable that returns formatted + text to be displayed at the bottom of the screen. + :param show_frame: `bool` or + :class:`~prompt_toolkit.filters.Filter`. When True, surround the input + with a frame. + :param enable_interrupt: `bool` or + :class:`~prompt_toolkit.filters.Filter`. When True, raise + the ``interrupt_exception`` (``KeyboardInterrupt`` by default) when + control-c has been pressed. + :param interrupt_exception: The exception type that will be raised when + there is a keyboard interrupt (control-c keypress). + """ + + def __init__( + self, + *, + message: AnyFormattedText, + options: Sequence[tuple[_T, AnyFormattedText]], + default: _T | None = None, + mouse_support: bool = False, + style: BaseStyle | None = None, + symbol: str = ">", + bottom_toolbar: AnyFormattedText = None, + show_frame: FilterOrBool = False, + enable_suspend: FilterOrBool = False, + enable_interrupt: FilterOrBool = True, + interrupt_exception: type[BaseException] = KeyboardInterrupt, + key_bindings: KeyBindingsBase | None = None, + ) -> None: + if style is None: + style = create_default_choice_input_style() + + self.message = message + self.default = default + self.options = options + self.mouse_support = mouse_support + self.style = style + self.symbol = symbol + self.show_frame = show_frame + self.enable_suspend = enable_suspend + self.interrupt_exception = interrupt_exception + self.enable_interrupt = enable_interrupt + self.bottom_toolbar = bottom_toolbar + self.key_bindings = key_bindings + + def _create_application(self) -> Application[_T]: + radio_list = RadioList( + values=self.options, + default=self.default, + select_on_focus=True, + open_character="", + select_character=self.symbol, + close_character="", + show_cursor=False, + show_numbers=True, + container_style="class:input-selection", + default_style="class:option", + selected_style="", + checked_style="class:selected-option", + number_style="class:number", + show_scrollbar=False, + ) + container: AnyContainer = HSplit( + [ + Box( + Label(text=self.message, dont_extend_height=True), + padding_top=0, + padding_left=1, + padding_right=1, + padding_bottom=0, + ), + Box( + radio_list, + padding_top=0, + padding_left=3, + padding_right=1, + padding_bottom=0, + ), + ] + ) + + @Condition + def show_frame_filter() -> bool: + return to_filter(self.show_frame)() + + show_bottom_toolbar = ( + Condition(lambda: self.bottom_toolbar is not None) + & ~is_done + & renderer_height_is_known + ) + + container = ConditionalContainer( + Frame(container), + alternative_content=container, + filter=show_frame_filter, + ) + + bottom_toolbar = ConditionalContainer( + Window( + FormattedTextControl( + lambda: self.bottom_toolbar, style="class:bottom-toolbar.text" + ), + style="class:bottom-toolbar", + dont_extend_height=True, + height=Dimension(min=1), + ), + filter=show_bottom_toolbar, + ) + + layout = Layout( + HSplit( + [ + container, + # Add an empty window between the selection input and the + # bottom toolbar, if the bottom toolbar is visible, in + # order to allow the bottom toolbar to be displayed at the + # bottom of the screen. + ConditionalContainer(Window(), filter=show_bottom_toolbar), + bottom_toolbar, + ] + ), + focused_element=radio_list, + ) + + kb = KeyBindings() + + @kb.add("enter", eager=True) + def _accept_input(event: E) -> None: + "Accept input when enter has been pressed." + event.app.exit(result=radio_list.current_value, style="class:accepted") + + @Condition + def enable_interrupt() -> bool: + return to_filter(self.enable_interrupt)() + + @kb.add("c-c", filter=enable_interrupt) + @kb.add("", filter=enable_interrupt) + def _keyboard_interrupt(event: E) -> None: + "Abort when Control-C has been pressed." + event.app.exit(exception=self.interrupt_exception(), style="class:aborting") + + suspend_supported = Condition(suspend_to_background_supported) + + @Condition + def enable_suspend() -> bool: + return to_filter(self.enable_suspend)() + + @kb.add("c-z", filter=suspend_supported & enable_suspend) + def _suspend(event: E) -> None: + """ + Suspend process to background. + """ + event.app.suspend_to_background() + + return Application( + layout=layout, + full_screen=False, + mouse_support=self.mouse_support, + key_bindings=merge_key_bindings( + [kb, DynamicKeyBindings(lambda: self.key_bindings)] + ), + style=self.style, + ) + + def prompt(self) -> _T: + return self._create_application().run() + + async def prompt_async(self) -> _T: + return await self._create_application().run_async() + + +def choice( + message: AnyFormattedText, + *, + options: Sequence[tuple[_T, AnyFormattedText]], + default: _T | None = None, + mouse_support: bool = False, + style: BaseStyle | None = None, + symbol: str = ">", + bottom_toolbar: AnyFormattedText = None, + show_frame: bool = False, + enable_suspend: FilterOrBool = False, + enable_interrupt: FilterOrBool = True, + interrupt_exception: type[BaseException] = KeyboardInterrupt, + key_bindings: KeyBindingsBase | None = None, +) -> _T: + """ + Choice selection prompt. Ask the user to choose among a set of options. + + Example usage:: + + result = choice( + message="Please select a dish:", + options=[ + ("pizza", "Pizza with mushrooms"), + ("salad", "Salad with tomatoes"), + ("sushi", "Sushi"), + ], + default="pizza", + ) + + :param message: Plain text or formatted text to be shown before the options. + :param options: Sequence of ``(value, label)`` tuples. The labels can be + formatted text. + :param default: Default value. If none is given, the first option is + considered the default. + :param mouse_support: Enable mouse support. + :param style: :class:`.Style` instance for the color scheme. + :param symbol: Symbol to be displayed in front of the selected choice. + :param bottom_toolbar: Formatted text or callable that returns formatted + text to be displayed at the bottom of the screen. + :param show_frame: `bool` or + :class:`~prompt_toolkit.filters.Filter`. When True, surround the input + with a frame. + :param enable_interrupt: `bool` or + :class:`~prompt_toolkit.filters.Filter`. When True, raise + the ``interrupt_exception`` (``KeyboardInterrupt`` by default) when + control-c has been pressed. + :param interrupt_exception: The exception type that will be raised when + there is a keyboard interrupt (control-c keypress). + """ + return ChoiceInput[_T]( + message=message, + options=options, + default=default, + mouse_support=mouse_support, + style=style, + symbol=symbol, + bottom_toolbar=bottom_toolbar, + show_frame=show_frame, + enable_suspend=enable_suspend, + enable_interrupt=enable_interrupt, + interrupt_exception=interrupt_exception, + key_bindings=key_bindings, + ).prompt() diff --git a/lib/prompt_toolkit/shortcuts/dialogs.py b/lib/prompt_toolkit/shortcuts/dialogs.py new file mode 100644 index 0000000..d78e7db --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/dialogs.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +import functools +from asyncio import get_running_loop +from typing import Any, Callable, Sequence, TypeVar + +from prompt_toolkit.application import Application +from prompt_toolkit.application.current import get_app +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.completion import Completer +from prompt_toolkit.eventloop import run_in_executor_with_context +from prompt_toolkit.filters import FilterOrBool +from prompt_toolkit.formatted_text import AnyFormattedText +from prompt_toolkit.key_binding.bindings.focus import focus_next, focus_previous +from prompt_toolkit.key_binding.defaults import load_key_bindings +from prompt_toolkit.key_binding.key_bindings import KeyBindings, merge_key_bindings +from prompt_toolkit.layout import Layout +from prompt_toolkit.layout.containers import AnyContainer, HSplit +from prompt_toolkit.layout.dimension import Dimension as D +from prompt_toolkit.styles import BaseStyle +from prompt_toolkit.validation import Validator +from prompt_toolkit.widgets import ( + Box, + Button, + CheckboxList, + Dialog, + Label, + ProgressBar, + RadioList, + TextArea, + ValidationToolbar, +) + +__all__ = [ + "yes_no_dialog", + "button_dialog", + "input_dialog", + "message_dialog", + "radiolist_dialog", + "checkboxlist_dialog", + "progress_dialog", +] + + +def yes_no_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + yes_text: str = "Yes", + no_text: str = "No", + style: BaseStyle | None = None, +) -> Application[bool]: + """ + Display a Yes/No dialog. + Return a boolean. + """ + + def yes_handler() -> None: + get_app().exit(result=True) + + def no_handler() -> None: + get_app().exit(result=False) + + dialog = Dialog( + title=title, + body=Label(text=text, dont_extend_height=True), + buttons=[ + Button(text=yes_text, handler=yes_handler), + Button(text=no_text, handler=no_handler), + ], + with_background=True, + ) + + return _create_app(dialog, style) + + +_T = TypeVar("_T") + + +def button_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + buttons: list[tuple[str, _T]] = [], + style: BaseStyle | None = None, +) -> Application[_T]: + """ + Display a dialog with button choices (given as a list of tuples). + Return the value associated with button. + """ + + def button_handler(v: _T) -> None: + get_app().exit(result=v) + + dialog = Dialog( + title=title, + body=Label(text=text, dont_extend_height=True), + buttons=[ + Button(text=t, handler=functools.partial(button_handler, v)) + for t, v in buttons + ], + with_background=True, + ) + + return _create_app(dialog, style) + + +def input_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + ok_text: str = "OK", + cancel_text: str = "Cancel", + completer: Completer | None = None, + validator: Validator | None = None, + password: FilterOrBool = False, + style: BaseStyle | None = None, + default: str = "", +) -> Application[str]: + """ + Display a text input box. + Return the given text, or None when cancelled. + """ + + def accept(buf: Buffer) -> bool: + get_app().layout.focus(ok_button) + return True # Keep text. + + def ok_handler() -> None: + get_app().exit(result=textfield.text) + + ok_button = Button(text=ok_text, handler=ok_handler) + cancel_button = Button(text=cancel_text, handler=_return_none) + + textfield = TextArea( + text=default, + multiline=False, + password=password, + completer=completer, + validator=validator, + accept_handler=accept, + ) + + dialog = Dialog( + title=title, + body=HSplit( + [ + Label(text=text, dont_extend_height=True), + textfield, + ValidationToolbar(), + ], + padding=D(preferred=1, max=1), + ), + buttons=[ok_button, cancel_button], + with_background=True, + ) + + return _create_app(dialog, style) + + +def message_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + ok_text: str = "Ok", + style: BaseStyle | None = None, +) -> Application[None]: + """ + Display a simple message box and wait until the user presses enter. + """ + dialog = Dialog( + title=title, + body=Label(text=text, dont_extend_height=True), + buttons=[Button(text=ok_text, handler=_return_none)], + with_background=True, + ) + + return _create_app(dialog, style) + + +def radiolist_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + ok_text: str = "Ok", + cancel_text: str = "Cancel", + values: Sequence[tuple[_T, AnyFormattedText]] | None = None, + default: _T | None = None, + style: BaseStyle | None = None, +) -> Application[_T]: + """ + Display a simple list of element the user can choose amongst. + + Only one element can be selected at a time using Arrow keys and Enter. + The focus can be moved between the list and the Ok/Cancel button with tab. + """ + if values is None: + values = [] + + def ok_handler() -> None: + get_app().exit(result=radio_list.current_value) + + radio_list = RadioList(values=values, default=default) + + dialog = Dialog( + title=title, + body=HSplit( + [Label(text=text, dont_extend_height=True), radio_list], + padding=1, + ), + buttons=[ + Button(text=ok_text, handler=ok_handler), + Button(text=cancel_text, handler=_return_none), + ], + with_background=True, + ) + + return _create_app(dialog, style) + + +def checkboxlist_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + ok_text: str = "Ok", + cancel_text: str = "Cancel", + values: Sequence[tuple[_T, AnyFormattedText]] | None = None, + default_values: Sequence[_T] | None = None, + style: BaseStyle | None = None, +) -> Application[list[_T]]: + """ + Display a simple list of element the user can choose multiple values amongst. + + Several elements can be selected at a time using Arrow keys and Enter. + The focus can be moved between the list and the Ok/Cancel button with tab. + """ + if values is None: + values = [] + + def ok_handler() -> None: + get_app().exit(result=cb_list.current_values) + + cb_list = CheckboxList(values=values, default_values=default_values) + + dialog = Dialog( + title=title, + body=HSplit( + [Label(text=text, dont_extend_height=True), cb_list], + padding=1, + ), + buttons=[ + Button(text=ok_text, handler=ok_handler), + Button(text=cancel_text, handler=_return_none), + ], + with_background=True, + ) + + return _create_app(dialog, style) + + +def progress_dialog( + title: AnyFormattedText = "", + text: AnyFormattedText = "", + run_callback: Callable[[Callable[[int], None], Callable[[str], None]], None] = ( + lambda *a: None + ), + style: BaseStyle | None = None, +) -> Application[None]: + """ + :param run_callback: A function that receives as input a `set_percentage` + function and it does the work. + """ + loop = get_running_loop() + progressbar = ProgressBar() + text_area = TextArea( + focusable=False, + # Prefer this text area as big as possible, to avoid having a window + # that keeps resizing when we add text to it. + height=D(preferred=10**10), + ) + + dialog = Dialog( + body=HSplit( + [ + Box(Label(text=text)), + Box(text_area, padding=D.exact(1)), + progressbar, + ] + ), + title=title, + with_background=True, + ) + app = _create_app(dialog, style) + + def set_percentage(value: int) -> None: + progressbar.percentage = int(value) + app.invalidate() + + def log_text(text: str) -> None: + loop.call_soon_threadsafe(text_area.buffer.insert_text, text) + app.invalidate() + + # Run the callback in the executor. When done, set a return value for the + # UI, so that it quits. + def start() -> None: + try: + run_callback(set_percentage, log_text) + finally: + app.exit() + + def pre_run() -> None: + run_in_executor_with_context(start) + + app.pre_run_callables.append(pre_run) + + return app + + +def _create_app(dialog: AnyContainer, style: BaseStyle | None) -> Application[Any]: + # Key bindings. + bindings = KeyBindings() + bindings.add("tab")(focus_next) + bindings.add("s-tab")(focus_previous) + + return Application( + layout=Layout(dialog), + key_bindings=merge_key_bindings([load_key_bindings(), bindings]), + mouse_support=True, + style=style, + full_screen=True, + ) + + +def _return_none() -> None: + "Button handler that returns None." + get_app().exit() diff --git a/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py b/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py new file mode 100644 index 0000000..2261a5b --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from .base import ProgressBar, ProgressBarCounter +from .formatters import ( + Bar, + Formatter, + IterationsPerSecond, + Label, + Percentage, + Progress, + Rainbow, + SpinningWheel, + Text, + TimeElapsed, + TimeLeft, +) + +__all__ = [ + "ProgressBar", + "ProgressBarCounter", + # Formatters. + "Formatter", + "Text", + "Label", + "Percentage", + "Bar", + "Progress", + "TimeElapsed", + "TimeLeft", + "IterationsPerSecond", + "SpinningWheel", + "Rainbow", +] diff --git a/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..8da3e6d Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..c6e6d0b Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc b/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc new file mode 100644 index 0000000..ba39b95 Binary files /dev/null and b/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/shortcuts/progress_bar/base.py b/lib/prompt_toolkit/shortcuts/progress_bar/base.py new file mode 100644 index 0000000..a7c2a52 --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/progress_bar/base.py @@ -0,0 +1,449 @@ +""" +Progress bar implementation on top of prompt_toolkit. + +:: + + with ProgressBar(...) as pb: + for item in pb(data): + ... +""" + +from __future__ import annotations + +import contextvars +import datetime +import functools +import os +import signal +import threading +import traceback +from typing import ( + Callable, + Generic, + Iterable, + Iterator, + Sequence, + Sized, + TextIO, + TypeVar, + cast, +) + +from prompt_toolkit.application import Application +from prompt_toolkit.application.current import get_app_session +from prompt_toolkit.filters import Condition, is_done, renderer_height_is_known +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + to_formatted_text, +) +from prompt_toolkit.input import Input +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.layout import ( + ConditionalContainer, + FormattedTextControl, + HSplit, + Layout, + VSplit, + Window, +) +from prompt_toolkit.layout.controls import UIContent, UIControl +from prompt_toolkit.layout.dimension import AnyDimension, D +from prompt_toolkit.output import ColorDepth, Output +from prompt_toolkit.styles import BaseStyle +from prompt_toolkit.utils import in_main_thread + +from .formatters import Formatter, create_default_formatters + +__all__ = ["ProgressBar"] + +E = KeyPressEvent + +_SIGWINCH = getattr(signal, "SIGWINCH", None) + + +def create_key_bindings(cancel_callback: Callable[[], None] | None) -> KeyBindings: + """ + Key bindings handled by the progress bar. + (The main thread is not supposed to handle any key bindings.) + """ + kb = KeyBindings() + + @kb.add("c-l") + def _clear(event: E) -> None: + event.app.renderer.clear() + + if cancel_callback is not None: + + @kb.add("c-c") + def _interrupt(event: E) -> None: + "Kill the 'body' of the progress bar, but only if we run from the main thread." + assert cancel_callback is not None + cancel_callback() + + return kb + + +_T = TypeVar("_T") + + +class ProgressBar: + """ + Progress bar context manager. + + Usage :: + + with ProgressBar(...) as pb: + for item in pb(data): + ... + + :param title: Text to be displayed above the progress bars. This can be a + callable or formatted text as well. + :param formatters: List of :class:`.Formatter` instances. + :param bottom_toolbar: Text to be displayed in the bottom toolbar. This + can be a callable or formatted text. + :param style: :class:`prompt_toolkit.styles.BaseStyle` instance. + :param key_bindings: :class:`.KeyBindings` instance. + :param cancel_callback: Callback function that's called when control-c is + pressed by the user. This can be used for instance to start "proper" + cancellation if the wrapped code supports it. + :param file: The file object used for rendering, by default `sys.stderr` is used. + + :param color_depth: `prompt_toolkit` `ColorDepth` instance. + :param output: :class:`~prompt_toolkit.output.Output` instance. + :param input: :class:`~prompt_toolkit.input.Input` instance. + """ + + def __init__( + self, + title: AnyFormattedText = None, + formatters: Sequence[Formatter] | None = None, + bottom_toolbar: AnyFormattedText = None, + style: BaseStyle | None = None, + key_bindings: KeyBindings | None = None, + cancel_callback: Callable[[], None] | None = None, + file: TextIO | None = None, + color_depth: ColorDepth | None = None, + output: Output | None = None, + input: Input | None = None, + ) -> None: + self.title = title + self.formatters = formatters or create_default_formatters() + self.bottom_toolbar = bottom_toolbar + self.counters: list[ProgressBarCounter[object]] = [] + self.style = style + self.key_bindings = key_bindings + self.cancel_callback = cancel_callback + + # If no `cancel_callback` was given, and we're creating the progress + # bar from the main thread. Cancel by sending a `KeyboardInterrupt` to + # the main thread. + if self.cancel_callback is None and in_main_thread(): + + def keyboard_interrupt_to_main_thread() -> None: + os.kill(os.getpid(), signal.SIGINT) + + self.cancel_callback = keyboard_interrupt_to_main_thread + + # Note that we use __stderr__ as default error output, because that + # works best with `patch_stdout`. + self.color_depth = color_depth + self.output = output or get_app_session().output + self.input = input or get_app_session().input + + self._thread: threading.Thread | None = None + + self._has_sigwinch = False + self._app_started = threading.Event() + + def __enter__(self) -> ProgressBar: + # Create UI Application. + title_toolbar = ConditionalContainer( + Window( + FormattedTextControl(lambda: self.title), + height=1, + style="class:progressbar,title", + ), + filter=Condition(lambda: self.title is not None), + ) + + bottom_toolbar = ConditionalContainer( + Window( + FormattedTextControl( + lambda: self.bottom_toolbar, style="class:bottom-toolbar.text" + ), + style="class:bottom-toolbar", + height=1, + ), + filter=~is_done + & renderer_height_is_known + & Condition(lambda: self.bottom_toolbar is not None), + ) + + def width_for_formatter(formatter: Formatter) -> AnyDimension: + # Needs to be passed as callable (partial) to the 'width' + # parameter, because we want to call it on every resize. + return formatter.get_width(progress_bar=self) + + progress_controls = [ + Window( + content=_ProgressControl(self, f, self.cancel_callback), + width=functools.partial(width_for_formatter, f), + ) + for f in self.formatters + ] + + self.app: Application[None] = Application( + min_redraw_interval=0.05, + layout=Layout( + HSplit( + [ + title_toolbar, + VSplit( + progress_controls, + height=lambda: D( + preferred=len(self.counters), max=len(self.counters) + ), + ), + Window(), + bottom_toolbar, + ] + ) + ), + style=self.style, + key_bindings=self.key_bindings, + refresh_interval=0.3, + color_depth=self.color_depth, + output=self.output, + input=self.input, + ) + + # Run application in different thread. + def run() -> None: + try: + self.app.run(pre_run=self._app_started.set) + except BaseException as e: + traceback.print_exc() + print(e) + + ctx: contextvars.Context = contextvars.copy_context() + + self._thread = threading.Thread(target=ctx.run, args=(run,)) + self._thread.start() + + return self + + def __exit__(self, *a: object) -> None: + # Wait for the app to be started. Make sure we don't quit earlier, + # otherwise `self.app.exit` won't terminate the app because + # `self.app.future` has not yet been set. + self._app_started.wait() + + # Quit UI application. + if self.app.is_running and self.app.loop is not None: + self.app.loop.call_soon_threadsafe(self.app.exit) + + if self._thread is not None: + self._thread.join() + + def __call__( + self, + data: Iterable[_T] | None = None, + label: AnyFormattedText = "", + remove_when_done: bool = False, + total: int | None = None, + ) -> ProgressBarCounter[_T]: + """ + Start a new counter. + + :param label: Title text or description for this progress. (This can be + formatted text as well). + :param remove_when_done: When `True`, hide this progress bar. + :param total: Specify the maximum value if it can't be calculated by + calling ``len``. + """ + counter = ProgressBarCounter( + self, data, label=label, remove_when_done=remove_when_done, total=total + ) + self.counters.append(counter) + return counter + + def invalidate(self) -> None: + self.app.invalidate() + + +class _ProgressControl(UIControl): + """ + User control for the progress bar. + """ + + def __init__( + self, + progress_bar: ProgressBar, + formatter: Formatter, + cancel_callback: Callable[[], None] | None, + ) -> None: + self.progress_bar = progress_bar + self.formatter = formatter + self._key_bindings = create_key_bindings(cancel_callback) + + def create_content(self, width: int, height: int) -> UIContent: + items: list[StyleAndTextTuples] = [] + + for pr in self.progress_bar.counters: + try: + text = self.formatter.format(self.progress_bar, pr, width) + except BaseException: + traceback.print_exc() + text = "ERROR" + + items.append(to_formatted_text(text)) + + def get_line(i: int) -> StyleAndTextTuples: + return items[i] + + return UIContent(get_line=get_line, line_count=len(items), show_cursor=False) + + def is_focusable(self) -> bool: + return True # Make sure that the key bindings work. + + def get_key_bindings(self) -> KeyBindings: + return self._key_bindings + + +_CounterItem = TypeVar("_CounterItem", covariant=True) + + +class ProgressBarCounter(Generic[_CounterItem]): + """ + An individual counter (A progress bar can have multiple counters). + """ + + def __init__( + self, + progress_bar: ProgressBar, + data: Iterable[_CounterItem] | None = None, + label: AnyFormattedText = "", + remove_when_done: bool = False, + total: int | None = None, + ) -> None: + self.start_time = datetime.datetime.now() + self.stop_time: datetime.datetime | None = None + self.progress_bar = progress_bar + self.data = data + self.items_completed = 0 + self.label = label + self.remove_when_done = remove_when_done + self._done = False + self.total: int | None + + if total is None: + try: + self.total = len(cast(Sized, data)) + except TypeError: + self.total = None # We don't know the total length. + else: + self.total = total + + def __iter__(self) -> Iterator[_CounterItem]: + if self.data is not None: + try: + for item in self.data: + yield item + self.item_completed() + + # Only done if we iterate to the very end. + self.done = True + finally: + # Ensure counter has stopped even if we did not iterate to the + # end (e.g. break or exceptions). + self.stopped = True + else: + raise NotImplementedError("No data defined to iterate over.") + + def item_completed(self) -> None: + """ + Start handling the next item. + + (Can be called manually in case we don't have a collection to loop through.) + """ + self.items_completed += 1 + self.progress_bar.invalidate() + + @property + def done(self) -> bool: + """Whether a counter has been completed. + + Done counter have been stopped (see stopped) and removed depending on + remove_when_done value. + + Contrast this with stopped. A stopped counter may be terminated before + 100% completion. A done counter has reached its 100% completion. + """ + return self._done + + @done.setter + def done(self, value: bool) -> None: + self._done = value + self.stopped = value + + if value and self.remove_when_done: + self.progress_bar.counters.remove(self) + + @property + def stopped(self) -> bool: + """Whether a counter has been stopped. + + Stopped counters no longer have increasing time_elapsed. This distinction is + also used to prevent the Bar formatter with unknown totals from continuing to run. + + A stopped counter (but not done) can be used to signal that a given counter has + encountered an error but allows other counters to continue + (e.g. download X of Y failed). Given how only done counters are removed + (see remove_when_done) this can help aggregate failures from a large number of + successes. + + Contrast this with done. A done counter has reached its 100% completion. + A stopped counter may be terminated before 100% completion. + """ + return self.stop_time is not None + + @stopped.setter + def stopped(self, value: bool) -> None: + if value: + # This counter has not already been stopped. + if not self.stop_time: + self.stop_time = datetime.datetime.now() + else: + # Clearing any previously set stop_time. + self.stop_time = None + + @property + def percentage(self) -> float: + if self.total is None: + return 0 + else: + return self.items_completed * 100 / max(self.total, 1) + + @property + def time_elapsed(self) -> datetime.timedelta: + """ + Return how much time has been elapsed since the start. + """ + if self.stop_time is None: + return datetime.datetime.now() - self.start_time + else: + return self.stop_time - self.start_time + + @property + def time_left(self) -> datetime.timedelta | None: + """ + Timedelta representing the time left. + """ + if self.total is None or not self.percentage: + return None + elif self.done or self.stopped: + return datetime.timedelta(0) + else: + return self.time_elapsed * (100 - self.percentage) / self.percentage diff --git a/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py b/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py new file mode 100644 index 0000000..202949c --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py @@ -0,0 +1,431 @@ +""" +Formatter classes for the progress bar. +Each progress bar consists of a list of these formatters. +""" + +from __future__ import annotations + +import datetime +import time +from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING + +from prompt_toolkit.formatted_text import ( + HTML, + AnyFormattedText, + StyleAndTextTuples, + to_formatted_text, +) +from prompt_toolkit.formatted_text.utils import fragment_list_width +from prompt_toolkit.layout.dimension import AnyDimension, D +from prompt_toolkit.layout.utils import explode_text_fragments +from prompt_toolkit.utils import get_cwidth + +if TYPE_CHECKING: + from .base import ProgressBar, ProgressBarCounter + +__all__ = [ + "Formatter", + "Text", + "Label", + "Percentage", + "Bar", + "Progress", + "TimeElapsed", + "TimeLeft", + "IterationsPerSecond", + "SpinningWheel", + "Rainbow", + "create_default_formatters", +] + + +class Formatter(metaclass=ABCMeta): + """ + Base class for any formatter. + """ + + @abstractmethod + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + pass + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + return D() + + +class Text(Formatter): + """ + Display plain text. + """ + + def __init__(self, text: AnyFormattedText, style: str = "") -> None: + self.text = to_formatted_text(text, style=style) + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + return self.text + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + return fragment_list_width(self.text) + + +class Label(Formatter): + """ + Display the name of the current task. + + :param width: If a `width` is given, use this width. Scroll the text if it + doesn't fit in this width. + :param suffix: String suffix to be added after the task name, e.g. ': '. + If no task name was given, no suffix will be added. + """ + + def __init__(self, width: AnyDimension = None, suffix: str = "") -> None: + self.width = width + self.suffix = suffix + + def _add_suffix(self, label: AnyFormattedText) -> StyleAndTextTuples: + label = to_formatted_text(label, style="class:label") + return label + [("", self.suffix)] + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + label = self._add_suffix(progress.label) + cwidth = fragment_list_width(label) + + if cwidth > width: + # It doesn't fit -> scroll task name. + label = explode_text_fragments(label) + max_scroll = cwidth - width + current_scroll = int(time.time() * 3 % max_scroll) + label = label[current_scroll:] + + return label + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + if self.width: + return self.width + + all_labels = [self._add_suffix(c.label) for c in progress_bar.counters] + if all_labels: + max_widths = max(fragment_list_width(l) for l in all_labels) + return D(preferred=max_widths, max=max_widths) + else: + return D() + + +class Percentage(Formatter): + """ + Display the progress as a percentage. + """ + + template = HTML("{percentage:>5}%") + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + return self.template.format(percentage=round(progress.percentage, 1)) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + return D.exact(6) + + +class Bar(Formatter): + """ + Display the progress bar itself. + """ + + template = HTML( + "{start}{bar_a}{bar_b}{bar_c}{end}" + ) + + def __init__( + self, + start: str = "[", + end: str = "]", + sym_a: str = "=", + sym_b: str = ">", + sym_c: str = " ", + unknown: str = "#", + ) -> None: + assert len(sym_a) == 1 and get_cwidth(sym_a) == 1 + assert len(sym_c) == 1 and get_cwidth(sym_c) == 1 + + self.start = start + self.end = end + self.sym_a = sym_a + self.sym_b = sym_b + self.sym_c = sym_c + self.unknown = unknown + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + if progress.done or progress.total or progress.stopped: + sym_a, sym_b, sym_c = self.sym_a, self.sym_b, self.sym_c + + # Compute pb_a based on done, total, or stopped states. + if progress.done: + # 100% completed irrelevant of how much was actually marked as completed. + percent = 1.0 + else: + # Show percentage completed. + percent = progress.percentage / 100 + else: + # Total is unknown and bar is still running. + sym_a, sym_b, sym_c = self.sym_c, self.unknown, self.sym_c + + # Compute percent based on the time. + percent = time.time() * 20 % 100 / 100 + + # Subtract left, sym_b, and right. + width -= get_cwidth(self.start + sym_b + self.end) + + # Scale percent by width + pb_a = int(percent * width) + bar_a = sym_a * pb_a + bar_b = sym_b + bar_c = sym_c * (width - pb_a) + + return self.template.format( + start=self.start, end=self.end, bar_a=bar_a, bar_b=bar_b, bar_c=bar_c + ) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + return D(min=9) + + +class Progress(Formatter): + """ + Display the progress as text. E.g. "8/20" + """ + + template = HTML("{current:>3}/{total:>3}") + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + return self.template.format( + current=progress.items_completed, total=progress.total or "?" + ) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + all_lengths = [ + len("{:>3}".format(c.total or "?")) for c in progress_bar.counters + ] + all_lengths.append(1) + return D.exact(max(all_lengths) * 2 + 1) + + +def _format_timedelta(timedelta: datetime.timedelta) -> str: + """ + Return hh:mm:ss, or mm:ss if the amount of hours is zero. + """ + result = f"{timedelta}".split(".")[0] + if result.startswith("0:"): + result = result[2:] + return result + + +class TimeElapsed(Formatter): + """ + Display the elapsed time. + """ + + template = HTML("{time_elapsed}") + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + text = _format_timedelta(progress.time_elapsed).rjust(width) + return self.template.format(time_elapsed=text) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + all_values = [ + len(_format_timedelta(c.time_elapsed)) for c in progress_bar.counters + ] + if all_values: + return max(all_values) + return 0 + + +class TimeLeft(Formatter): + """ + Display the time left. + """ + + template = HTML("{time_left}") + unknown = "?:??:??" + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + time_left = progress.time_left + if time_left is not None: + formatted_time_left = _format_timedelta(time_left) + else: + formatted_time_left = self.unknown + + return self.template.format(time_left=formatted_time_left.rjust(width)) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + all_values = [ + len(_format_timedelta(c.time_left)) if c.time_left is not None else 7 + for c in progress_bar.counters + ] + if all_values: + return max(all_values) + return 0 + + +class IterationsPerSecond(Formatter): + """ + Display the iterations per second. + """ + + template = HTML( + "{iterations_per_second:.2f}" + ) + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + value = progress.items_completed / progress.time_elapsed.total_seconds() + return self.template.format(iterations_per_second=value) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + all_values = [ + len(f"{c.items_completed / c.time_elapsed.total_seconds():.2f}") + for c in progress_bar.counters + ] + if all_values: + return max(all_values) + return 0 + + +class SpinningWheel(Formatter): + """ + Display a spinning wheel. + """ + + template = HTML("{0}") + characters = r"/-\|" + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + index = int(time.time() * 3) % len(self.characters) + return self.template.format(self.characters[index]) + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + return D.exact(1) + + +def _hue_to_rgb(hue: float) -> tuple[int, int, int]: + """ + Take hue between 0 and 1, return (r, g, b). + """ + i = int(hue * 6.0) + f = (hue * 6.0) - i + + q = int(255 * (1.0 - f)) + t = int(255 * (1.0 - (1.0 - f))) + + i %= 6 + + return [ + (255, t, 0), + (q, 255, 0), + (0, 255, t), + (0, q, 255), + (t, 0, 255), + (255, 0, q), + ][i] + + +class Rainbow(Formatter): + """ + For the fun. Add rainbow colors to any of the other formatters. + """ + + colors = ["#%.2x%.2x%.2x" % _hue_to_rgb(h / 100.0) for h in range(0, 100)] + + def __init__(self, formatter: Formatter) -> None: + self.formatter = formatter + + def format( + self, + progress_bar: ProgressBar, + progress: ProgressBarCounter[object], + width: int, + ) -> AnyFormattedText: + # Get formatted text from nested formatter, and explode it in + # text/style tuples. + result = self.formatter.format(progress_bar, progress, width) + result = explode_text_fragments(to_formatted_text(result)) + + # Insert colors. + result2: StyleAndTextTuples = [] + shift = int(time.time() * 3) % len(self.colors) + + for i, (style, text, *_) in enumerate(result): + result2.append( + (style + " " + self.colors[(i + shift) % len(self.colors)], text) + ) + return result2 + + def get_width(self, progress_bar: ProgressBar) -> AnyDimension: + return self.formatter.get_width(progress_bar) + + +def create_default_formatters() -> list[Formatter]: + """ + Return the list of default formatters. + """ + return [ + Label(), + Text(" "), + Percentage(), + Text(" "), + Bar(), + Text(" "), + Progress(), + Text(" "), + Text("eta [", style="class:time-left"), + TimeLeft(), + Text("]", style="class:time-left"), + Text(" "), + ] diff --git a/lib/prompt_toolkit/shortcuts/prompt.py b/lib/prompt_toolkit/shortcuts/prompt.py new file mode 100644 index 0000000..68cfeb9 --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/prompt.py @@ -0,0 +1,1538 @@ +""" +Line editing functionality. +--------------------------- + +This provides a UI for a line input, similar to GNU Readline, libedit and +linenoise. + +Either call the `prompt` function for every line input. Or create an instance +of the :class:`.PromptSession` class and call the `prompt` method from that +class. In the second case, we'll have a 'session' that keeps all the state like +the history in between several calls. + +There is a lot of overlap between the arguments taken by the `prompt` function +and the `PromptSession` (like `completer`, `style`, etcetera). There we have +the freedom to decide which settings we want for the whole 'session', and which +we want for an individual `prompt`. + +Example:: + + # Simple `prompt` call. + result = prompt('Say something: ') + + # Using a 'session'. + s = PromptSession() + result = s.prompt('Say something: ') +""" + +from __future__ import annotations + +from asyncio import get_running_loop +from contextlib import contextmanager +from enum import Enum +from functools import partial +from typing import TYPE_CHECKING, Callable, Generic, Iterator, TypeVar, Union, cast + +from prompt_toolkit.application import Application +from prompt_toolkit.application.current import get_app +from prompt_toolkit.auto_suggest import AutoSuggest, DynamicAutoSuggest +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.clipboard import Clipboard, DynamicClipboard, InMemoryClipboard +from prompt_toolkit.completion import Completer, DynamicCompleter, ThreadedCompleter +from prompt_toolkit.cursor_shapes import ( + AnyCursorShapeConfig, + CursorShapeConfig, + DynamicCursorShapeConfig, +) +from prompt_toolkit.document import Document +from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER, EditingMode +from prompt_toolkit.eventloop import InputHook +from prompt_toolkit.filters import ( + Condition, + FilterOrBool, + has_arg, + has_focus, + is_done, + is_true, + renderer_height_is_known, + to_filter, +) +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + fragment_list_to_text, + merge_formatted_text, + to_formatted_text, +) +from prompt_toolkit.history import History, InMemoryHistory +from prompt_toolkit.input.base import Input +from prompt_toolkit.key_binding.bindings.auto_suggest import load_auto_suggest_bindings +from prompt_toolkit.key_binding.bindings.completion import ( + display_completions_like_readline, +) +from prompt_toolkit.key_binding.bindings.open_in_editor import ( + load_open_in_editor_bindings, +) +from prompt_toolkit.key_binding.key_bindings import ( + ConditionalKeyBindings, + DynamicKeyBindings, + KeyBindings, + KeyBindingsBase, + merge_key_bindings, +) +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.layout import Float, FloatContainer, HSplit, Window +from prompt_toolkit.layout.containers import ConditionalContainer, WindowAlign +from prompt_toolkit.layout.controls import ( + BufferControl, + FormattedTextControl, + SearchBufferControl, +) +from prompt_toolkit.layout.dimension import Dimension +from prompt_toolkit.layout.layout import Layout +from prompt_toolkit.layout.menus import CompletionsMenu, MultiColumnCompletionsMenu +from prompt_toolkit.layout.processors import ( + AfterInput, + AppendAutoSuggestion, + ConditionalProcessor, + DisplayMultipleCursors, + DynamicProcessor, + HighlightIncrementalSearchProcessor, + HighlightSelectionProcessor, + PasswordProcessor, + Processor, + ReverseSearchProcessor, + merge_processors, +) +from prompt_toolkit.layout.utils import explode_text_fragments +from prompt_toolkit.lexers import DynamicLexer, Lexer +from prompt_toolkit.output import ColorDepth, DummyOutput, Output +from prompt_toolkit.styles import ( + BaseStyle, + ConditionalStyleTransformation, + DynamicStyle, + DynamicStyleTransformation, + StyleTransformation, + SwapLightAndDarkStyleTransformation, + merge_style_transformations, +) +from prompt_toolkit.utils import ( + get_cwidth, + is_dumb_terminal, + suspend_to_background_supported, + to_str, +) +from prompt_toolkit.validation import DynamicValidator, Validator +from prompt_toolkit.widgets import Frame +from prompt_toolkit.widgets.toolbars import ( + SearchToolbar, + SystemToolbar, + ValidationToolbar, +) + +if TYPE_CHECKING: + from prompt_toolkit.formatted_text.base import MagicFormattedText + +__all__ = [ + "PromptSession", + "prompt", + "confirm", + "create_confirm_session", # Used by '_display_completions_like_readline'. + "CompleteStyle", +] + +_StyleAndTextTuplesCallable = Callable[[], StyleAndTextTuples] +E = KeyPressEvent + + +def _split_multiline_prompt( + get_prompt_text: _StyleAndTextTuplesCallable, +) -> tuple[ + Callable[[], bool], _StyleAndTextTuplesCallable, _StyleAndTextTuplesCallable +]: + """ + Take a `get_prompt_text` function and return three new functions instead. + One that tells whether this prompt consists of multiple lines; one that + returns the fragments to be shown on the lines above the input; and another + one with the fragments to be shown at the first line of the input. + """ + + def has_before_fragments() -> bool: + for fragment, char, *_ in get_prompt_text(): + if "\n" in char: + return True + return False + + def before() -> StyleAndTextTuples: + result: StyleAndTextTuples = [] + found_nl = False + for fragment, char, *_ in reversed(explode_text_fragments(get_prompt_text())): + if found_nl: + result.insert(0, (fragment, char)) + elif char == "\n": + found_nl = True + return result + + def first_input_line() -> StyleAndTextTuples: + result: StyleAndTextTuples = [] + for fragment, char, *_ in reversed(explode_text_fragments(get_prompt_text())): + if char == "\n": + break + else: + result.insert(0, (fragment, char)) + return result + + return has_before_fragments, before, first_input_line + + +class _RPrompt(Window): + """ + The prompt that is displayed on the right side of the Window. + """ + + def __init__(self, text: AnyFormattedText) -> None: + super().__init__( + FormattedTextControl(text=text), + align=WindowAlign.RIGHT, + style="class:rprompt", + ) + + +class CompleteStyle(str, Enum): + """ + How to display autocompletions for the prompt. + """ + + value: str + + COLUMN = "COLUMN" + MULTI_COLUMN = "MULTI_COLUMN" + READLINE_LIKE = "READLINE_LIKE" + + +# Formatted text for the continuation prompt. It's the same like other +# formatted text, except that if it's a callable, it takes three arguments. +PromptContinuationText = Union[ + str, + "MagicFormattedText", + StyleAndTextTuples, + # (prompt_width, line_number, wrap_count) -> AnyFormattedText. + Callable[[int, int, int], AnyFormattedText], +] + +_T = TypeVar("_T") + + +class PromptSession(Generic[_T]): + """ + PromptSession for a prompt application, which can be used as a GNU Readline + replacement. + + This is a wrapper around a lot of ``prompt_toolkit`` functionality and can + be a replacement for `raw_input`. + + All parameters that expect "formatted text" can take either just plain text + (a unicode object), a list of ``(style_str, text)`` tuples or an HTML object. + + Example usage:: + + s = PromptSession(message='>') + text = s.prompt() + + :param message: Plain text or formatted text to be shown before the prompt. + This can also be a callable that returns formatted text. + :param multiline: `bool` or :class:`~prompt_toolkit.filters.Filter`. + When True, prefer a layout that is more adapted for multiline input. + Text after newlines is automatically indented, and search/arg input is + shown below the input, instead of replacing the prompt. + :param wrap_lines: `bool` or :class:`~prompt_toolkit.filters.Filter`. + When True (the default), automatically wrap long lines instead of + scrolling horizontally. + :param is_password: Show asterisks instead of the actual typed characters. + :param editing_mode: ``EditingMode.VI`` or ``EditingMode.EMACS``. + :param vi_mode: `bool`, if True, Identical to ``editing_mode=EditingMode.VI``. + :param complete_while_typing: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Enable autocompletion while + typing. + :param validate_while_typing: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Enable input validation while + typing. + :param enable_history_search: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Enable up-arrow parting + string matching. + :param search_ignore_case: + :class:`~prompt_toolkit.filters.Filter`. Search case insensitive. + :param lexer: :class:`~prompt_toolkit.lexers.Lexer` to be used for the + syntax highlighting. + :param validator: :class:`~prompt_toolkit.validation.Validator` instance + for input validation. + :param completer: :class:`~prompt_toolkit.completion.Completer` instance + for input completion. + :param complete_in_thread: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Run the completer code in a + background thread in order to avoid blocking the user interface. + For ``CompleteStyle.READLINE_LIKE``, this setting has no effect. There + we always run the completions in the main thread. + :param reserve_space_for_menu: Space to be reserved for displaying the menu. + (0 means that no space needs to be reserved.) + :param auto_suggest: :class:`~prompt_toolkit.auto_suggest.AutoSuggest` + instance for input suggestions. + :param style: :class:`.Style` instance for the color scheme. + :param include_default_pygments_style: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Tell whether the default + styling for Pygments lexers has to be included. By default, this is + true, but it is recommended to be disabled if another Pygments style is + passed as the `style` argument, otherwise, two Pygments styles will be + merged. + :param style_transformation: + :class:`~prompt_toolkit.style.StyleTransformation` instance. + :param swap_light_and_dark_colors: `bool` or + :class:`~prompt_toolkit.filters.Filter`. When enabled, apply + :class:`~prompt_toolkit.style.SwapLightAndDarkStyleTransformation`. + This is useful for switching between dark and light terminal + backgrounds. + :param enable_system_prompt: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Pressing Meta+'!' will show + a system prompt. + :param enable_suspend: `bool` or :class:`~prompt_toolkit.filters.Filter`. + Enable Control-Z style suspension. + :param enable_open_in_editor: `bool` or + :class:`~prompt_toolkit.filters.Filter`. Pressing 'v' in Vi mode or + C-X C-E in emacs mode will open an external editor. + :param history: :class:`~prompt_toolkit.history.History` instance. + :param clipboard: :class:`~prompt_toolkit.clipboard.Clipboard` instance. + (e.g. :class:`~prompt_toolkit.clipboard.InMemoryClipboard`) + :param rprompt: Text or formatted text to be displayed on the right side. + This can also be a callable that returns (formatted) text. + :param bottom_toolbar: Formatted text or callable that returns formatted + text to be displayed at the bottom of the screen. + :param prompt_continuation: Text that needs to be displayed for a multiline + prompt continuation. This can either be formatted text or a callable + that takes a `prompt_width`, `line_number` and `wrap_count` as input + and returns formatted text. When this is `None` (the default), then + `prompt_width` spaces will be used. + :param complete_style: ``CompleteStyle.COLUMN``, + ``CompleteStyle.MULTI_COLUMN`` or ``CompleteStyle.READLINE_LIKE``. + :param mouse_support: `bool` or :class:`~prompt_toolkit.filters.Filter` + to enable mouse support. + :param placeholder: Text to be displayed when no input has been given + yet. Unlike the `default` parameter, this won't be returned as part of + the output ever. This can be formatted text or a callable that returns + formatted text. + :param show_frame: `bool` or + :class:`~prompt_toolkit.filters.Filter`. When True, surround the input + with a frame. + :param refresh_interval: (number; in seconds) When given, refresh the UI + every so many seconds. + :param input: `Input` object. (Note that the preferred way to change the + input/output is by creating an `AppSession`.) + :param output: `Output` object. + :param interrupt_exception: The exception type that will be raised when + there is a keyboard interrupt (control-c keypress). + :param eof_exception: The exception type that will be raised when there is + an end-of-file/exit event (control-d keypress). + """ + + _fields = ( + "message", + "lexer", + "completer", + "complete_in_thread", + "is_password", + "editing_mode", + "key_bindings", + "is_password", + "bottom_toolbar", + "style", + "style_transformation", + "swap_light_and_dark_colors", + "color_depth", + "cursor", + "include_default_pygments_style", + "rprompt", + "multiline", + "prompt_continuation", + "wrap_lines", + "enable_history_search", + "search_ignore_case", + "complete_while_typing", + "validate_while_typing", + "complete_style", + "mouse_support", + "auto_suggest", + "clipboard", + "validator", + "refresh_interval", + "input_processors", + "placeholder", + "enable_system_prompt", + "enable_suspend", + "enable_open_in_editor", + "reserve_space_for_menu", + "tempfile_suffix", + "tempfile", + "show_frame", + ) + + def __init__( + self, + message: AnyFormattedText = "", + *, + multiline: FilterOrBool = False, + wrap_lines: FilterOrBool = True, + is_password: FilterOrBool = False, + vi_mode: bool = False, + editing_mode: EditingMode = EditingMode.EMACS, + complete_while_typing: FilterOrBool = True, + validate_while_typing: FilterOrBool = True, + enable_history_search: FilterOrBool = False, + search_ignore_case: FilterOrBool = False, + lexer: Lexer | None = None, + enable_system_prompt: FilterOrBool = False, + enable_suspend: FilterOrBool = False, + enable_open_in_editor: FilterOrBool = False, + validator: Validator | None = None, + completer: Completer | None = None, + complete_in_thread: bool = False, + reserve_space_for_menu: int = 8, + complete_style: CompleteStyle = CompleteStyle.COLUMN, + auto_suggest: AutoSuggest | None = None, + style: BaseStyle | None = None, + style_transformation: StyleTransformation | None = None, + swap_light_and_dark_colors: FilterOrBool = False, + color_depth: ColorDepth | None = None, + cursor: AnyCursorShapeConfig = None, + include_default_pygments_style: FilterOrBool = True, + history: History | None = None, + clipboard: Clipboard | None = None, + prompt_continuation: PromptContinuationText | None = None, + rprompt: AnyFormattedText = None, + bottom_toolbar: AnyFormattedText = None, + mouse_support: FilterOrBool = False, + input_processors: list[Processor] | None = None, + placeholder: AnyFormattedText | None = None, + key_bindings: KeyBindingsBase | None = None, + erase_when_done: bool = False, + tempfile_suffix: str | Callable[[], str] | None = ".txt", + tempfile: str | Callable[[], str] | None = None, + refresh_interval: float = 0, + show_frame: FilterOrBool = False, + input: Input | None = None, + output: Output | None = None, + interrupt_exception: type[BaseException] = KeyboardInterrupt, + eof_exception: type[BaseException] = EOFError, + ) -> None: + history = history or InMemoryHistory() + clipboard = clipboard or InMemoryClipboard() + + # Ensure backwards-compatibility, when `vi_mode` is passed. + if vi_mode: + editing_mode = EditingMode.VI + + # Store all settings in this class. + self._input = input + self._output = output + + # Store attributes. + # (All except 'editing_mode'.) + self.message = message + self.lexer = lexer + self.completer = completer + self.complete_in_thread = complete_in_thread + self.is_password = is_password + self.key_bindings = key_bindings + self.bottom_toolbar = bottom_toolbar + self.style = style + self.style_transformation = style_transformation + self.swap_light_and_dark_colors = swap_light_and_dark_colors + self.color_depth = color_depth + self.cursor = cursor + self.include_default_pygments_style = include_default_pygments_style + self.rprompt = rprompt + self.multiline = multiline + self.prompt_continuation = prompt_continuation + self.wrap_lines = wrap_lines + self.enable_history_search = enable_history_search + self.search_ignore_case = search_ignore_case + self.complete_while_typing = complete_while_typing + self.validate_while_typing = validate_while_typing + self.complete_style = complete_style + self.mouse_support = mouse_support + self.auto_suggest = auto_suggest + self.clipboard = clipboard + self.validator = validator + self.refresh_interval = refresh_interval + self.input_processors = input_processors + self.placeholder = placeholder + self.enable_system_prompt = enable_system_prompt + self.enable_suspend = enable_suspend + self.enable_open_in_editor = enable_open_in_editor + self.reserve_space_for_menu = reserve_space_for_menu + self.tempfile_suffix = tempfile_suffix + self.tempfile = tempfile + self.show_frame = show_frame + self.interrupt_exception = interrupt_exception + self.eof_exception = eof_exception + + # Create buffers, layout and Application. + self.history = history + self.default_buffer = self._create_default_buffer() + self.search_buffer = self._create_search_buffer() + self.layout = self._create_layout() + self.app = self._create_application(editing_mode, erase_when_done) + + def _dyncond(self, attr_name: str) -> Condition: + """ + Dynamically take this setting from this 'PromptSession' class. + `attr_name` represents an attribute name of this class. Its value + can either be a boolean or a `Filter`. + + This returns something that can be used as either a `Filter` + or `Filter`. + """ + + @Condition + def dynamic() -> bool: + value = cast(FilterOrBool, getattr(self, attr_name)) + return to_filter(value)() + + return dynamic + + def _create_default_buffer(self) -> Buffer: + """ + Create and return the default input buffer. + """ + dyncond = self._dyncond + + # Create buffers list. + def accept(buff: Buffer) -> bool: + """Accept the content of the default buffer. This is called when + the validation succeeds.""" + cast(Application[str], get_app()).exit( + result=buff.document.text, style="class:accepted" + ) + return True # Keep text, we call 'reset' later on. + + return Buffer( + name=DEFAULT_BUFFER, + # Make sure that complete_while_typing is disabled when + # enable_history_search is enabled. (First convert to Filter, + # to avoid doing bitwise operations on bool objects.) + complete_while_typing=Condition( + lambda: is_true(self.complete_while_typing) + and not is_true(self.enable_history_search) + and not self.complete_style == CompleteStyle.READLINE_LIKE + ), + validate_while_typing=dyncond("validate_while_typing"), + enable_history_search=dyncond("enable_history_search"), + validator=DynamicValidator(lambda: self.validator), + completer=DynamicCompleter( + lambda: ThreadedCompleter(self.completer) + if self.complete_in_thread and self.completer + else self.completer + ), + history=self.history, + auto_suggest=DynamicAutoSuggest(lambda: self.auto_suggest), + accept_handler=accept, + tempfile_suffix=lambda: to_str(self.tempfile_suffix or ""), + tempfile=lambda: to_str(self.tempfile or ""), + ) + + def _create_search_buffer(self) -> Buffer: + return Buffer(name=SEARCH_BUFFER) + + def _create_layout(self) -> Layout: + """ + Create `Layout` for this prompt. + """ + dyncond = self._dyncond + + # Create functions that will dynamically split the prompt. (If we have + # a multiline prompt.) + ( + has_before_fragments, + get_prompt_text_1, + get_prompt_text_2, + ) = _split_multiline_prompt(self._get_prompt) + + default_buffer = self.default_buffer + search_buffer = self.search_buffer + + # Create processors list. + @Condition + def display_placeholder() -> bool: + return self.placeholder is not None and self.default_buffer.text == "" + + all_input_processors = [ + HighlightIncrementalSearchProcessor(), + HighlightSelectionProcessor(), + ConditionalProcessor( + AppendAutoSuggestion(), has_focus(default_buffer) & ~is_done + ), + ConditionalProcessor(PasswordProcessor(), dyncond("is_password")), + DisplayMultipleCursors(), + # Users can insert processors here. + DynamicProcessor(lambda: merge_processors(self.input_processors or [])), + ConditionalProcessor( + AfterInput(lambda: self.placeholder), + filter=display_placeholder, + ), + ] + + # Create bottom toolbars. + bottom_toolbar = ConditionalContainer( + Window( + FormattedTextControl( + lambda: self.bottom_toolbar, style="class:bottom-toolbar.text" + ), + style="class:bottom-toolbar", + dont_extend_height=True, + height=Dimension(min=1), + ), + filter=Condition(lambda: self.bottom_toolbar is not None) + & ~is_done + & renderer_height_is_known, + ) + + search_toolbar = SearchToolbar( + search_buffer, ignore_case=dyncond("search_ignore_case") + ) + + search_buffer_control = SearchBufferControl( + buffer=search_buffer, + input_processors=[ReverseSearchProcessor()], + ignore_case=dyncond("search_ignore_case"), + ) + + system_toolbar = SystemToolbar( + enable_global_bindings=dyncond("enable_system_prompt") + ) + + def get_search_buffer_control() -> SearchBufferControl: + "Return the UIControl to be focused when searching start." + if is_true(self.multiline): + return search_toolbar.control + else: + return search_buffer_control + + default_buffer_control = BufferControl( + buffer=default_buffer, + search_buffer_control=get_search_buffer_control, + input_processors=all_input_processors, + include_default_input_processors=False, + lexer=DynamicLexer(lambda: self.lexer), + preview_search=True, + ) + + default_buffer_window = Window( + default_buffer_control, + height=self._get_default_buffer_control_height, + get_line_prefix=partial( + self._get_line_prefix, get_prompt_text_2=get_prompt_text_2 + ), + wrap_lines=dyncond("wrap_lines"), + ) + + @Condition + def multi_column_complete_style() -> bool: + return self.complete_style == CompleteStyle.MULTI_COLUMN + + # Build the layout. + + # The main input, with completion menus floating on top of it. + main_input_container = FloatContainer( + HSplit( + [ + ConditionalContainer( + Window( + FormattedTextControl(get_prompt_text_1), + dont_extend_height=True, + ), + Condition(has_before_fragments), + ), + ConditionalContainer( + default_buffer_window, + Condition( + lambda: get_app().layout.current_control + != search_buffer_control + ), + ), + ConditionalContainer( + Window(search_buffer_control), + Condition( + lambda: get_app().layout.current_control + == search_buffer_control + ), + ), + ] + ), + [ + # Completion menus. + # NOTE: Especially the multi-column menu needs to be + # transparent, because the shape is not always + # rectangular due to the meta-text below the menu. + Float( + xcursor=True, + ycursor=True, + transparent=True, + content=CompletionsMenu( + max_height=16, + scroll_offset=1, + extra_filter=has_focus(default_buffer) + & ~multi_column_complete_style, + ), + ), + Float( + xcursor=True, + ycursor=True, + transparent=True, + content=MultiColumnCompletionsMenu( + show_meta=True, + extra_filter=has_focus(default_buffer) + & multi_column_complete_style, + ), + ), + # The right prompt. + Float( + right=0, + top=0, + hide_when_covering_content=True, + content=_RPrompt(lambda: self.rprompt), + ), + ], + ) + + layout = HSplit( + [ + # Wrap the main input in a frame, if requested. + ConditionalContainer( + Frame(main_input_container), + filter=dyncond("show_frame"), + alternative_content=main_input_container, + ), + ConditionalContainer(ValidationToolbar(), filter=~is_done), + ConditionalContainer( + system_toolbar, dyncond("enable_system_prompt") & ~is_done + ), + # In multiline mode, we use two toolbars for 'arg' and 'search'. + ConditionalContainer( + Window(FormattedTextControl(self._get_arg_text), height=1), + dyncond("multiline") & has_arg, + ), + ConditionalContainer(search_toolbar, dyncond("multiline") & ~is_done), + bottom_toolbar, + ] + ) + + return Layout(layout, default_buffer_window) + + def _create_application( + self, editing_mode: EditingMode, erase_when_done: bool + ) -> Application[_T]: + """ + Create the `Application` object. + """ + dyncond = self._dyncond + + # Default key bindings. + auto_suggest_bindings = load_auto_suggest_bindings() + open_in_editor_bindings = load_open_in_editor_bindings() + prompt_bindings = self._create_prompt_bindings() + + # Create application + application: Application[_T] = Application( + layout=self.layout, + style=DynamicStyle(lambda: self.style), + style_transformation=merge_style_transformations( + [ + DynamicStyleTransformation(lambda: self.style_transformation), + ConditionalStyleTransformation( + SwapLightAndDarkStyleTransformation(), + dyncond("swap_light_and_dark_colors"), + ), + ] + ), + include_default_pygments_style=dyncond("include_default_pygments_style"), + clipboard=DynamicClipboard(lambda: self.clipboard), + key_bindings=merge_key_bindings( + [ + merge_key_bindings( + [ + auto_suggest_bindings, + ConditionalKeyBindings( + open_in_editor_bindings, + dyncond("enable_open_in_editor") + & has_focus(DEFAULT_BUFFER), + ), + prompt_bindings, + ] + ), + DynamicKeyBindings(lambda: self.key_bindings), + ] + ), + mouse_support=dyncond("mouse_support"), + editing_mode=editing_mode, + erase_when_done=erase_when_done, + reverse_vi_search_direction=True, + color_depth=lambda: self.color_depth, + cursor=DynamicCursorShapeConfig(lambda: self.cursor), + refresh_interval=self.refresh_interval, + input=self._input, + output=self._output, + ) + + # During render time, make sure that we focus the right search control + # (if we are searching). - This could be useful if people make the + # 'multiline' property dynamic. + """ + def on_render(app): + multiline = is_true(self.multiline) + current_control = app.layout.current_control + + if multiline: + if current_control == search_buffer_control: + app.layout.current_control = search_toolbar.control + app.invalidate() + else: + if current_control == search_toolbar.control: + app.layout.current_control = search_buffer_control + app.invalidate() + + app.on_render += on_render + """ + + return application + + def _create_prompt_bindings(self) -> KeyBindings: + """ + Create the KeyBindings for a prompt application. + """ + kb = KeyBindings() + handle = kb.add + default_focused = has_focus(DEFAULT_BUFFER) + + @Condition + def do_accept() -> bool: + return not is_true(self.multiline) and self.app.layout.has_focus( + DEFAULT_BUFFER + ) + + @handle("enter", filter=do_accept & default_focused) + def _accept_input(event: E) -> None: + "Accept input when enter has been pressed." + self.default_buffer.validate_and_handle() + + @Condition + def readline_complete_style() -> bool: + return self.complete_style == CompleteStyle.READLINE_LIKE + + @handle("tab", filter=readline_complete_style & default_focused) + def _complete_like_readline(event: E) -> None: + "Display completions (like Readline)." + display_completions_like_readline(event) + + @handle("c-c", filter=default_focused) + @handle("") + def _keyboard_interrupt(event: E) -> None: + "Abort when Control-C has been pressed." + event.app.exit(exception=self.interrupt_exception(), style="class:aborting") + + @Condition + def ctrl_d_condition() -> bool: + """Ctrl-D binding is only active when the default buffer is selected + and empty.""" + app = get_app() + return ( + app.current_buffer.name == DEFAULT_BUFFER + and not app.current_buffer.text + ) + + @handle("c-d", filter=ctrl_d_condition & default_focused) + def _eof(event: E) -> None: + "Exit when Control-D has been pressed." + event.app.exit(exception=self.eof_exception(), style="class:exiting") + + suspend_supported = Condition(suspend_to_background_supported) + + @Condition + def enable_suspend() -> bool: + return to_filter(self.enable_suspend)() + + @handle("c-z", filter=suspend_supported & enable_suspend) + def _suspend(event: E) -> None: + """ + Suspend process to background. + """ + event.app.suspend_to_background() + + return kb + + def prompt( + self, + # When any of these arguments are passed, this value is overwritten + # in this PromptSession. + message: AnyFormattedText | None = None, + # `message` should go first, because people call it as + # positional argument. + *, + editing_mode: EditingMode | None = None, + refresh_interval: float | None = None, + vi_mode: bool | None = None, + lexer: Lexer | None = None, + completer: Completer | None = None, + complete_in_thread: bool | None = None, + is_password: bool | None = None, + key_bindings: KeyBindingsBase | None = None, + bottom_toolbar: AnyFormattedText | None = None, + style: BaseStyle | None = None, + color_depth: ColorDepth | None = None, + cursor: AnyCursorShapeConfig | None = None, + include_default_pygments_style: FilterOrBool | None = None, + style_transformation: StyleTransformation | None = None, + swap_light_and_dark_colors: FilterOrBool | None = None, + rprompt: AnyFormattedText | None = None, + multiline: FilterOrBool | None = None, + prompt_continuation: PromptContinuationText | None = None, + wrap_lines: FilterOrBool | None = None, + enable_history_search: FilterOrBool | None = None, + search_ignore_case: FilterOrBool | None = None, + complete_while_typing: FilterOrBool | None = None, + validate_while_typing: FilterOrBool | None = None, + complete_style: CompleteStyle | None = None, + auto_suggest: AutoSuggest | None = None, + validator: Validator | None = None, + clipboard: Clipboard | None = None, + mouse_support: FilterOrBool | None = None, + input_processors: list[Processor] | None = None, + placeholder: AnyFormattedText | None = None, + reserve_space_for_menu: int | None = None, + enable_system_prompt: FilterOrBool | None = None, + enable_suspend: FilterOrBool | None = None, + enable_open_in_editor: FilterOrBool | None = None, + tempfile_suffix: str | Callable[[], str] | None = None, + tempfile: str | Callable[[], str] | None = None, + show_frame: FilterOrBool | None = None, + # Following arguments are specific to the current `prompt()` call. + default: str | Document = "", + accept_default: bool = False, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + in_thread: bool = False, + inputhook: InputHook | None = None, + ) -> _T: + """ + Display the prompt. + + The first set of arguments is a subset of the :class:`~.PromptSession` + class itself. For these, passing in ``None`` will keep the current + values that are active in the session. Passing in a value will set the + attribute for the session, which means that it applies to the current, + but also to the next prompts. + + Note that in order to erase a ``Completer``, ``Validator`` or + ``AutoSuggest``, you can't use ``None``. Instead pass in a + ``DummyCompleter``, ``DummyValidator`` or ``DummyAutoSuggest`` instance + respectively. For a ``Lexer`` you can pass in an empty ``SimpleLexer``. + + Additional arguments, specific for this prompt: + + :param default: The default input text to be shown. (This can be edited + by the user). + :param accept_default: When `True`, automatically accept the default + value without allowing the user to edit the input. + :param pre_run: Callable, called at the start of `Application.run`. + :param in_thread: Run the prompt in a background thread; block the + current thread. This avoids interference with an event loop in the + current thread. Like `Application.run(in_thread=True)`. + + This method will raise ``KeyboardInterrupt`` when control-c has been + pressed (for abort) and ``EOFError`` when control-d has been pressed + (for exit). + """ + # NOTE: We used to create a backup of the PromptSession attributes and + # restore them after exiting the prompt. This code has been + # removed, because it was confusing and didn't really serve a use + # case. (People were changing `Application.editing_mode` + # dynamically and surprised that it was reset after every call.) + + # NOTE 2: YES, this is a lot of repeation below... + # However, it is a very convenient for a user to accept all + # these parameters in this `prompt` method as well. We could + # use `locals()` and `setattr` to avoid the repetition, but + # then we loose the advantage of mypy and pyflakes to be able + # to verify the code. + if message is not None: + self.message = message + if editing_mode is not None: + self.editing_mode = editing_mode + if refresh_interval is not None: + self.refresh_interval = refresh_interval + if vi_mode: + self.editing_mode = EditingMode.VI + if lexer is not None: + self.lexer = lexer + if completer is not None: + self.completer = completer + if complete_in_thread is not None: + self.complete_in_thread = complete_in_thread + if is_password is not None: + self.is_password = is_password + if key_bindings is not None: + self.key_bindings = key_bindings + if bottom_toolbar is not None: + self.bottom_toolbar = bottom_toolbar + if style is not None: + self.style = style + if color_depth is not None: + self.color_depth = color_depth + if cursor is not None: + self.cursor = cursor + if include_default_pygments_style is not None: + self.include_default_pygments_style = include_default_pygments_style + if style_transformation is not None: + self.style_transformation = style_transformation + if swap_light_and_dark_colors is not None: + self.swap_light_and_dark_colors = swap_light_and_dark_colors + if rprompt is not None: + self.rprompt = rprompt + if multiline is not None: + self.multiline = multiline + if prompt_continuation is not None: + self.prompt_continuation = prompt_continuation + if wrap_lines is not None: + self.wrap_lines = wrap_lines + if enable_history_search is not None: + self.enable_history_search = enable_history_search + if search_ignore_case is not None: + self.search_ignore_case = search_ignore_case + if complete_while_typing is not None: + self.complete_while_typing = complete_while_typing + if validate_while_typing is not None: + self.validate_while_typing = validate_while_typing + if complete_style is not None: + self.complete_style = complete_style + if auto_suggest is not None: + self.auto_suggest = auto_suggest + if validator is not None: + self.validator = validator + if clipboard is not None: + self.clipboard = clipboard + if mouse_support is not None: + self.mouse_support = mouse_support + if input_processors is not None: + self.input_processors = input_processors + if placeholder is not None: + self.placeholder = placeholder + if reserve_space_for_menu is not None: + self.reserve_space_for_menu = reserve_space_for_menu + if enable_system_prompt is not None: + self.enable_system_prompt = enable_system_prompt + if enable_suspend is not None: + self.enable_suspend = enable_suspend + if enable_open_in_editor is not None: + self.enable_open_in_editor = enable_open_in_editor + if tempfile_suffix is not None: + self.tempfile_suffix = tempfile_suffix + if tempfile is not None: + self.tempfile = tempfile + if show_frame is not None: + self.show_frame = show_frame + + self._add_pre_run_callables(pre_run, accept_default) + self.default_buffer.reset( + default if isinstance(default, Document) else Document(default) + ) + self.app.refresh_interval = self.refresh_interval # This is not reactive. + + # If we are using the default output, and have a dumb terminal. Use the + # dumb prompt. + if self._output is None and is_dumb_terminal(): + with self._dumb_prompt(self.message) as dump_app: + return dump_app.run(in_thread=in_thread, handle_sigint=handle_sigint) + + return self.app.run( + set_exception_handler=set_exception_handler, + in_thread=in_thread, + handle_sigint=handle_sigint, + inputhook=inputhook, + ) + + @contextmanager + def _dumb_prompt(self, message: AnyFormattedText = "") -> Iterator[Application[_T]]: + """ + Create prompt `Application` for prompt function for dumb terminals. + + Dumb terminals have minimum rendering capabilities. We can only print + text to the screen. We can't use colors, and we can't do cursor + movements. The Emacs inferior shell is an example of a dumb terminal. + + We will show the prompt, and wait for the input. We still handle arrow + keys, and all custom key bindings, but we don't really render the + cursor movements. Instead we only print the typed character that's + right before the cursor. + """ + # Send prompt to output. + self.output.write(fragment_list_to_text(to_formatted_text(self.message))) + self.output.flush() + + # Key bindings for the dumb prompt: mostly the same as the full prompt. + key_bindings: KeyBindingsBase = self._create_prompt_bindings() + if self.key_bindings: + key_bindings = merge_key_bindings([self.key_bindings, key_bindings]) + + # Create and run application. + application = cast( + Application[_T], + Application( + input=self.input, + output=DummyOutput(), + layout=self.layout, + key_bindings=key_bindings, + ), + ) + + def on_text_changed(_: object) -> None: + self.output.write(self.default_buffer.document.text_before_cursor[-1:]) + self.output.flush() + + self.default_buffer.on_text_changed += on_text_changed + + try: + yield application + finally: + # Render line ending. + self.output.write("\r\n") + self.output.flush() + + self.default_buffer.on_text_changed -= on_text_changed + + async def prompt_async( + self, + # When any of these arguments are passed, this value is overwritten + # in this PromptSession. + message: AnyFormattedText | None = None, + # `message` should go first, because people call it as + # positional argument. + *, + editing_mode: EditingMode | None = None, + refresh_interval: float | None = None, + vi_mode: bool | None = None, + lexer: Lexer | None = None, + completer: Completer | None = None, + complete_in_thread: bool | None = None, + is_password: bool | None = None, + key_bindings: KeyBindingsBase | None = None, + bottom_toolbar: AnyFormattedText | None = None, + style: BaseStyle | None = None, + color_depth: ColorDepth | None = None, + cursor: CursorShapeConfig | None = None, + include_default_pygments_style: FilterOrBool | None = None, + style_transformation: StyleTransformation | None = None, + swap_light_and_dark_colors: FilterOrBool | None = None, + rprompt: AnyFormattedText | None = None, + multiline: FilterOrBool | None = None, + prompt_continuation: PromptContinuationText | None = None, + wrap_lines: FilterOrBool | None = None, + enable_history_search: FilterOrBool | None = None, + search_ignore_case: FilterOrBool | None = None, + complete_while_typing: FilterOrBool | None = None, + validate_while_typing: FilterOrBool | None = None, + complete_style: CompleteStyle | None = None, + auto_suggest: AutoSuggest | None = None, + validator: Validator | None = None, + clipboard: Clipboard | None = None, + mouse_support: FilterOrBool | None = None, + input_processors: list[Processor] | None = None, + placeholder: AnyFormattedText | None = None, + reserve_space_for_menu: int | None = None, + enable_system_prompt: FilterOrBool | None = None, + enable_suspend: FilterOrBool | None = None, + enable_open_in_editor: FilterOrBool | None = None, + tempfile_suffix: str | Callable[[], str] | None = None, + tempfile: str | Callable[[], str] | None = None, + show_frame: FilterOrBool = False, + # Following arguments are specific to the current `prompt()` call. + default: str | Document = "", + accept_default: bool = False, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + ) -> _T: + if message is not None: + self.message = message + if editing_mode is not None: + self.editing_mode = editing_mode + if refresh_interval is not None: + self.refresh_interval = refresh_interval + if vi_mode: + self.editing_mode = EditingMode.VI + if lexer is not None: + self.lexer = lexer + if completer is not None: + self.completer = completer + if complete_in_thread is not None: + self.complete_in_thread = complete_in_thread + if is_password is not None: + self.is_password = is_password + if key_bindings is not None: + self.key_bindings = key_bindings + if bottom_toolbar is not None: + self.bottom_toolbar = bottom_toolbar + if style is not None: + self.style = style + if color_depth is not None: + self.color_depth = color_depth + if cursor is not None: + self.cursor = cursor + if include_default_pygments_style is not None: + self.include_default_pygments_style = include_default_pygments_style + if style_transformation is not None: + self.style_transformation = style_transformation + if swap_light_and_dark_colors is not None: + self.swap_light_and_dark_colors = swap_light_and_dark_colors + if rprompt is not None: + self.rprompt = rprompt + if multiline is not None: + self.multiline = multiline + if prompt_continuation is not None: + self.prompt_continuation = prompt_continuation + if wrap_lines is not None: + self.wrap_lines = wrap_lines + if enable_history_search is not None: + self.enable_history_search = enable_history_search + if search_ignore_case is not None: + self.search_ignore_case = search_ignore_case + if complete_while_typing is not None: + self.complete_while_typing = complete_while_typing + if validate_while_typing is not None: + self.validate_while_typing = validate_while_typing + if complete_style is not None: + self.complete_style = complete_style + if auto_suggest is not None: + self.auto_suggest = auto_suggest + if validator is not None: + self.validator = validator + if clipboard is not None: + self.clipboard = clipboard + if mouse_support is not None: + self.mouse_support = mouse_support + if input_processors is not None: + self.input_processors = input_processors + if placeholder is not None: + self.placeholder = placeholder + if reserve_space_for_menu is not None: + self.reserve_space_for_menu = reserve_space_for_menu + if enable_system_prompt is not None: + self.enable_system_prompt = enable_system_prompt + if enable_suspend is not None: + self.enable_suspend = enable_suspend + if enable_open_in_editor is not None: + self.enable_open_in_editor = enable_open_in_editor + if tempfile_suffix is not None: + self.tempfile_suffix = tempfile_suffix + if tempfile is not None: + self.tempfile = tempfile + if show_frame is not None: + self.show_frame = show_frame + + self._add_pre_run_callables(pre_run, accept_default) + self.default_buffer.reset( + default if isinstance(default, Document) else Document(default) + ) + self.app.refresh_interval = self.refresh_interval # This is not reactive. + + # If we are using the default output, and have a dumb terminal. Use the + # dumb prompt. + if self._output is None and is_dumb_terminal(): + with self._dumb_prompt(self.message) as dump_app: + return await dump_app.run_async(handle_sigint=handle_sigint) + + return await self.app.run_async( + set_exception_handler=set_exception_handler, handle_sigint=handle_sigint + ) + + def _add_pre_run_callables( + self, pre_run: Callable[[], None] | None, accept_default: bool + ) -> None: + def pre_run2() -> None: + if pre_run: + pre_run() + + if accept_default: + # Validate and handle input. We use `call_from_executor` in + # order to run it "soon" (during the next iteration of the + # event loop), instead of right now. Otherwise, it won't + # display the default value. + get_running_loop().call_soon(self.default_buffer.validate_and_handle) + + self.app.pre_run_callables.append(pre_run2) + + @property + def editing_mode(self) -> EditingMode: + return self.app.editing_mode + + @editing_mode.setter + def editing_mode(self, value: EditingMode) -> None: + self.app.editing_mode = value + + def _get_default_buffer_control_height(self) -> Dimension: + # If there is an autocompletion menu to be shown, make sure that our + # layout has at least a minimal height in order to display it. + if ( + self.completer is not None + and self.complete_style != CompleteStyle.READLINE_LIKE + ): + space = self.reserve_space_for_menu + else: + space = 0 + + if space and not get_app().is_done: + buff = self.default_buffer + + # Reserve the space, either when there are completions, or when + # `complete_while_typing` is true and we expect completions very + # soon. + if buff.complete_while_typing() or buff.complete_state is not None: + return Dimension(min=space) + + return Dimension() + + def _get_prompt(self) -> StyleAndTextTuples: + return to_formatted_text(self.message, style="class:prompt") + + def _get_continuation( + self, width: int, line_number: int, wrap_count: int + ) -> StyleAndTextTuples: + """ + Insert the prompt continuation. + + :param width: The width that was used for the prompt. (more or less can + be used.) + :param line_number: + :param wrap_count: Amount of times that the line has been wrapped. + """ + prompt_continuation = self.prompt_continuation + + if callable(prompt_continuation): + continuation: AnyFormattedText = prompt_continuation( + width, line_number, wrap_count + ) + else: + continuation = prompt_continuation + + # When the continuation prompt is not given, choose the same width as + # the actual prompt. + if continuation is None and is_true(self.multiline): + continuation = " " * width + + return to_formatted_text(continuation, style="class:prompt-continuation") + + def _get_line_prefix( + self, + line_number: int, + wrap_count: int, + get_prompt_text_2: _StyleAndTextTuplesCallable, + ) -> StyleAndTextTuples: + """ + Return whatever needs to be inserted before every line. + (the prompt, or a line continuation.) + """ + # First line: display the "arg" or the prompt. + if line_number == 0 and wrap_count == 0: + if not is_true(self.multiline) and get_app().key_processor.arg is not None: + return self._inline_arg() + else: + return get_prompt_text_2() + + # For the next lines, display the appropriate continuation. + prompt_width = get_cwidth(fragment_list_to_text(get_prompt_text_2())) + return self._get_continuation(prompt_width, line_number, wrap_count) + + def _get_arg_text(self) -> StyleAndTextTuples: + "'arg' toolbar, for in multiline mode." + arg = self.app.key_processor.arg + if arg is None: + # Should not happen because of the `has_arg` filter in the layout. + return [] + + if arg == "-": + arg = "-1" + + return [("class:arg-toolbar", "Repeat: "), ("class:arg-toolbar.text", arg)] + + def _inline_arg(self) -> StyleAndTextTuples: + "'arg' prefix, for in single line mode." + app = get_app() + if app.key_processor.arg is None: + return [] + else: + arg = app.key_processor.arg + + return [ + ("class:prompt.arg", "(arg: "), + ("class:prompt.arg.text", str(arg)), + ("class:prompt.arg", ") "), + ] + + # Expose the Input and Output objects as attributes, mainly for + # backward-compatibility. + + @property + def input(self) -> Input: + return self.app.input + + @property + def output(self) -> Output: + return self.app.output + + +def prompt( + message: AnyFormattedText | None = None, + *, + history: History | None = None, + editing_mode: EditingMode | None = None, + refresh_interval: float | None = None, + vi_mode: bool | None = None, + lexer: Lexer | None = None, + completer: Completer | None = None, + complete_in_thread: bool | None = None, + is_password: bool | None = None, + key_bindings: KeyBindingsBase | None = None, + bottom_toolbar: AnyFormattedText | None = None, + style: BaseStyle | None = None, + color_depth: ColorDepth | None = None, + cursor: AnyCursorShapeConfig = None, + include_default_pygments_style: FilterOrBool | None = None, + style_transformation: StyleTransformation | None = None, + swap_light_and_dark_colors: FilterOrBool | None = None, + rprompt: AnyFormattedText | None = None, + multiline: FilterOrBool | None = None, + prompt_continuation: PromptContinuationText | None = None, + wrap_lines: FilterOrBool | None = None, + enable_history_search: FilterOrBool | None = None, + search_ignore_case: FilterOrBool | None = None, + complete_while_typing: FilterOrBool | None = None, + validate_while_typing: FilterOrBool | None = None, + complete_style: CompleteStyle | None = None, + auto_suggest: AutoSuggest | None = None, + validator: Validator | None = None, + clipboard: Clipboard | None = None, + mouse_support: FilterOrBool | None = None, + input_processors: list[Processor] | None = None, + placeholder: AnyFormattedText | None = None, + reserve_space_for_menu: int | None = None, + enable_system_prompt: FilterOrBool | None = None, + enable_suspend: FilterOrBool | None = None, + enable_open_in_editor: FilterOrBool | None = None, + tempfile_suffix: str | Callable[[], str] | None = None, + tempfile: str | Callable[[], str] | None = None, + show_frame: FilterOrBool | None = None, + # Following arguments are specific to the current `prompt()` call. + default: str = "", + accept_default: bool = False, + pre_run: Callable[[], None] | None = None, + set_exception_handler: bool = True, + handle_sigint: bool = True, + in_thread: bool = False, + inputhook: InputHook | None = None, +) -> str: + """ + The global `prompt` function. This will create a new `PromptSession` + instance for every call. + """ + # The history is the only attribute that has to be passed to the + # `PromptSession`, it can't be passed into the `prompt()` method. + session: PromptSession[str] = PromptSession(history=history) + + return session.prompt( + message, + editing_mode=editing_mode, + refresh_interval=refresh_interval, + vi_mode=vi_mode, + lexer=lexer, + completer=completer, + complete_in_thread=complete_in_thread, + is_password=is_password, + key_bindings=key_bindings, + bottom_toolbar=bottom_toolbar, + style=style, + color_depth=color_depth, + cursor=cursor, + include_default_pygments_style=include_default_pygments_style, + style_transformation=style_transformation, + swap_light_and_dark_colors=swap_light_and_dark_colors, + rprompt=rprompt, + multiline=multiline, + prompt_continuation=prompt_continuation, + wrap_lines=wrap_lines, + enable_history_search=enable_history_search, + search_ignore_case=search_ignore_case, + complete_while_typing=complete_while_typing, + validate_while_typing=validate_while_typing, + complete_style=complete_style, + auto_suggest=auto_suggest, + validator=validator, + clipboard=clipboard, + mouse_support=mouse_support, + input_processors=input_processors, + placeholder=placeholder, + reserve_space_for_menu=reserve_space_for_menu, + enable_system_prompt=enable_system_prompt, + enable_suspend=enable_suspend, + enable_open_in_editor=enable_open_in_editor, + tempfile_suffix=tempfile_suffix, + tempfile=tempfile, + show_frame=show_frame, + default=default, + accept_default=accept_default, + pre_run=pre_run, + set_exception_handler=set_exception_handler, + handle_sigint=handle_sigint, + in_thread=in_thread, + inputhook=inputhook, + ) + + +prompt.__doc__ = PromptSession.prompt.__doc__ + + +def create_confirm_session( + message: AnyFormattedText, suffix: str = " (y/n) " +) -> PromptSession[bool]: + """ + Create a `PromptSession` object for the 'confirm' function. + """ + bindings = KeyBindings() + + @bindings.add("y") + @bindings.add("Y") + def yes(event: E) -> None: + session.default_buffer.text = "y" + event.app.exit(result=True) + + @bindings.add("n") + @bindings.add("N") + def no(event: E) -> None: + session.default_buffer.text = "n" + event.app.exit(result=False) + + @bindings.add(Keys.Any) + def _(event: E) -> None: + "Disallow inserting other text." + pass + + complete_message = merge_formatted_text([message, suffix]) + session: PromptSession[bool] = PromptSession( + complete_message, key_bindings=bindings + ) + return session + + +def confirm(message: AnyFormattedText = "Confirm?", suffix: str = " (y/n) ") -> bool: + """ + Display a confirmation prompt that returns True/False. + """ + session = create_confirm_session(message, suffix) + return session.prompt() diff --git a/lib/prompt_toolkit/shortcuts/utils.py b/lib/prompt_toolkit/shortcuts/utils.py new file mode 100644 index 0000000..abf4fd2 --- /dev/null +++ b/lib/prompt_toolkit/shortcuts/utils.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +from asyncio.events import AbstractEventLoop +from typing import TYPE_CHECKING, Any, TextIO + +from prompt_toolkit.application import Application +from prompt_toolkit.application.current import get_app_or_none, get_app_session +from prompt_toolkit.application.run_in_terminal import run_in_terminal +from prompt_toolkit.formatted_text import ( + FormattedText, + StyleAndTextTuples, + to_formatted_text, +) +from prompt_toolkit.input import DummyInput +from prompt_toolkit.layout import Layout +from prompt_toolkit.output import ColorDepth, Output +from prompt_toolkit.output.defaults import create_output +from prompt_toolkit.renderer import ( + print_formatted_text as renderer_print_formatted_text, +) +from prompt_toolkit.styles import ( + BaseStyle, + StyleTransformation, + default_pygments_style, + default_ui_style, + merge_styles, +) + +if TYPE_CHECKING: + from prompt_toolkit.layout.containers import AnyContainer + +__all__ = [ + "print_formatted_text", + "print_container", + "clear", + "set_title", + "clear_title", +] + + +def print_formatted_text( + *values: Any, + sep: str = " ", + end: str = "\n", + file: TextIO | None = None, + flush: bool = False, + style: BaseStyle | None = None, + output: Output | None = None, + color_depth: ColorDepth | None = None, + style_transformation: StyleTransformation | None = None, + include_default_pygments_style: bool = True, +) -> None: + """ + :: + + print_formatted_text(*values, sep=' ', end='\\n', file=None, flush=False, style=None, output=None) + + Print text to stdout. This is supposed to be compatible with Python's print + function, but supports printing of formatted text. You can pass a + :class:`~prompt_toolkit.formatted_text.FormattedText`, + :class:`~prompt_toolkit.formatted_text.HTML` or + :class:`~prompt_toolkit.formatted_text.ANSI` object to print formatted + text. + + * Print HTML as follows:: + + print_formatted_text(HTML('Some italic text This is red!')) + + style = Style.from_dict({ + 'hello': '#ff0066', + 'world': '#884444 italic', + }) + print_formatted_text(HTML('Hello world!'), style=style) + + * Print a list of (style_str, text) tuples in the given style to the + output. E.g.:: + + style = Style.from_dict({ + 'hello': '#ff0066', + 'world': '#884444 italic', + }) + fragments = FormattedText([ + ('class:hello', 'Hello'), + ('class:world', 'World'), + ]) + print_formatted_text(fragments, style=style) + + If you want to print a list of Pygments tokens, wrap it in + :class:`~prompt_toolkit.formatted_text.PygmentsTokens` to do the + conversion. + + If a prompt_toolkit `Application` is currently running, this will always + print above the application or prompt (similar to `patch_stdout`). So, + `print_formatted_text` will erase the current application, print the text, + and render the application again. + + :param values: Any kind of printable object, or formatted string. + :param sep: String inserted between values, default a space. + :param end: String appended after the last value, default a newline. + :param style: :class:`.Style` instance for the color scheme. + :param include_default_pygments_style: `bool`. Include the default Pygments + style when set to `True` (the default). + """ + assert not (output and file) + + # Create Output object. + if output is None: + if file: + output = create_output(stdout=file) + else: + output = get_app_session().output + + assert isinstance(output, Output) + + # Get color depth. + color_depth = color_depth or output.get_default_color_depth() + + # Merges values. + def to_text(val: Any) -> StyleAndTextTuples: + # Normal lists which are not instances of `FormattedText` are + # considered plain text. + if isinstance(val, list) and not isinstance(val, FormattedText): + return to_formatted_text(f"{val}") + return to_formatted_text(val, auto_convert=True) + + fragments = [] + for i, value in enumerate(values): + fragments.extend(to_text(value)) + + if sep and i != len(values) - 1: + fragments.extend(to_text(sep)) + + fragments.extend(to_text(end)) + + # Print output. + def render() -> None: + assert isinstance(output, Output) + + renderer_print_formatted_text( + output, + fragments, + _create_merged_style( + style, include_default_pygments_style=include_default_pygments_style + ), + color_depth=color_depth, + style_transformation=style_transformation, + ) + + # Flush the output stream. + if flush: + output.flush() + + # If an application is running, print above the app. This does not require + # `patch_stdout`. + loop: AbstractEventLoop | None = None + + app = get_app_or_none() + if app is not None: + loop = app.loop + + if loop is not None: + loop.call_soon_threadsafe(lambda: run_in_terminal(render)) + else: + render() + + +def print_container( + container: AnyContainer, + file: TextIO | None = None, + style: BaseStyle | None = None, + include_default_pygments_style: bool = True, +) -> None: + """ + Print any layout to the output in a non-interactive way. + + Example usage:: + + from prompt_toolkit.widgets import Frame, TextArea + print_container( + Frame(TextArea(text='Hello world!'))) + """ + if file: + output = create_output(stdout=file) + else: + output = get_app_session().output + + app: Application[None] = Application( + layout=Layout(container=container), + output=output, + # `DummyInput` will cause the application to terminate immediately. + input=DummyInput(), + style=_create_merged_style( + style, include_default_pygments_style=include_default_pygments_style + ), + ) + try: + app.run(in_thread=True) + except EOFError: + pass + + +def _create_merged_style( + style: BaseStyle | None, include_default_pygments_style: bool +) -> BaseStyle: + """ + Merge user defined style with built-in style. + """ + styles = [default_ui_style()] + if include_default_pygments_style: + styles.append(default_pygments_style()) + if style: + styles.append(style) + + return merge_styles(styles) + + +def clear() -> None: + """ + Clear the screen. + """ + output = get_app_session().output + output.erase_screen() + output.cursor_goto(0, 0) + output.flush() + + +def set_title(text: str) -> None: + """ + Set the terminal title. + """ + output = get_app_session().output + output.set_title(text) + + +def clear_title() -> None: + """ + Erase the current title. + """ + set_title("") diff --git a/lib/prompt_toolkit/styles/__init__.py b/lib/prompt_toolkit/styles/__init__.py new file mode 100644 index 0000000..39e97ca --- /dev/null +++ b/lib/prompt_toolkit/styles/__init__.py @@ -0,0 +1,67 @@ +""" +Styling for prompt_toolkit applications. +""" + +from __future__ import annotations + +from .base import ( + ANSI_COLOR_NAMES, + DEFAULT_ATTRS, + Attrs, + BaseStyle, + DummyStyle, + DynamicStyle, +) +from .defaults import default_pygments_style, default_ui_style +from .named_colors import NAMED_COLORS +from .pygments import ( + pygments_token_to_classname, + style_from_pygments_cls, + style_from_pygments_dict, +) +from .style import Priority, Style, merge_styles, parse_color +from .style_transformation import ( + AdjustBrightnessStyleTransformation, + ConditionalStyleTransformation, + DummyStyleTransformation, + DynamicStyleTransformation, + ReverseStyleTransformation, + SetDefaultColorStyleTransformation, + StyleTransformation, + SwapLightAndDarkStyleTransformation, + merge_style_transformations, +) + +__all__ = [ + # Base. + "Attrs", + "DEFAULT_ATTRS", + "ANSI_COLOR_NAMES", + "BaseStyle", + "DummyStyle", + "DynamicStyle", + # Defaults. + "default_ui_style", + "default_pygments_style", + # Style. + "Style", + "Priority", + "merge_styles", + "parse_color", + # Style transformation. + "StyleTransformation", + "SwapLightAndDarkStyleTransformation", + "ReverseStyleTransformation", + "SetDefaultColorStyleTransformation", + "AdjustBrightnessStyleTransformation", + "DummyStyleTransformation", + "ConditionalStyleTransformation", + "DynamicStyleTransformation", + "merge_style_transformations", + # Pygments. + "style_from_pygments_cls", + "style_from_pygments_dict", + "pygments_token_to_classname", + # Named colors. + "NAMED_COLORS", +] diff --git a/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..37c357f Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..8e46006 Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc new file mode 100644 index 0000000..47e5547 Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc new file mode 100644 index 0000000..b4ba32a Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc new file mode 100644 index 0000000..d1d3a11 Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc new file mode 100644 index 0000000..8fe9452 Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc b/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc new file mode 100644 index 0000000..6c69bb2 Binary files /dev/null and b/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/styles/base.py b/lib/prompt_toolkit/styles/base.py new file mode 100644 index 0000000..1e04467 --- /dev/null +++ b/lib/prompt_toolkit/styles/base.py @@ -0,0 +1,188 @@ +""" +The base classes for the styling. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import Callable, Hashable, NamedTuple + +__all__ = [ + "Attrs", + "DEFAULT_ATTRS", + "ANSI_COLOR_NAMES", + "ANSI_COLOR_NAMES_ALIASES", + "BaseStyle", + "DummyStyle", + "DynamicStyle", +] + + +#: Style attributes. +class Attrs(NamedTuple): + color: str | None + bgcolor: str | None + bold: bool | None + underline: bool | None + strike: bool | None + italic: bool | None + blink: bool | None + reverse: bool | None + hidden: bool | None + dim: bool | None + + +""" +:param color: Hexadecimal string. E.g. '000000' or Ansi color name: e.g. 'ansiblue' +:param bgcolor: Hexadecimal string. E.g. 'ffffff' or Ansi color name: e.g. 'ansired' +:param bold: Boolean +:param underline: Boolean +:param strike: Boolean +:param italic: Boolean +:param blink: Boolean +:param reverse: Boolean +:param hidden: Boolean +:param dim: Boolean +""" + +#: The default `Attrs`. +DEFAULT_ATTRS = Attrs( + color="", + bgcolor="", + bold=False, + underline=False, + strike=False, + italic=False, + blink=False, + reverse=False, + hidden=False, + dim=False, +) + + +#: ``Attrs.bgcolor/fgcolor`` can be in either 'ffffff' format, or can be any of +#: the following in case we want to take colors from the 8/16 color palette. +#: Usually, in that case, the terminal application allows to configure the RGB +#: values for these names. +#: ISO 6429 colors +ANSI_COLOR_NAMES = [ + "ansidefault", + # Low intensity, dark. (One or two components 0x80, the other 0x00.) + "ansiblack", + "ansired", + "ansigreen", + "ansiyellow", + "ansiblue", + "ansimagenta", + "ansicyan", + "ansigray", + # High intensity, bright. (One or two components 0xff, the other 0x00. Not supported everywhere.) + "ansibrightblack", + "ansibrightred", + "ansibrightgreen", + "ansibrightyellow", + "ansibrightblue", + "ansibrightmagenta", + "ansibrightcyan", + "ansiwhite", +] + + +# People don't use the same ANSI color names everywhere. In prompt_toolkit 1.0 +# we used some unconventional names (which were contributed like that to +# Pygments). This is fixed now, but we still support the old names. + +# The table below maps the old aliases to the current names. +ANSI_COLOR_NAMES_ALIASES: dict[str, str] = { + "ansidarkgray": "ansibrightblack", + "ansiteal": "ansicyan", + "ansiturquoise": "ansibrightcyan", + "ansibrown": "ansiyellow", + "ansipurple": "ansimagenta", + "ansifuchsia": "ansibrightmagenta", + "ansilightgray": "ansigray", + "ansidarkred": "ansired", + "ansidarkgreen": "ansigreen", + "ansidarkblue": "ansiblue", +} +assert set(ANSI_COLOR_NAMES_ALIASES.values()).issubset(set(ANSI_COLOR_NAMES)) +assert not (set(ANSI_COLOR_NAMES_ALIASES.keys()) & set(ANSI_COLOR_NAMES)) + + +class BaseStyle(metaclass=ABCMeta): + """ + Abstract base class for prompt_toolkit styles. + """ + + @abstractmethod + def get_attrs_for_style_str( + self, style_str: str, default: Attrs = DEFAULT_ATTRS + ) -> Attrs: + """ + Return :class:`.Attrs` for the given style string. + + :param style_str: The style string. This can contain inline styling as + well as classnames (e.g. "class:title"). + :param default: `Attrs` to be used if no styling was defined. + """ + + @property + @abstractmethod + def style_rules(self) -> list[tuple[str, str]]: + """ + The list of style rules, used to create this style. + (Required for `DynamicStyle` and `_MergedStyle` to work.) + """ + return [] + + @abstractmethod + def invalidation_hash(self) -> Hashable: + """ + Invalidation hash for the style. When this changes over time, the + renderer knows that something in the style changed, and that everything + has to be redrawn. + """ + + +class DummyStyle(BaseStyle): + """ + A style that doesn't style anything. + """ + + def get_attrs_for_style_str( + self, style_str: str, default: Attrs = DEFAULT_ATTRS + ) -> Attrs: + return default + + def invalidation_hash(self) -> Hashable: + return 1 # Always the same value. + + @property + def style_rules(self) -> list[tuple[str, str]]: + return [] + + +class DynamicStyle(BaseStyle): + """ + Style class that can dynamically returns an other Style. + + :param get_style: Callable that returns a :class:`.Style` instance. + """ + + def __init__(self, get_style: Callable[[], BaseStyle | None]): + self.get_style = get_style + self._dummy = DummyStyle() + + def get_attrs_for_style_str( + self, style_str: str, default: Attrs = DEFAULT_ATTRS + ) -> Attrs: + style = self.get_style() or self._dummy + + return style.get_attrs_for_style_str(style_str, default) + + def invalidation_hash(self) -> Hashable: + return (self.get_style() or self._dummy).invalidation_hash() + + @property + def style_rules(self) -> list[tuple[str, str]]: + return (self.get_style() or self._dummy).style_rules diff --git a/lib/prompt_toolkit/styles/defaults.py b/lib/prompt_toolkit/styles/defaults.py new file mode 100644 index 0000000..2faba94 --- /dev/null +++ b/lib/prompt_toolkit/styles/defaults.py @@ -0,0 +1,236 @@ +""" +The default styling. +""" + +from __future__ import annotations + +from prompt_toolkit.cache import memoized + +from .base import ANSI_COLOR_NAMES, BaseStyle +from .named_colors import NAMED_COLORS +from .style import Style, merge_styles + +__all__ = [ + "default_ui_style", + "default_pygments_style", +] + +#: Default styling. Mapping from classnames to their style definition. +PROMPT_TOOLKIT_STYLE = [ + # Highlighting of search matches in document. + ("search", "bg:ansibrightyellow ansiblack"), + ("search.current", ""), + # Incremental search. + ("incsearch", ""), + ("incsearch.current", "reverse"), + # Highlighting of select text in document. + ("selected", "reverse"), + ("cursor-column", "bg:#dddddd"), + ("cursor-line", "underline"), + ("color-column", "bg:#ccaacc"), + # Highlighting of matching brackets. + ("matching-bracket", ""), + ("matching-bracket.other", "#000000 bg:#aacccc"), + ("matching-bracket.cursor", "#ff8888 bg:#880000"), + # Styling of other cursors, in case of block editing. + ("multiple-cursors", "#000000 bg:#ccccaa"), + # Line numbers. + ("line-number", "#888888"), + ("line-number.current", "bold"), + ("tilde", "#8888ff"), + # Default prompt. + ("prompt", ""), + ("prompt.arg", "noinherit"), + ("prompt.arg.text", ""), + ("prompt.search", "noinherit"), + ("prompt.search.text", ""), + # Search toolbar. + ("search-toolbar", "bold"), + ("search-toolbar.text", "nobold"), + # System toolbar + ("system-toolbar", "bold"), + ("system-toolbar.text", "nobold"), + # "arg" toolbar. + ("arg-toolbar", "bold"), + ("arg-toolbar.text", "nobold"), + # Validation toolbar. + ("validation-toolbar", "bg:#550000 #ffffff"), + ("window-too-small", "bg:#550000 #ffffff"), + # Completions toolbar. + ("completion-toolbar", "bg:#bbbbbb #000000"), + ("completion-toolbar.arrow", "bg:#bbbbbb #000000 bold"), + ("completion-toolbar.completion", "bg:#bbbbbb #000000"), + ("completion-toolbar.completion.current", "bg:#444444 #ffffff"), + # Completions menu. + ("completion-menu", "bg:#bbbbbb #000000"), + ("completion-menu.completion", ""), + # (Note: for the current completion, we use 'reverse' on top of fg/bg + # colors. This is to have proper rendering with NO_COLOR=1). + ("completion-menu.completion.current", "fg:#888888 bg:#ffffff reverse"), + ("completion-menu.meta.completion", "bg:#999999 #000000"), + ("completion-menu.meta.completion.current", "bg:#aaaaaa #000000"), + ("completion-menu.multi-column-meta", "bg:#aaaaaa #000000"), + # Fuzzy matches in completion menu (for FuzzyCompleter). + ("completion-menu.completion fuzzymatch.outside", "fg:#444444"), + ("completion-menu.completion fuzzymatch.inside", "bold"), + ("completion-menu.completion fuzzymatch.inside.character", "underline"), + ("completion-menu.completion.current fuzzymatch.outside", "fg:default"), + ("completion-menu.completion.current fuzzymatch.inside", "nobold"), + # Styling of readline-like completions. + ("readline-like-completions", ""), + ("readline-like-completions.completion", ""), + ("readline-like-completions.completion fuzzymatch.outside", "#888888"), + ("readline-like-completions.completion fuzzymatch.inside", ""), + ("readline-like-completions.completion fuzzymatch.inside.character", "underline"), + # Scrollbars. + ("scrollbar.background", "bg:#aaaaaa"), + ("scrollbar.button", "bg:#444444"), + ("scrollbar.arrow", "noinherit bold"), + # Start/end of scrollbars. Adding 'underline' here provides a nice little + # detail to the progress bar, but it doesn't look good on all terminals. + # ('scrollbar.start', 'underline #ffffff'), + # ('scrollbar.end', 'underline #000000'), + # Auto suggestion text. + ("auto-suggestion", "#666666"), + # Trailing whitespace and tabs. + ("trailing-whitespace", "#999999"), + ("tab", "#999999"), + # When Control-C/D has been pressed. Grayed. + ("aborting", "#888888 bg:default noreverse noitalic nounderline noblink"), + ("exiting", "#888888 bg:default noreverse noitalic nounderline noblink"), + # Entering a Vi digraph. + ("digraph", "#4444ff"), + # Control characters, like ^C, ^X. + ("control-character", "ansiblue"), + # Non-breaking space. + ("nbsp", "underline ansiyellow"), + # Default styling of HTML elements. + ("i", "italic"), + ("u", "underline"), + ("s", "strike"), + ("b", "bold"), + ("em", "italic"), + ("strong", "bold"), + ("del", "strike"), + ("hidden", "hidden"), + # It should be possible to use the style names in HTML. + # ... or .... + ("italic", "italic"), + ("underline", "underline"), + ("strike", "strike"), + ("bold", "bold"), + ("reverse", "reverse"), + ("noitalic", "noitalic"), + ("nounderline", "nounderline"), + ("nostrike", "nostrike"), + ("nobold", "nobold"), + ("noreverse", "noreverse"), + # Prompt bottom toolbar + ("bottom-toolbar", "reverse"), +] + + +# Style that will turn for instance the class 'red' into 'red'. +COLORS_STYLE = [(name, "fg:" + name) for name in ANSI_COLOR_NAMES] + [ + (name.lower(), "fg:" + name) for name in NAMED_COLORS +] + + +WIDGETS_STYLE = [ + # Dialog windows. + ("dialog", "bg:#4444ff"), + ("dialog.body", "bg:#ffffff #000000"), + ("dialog.body text-area", "bg:#cccccc"), + ("dialog.body text-area last-line", "underline"), + ("dialog frame.label", "#ff0000 bold"), + # Scrollbars in dialogs. + ("dialog.body scrollbar.background", ""), + ("dialog.body scrollbar.button", "bg:#000000"), + ("dialog.body scrollbar.arrow", ""), + ("dialog.body scrollbar.start", "nounderline"), + ("dialog.body scrollbar.end", "nounderline"), + # Buttons. + ("button", ""), + ("button.arrow", "bold"), + ("button.focused", "bg:#aa0000 #ffffff"), + # Menu bars. + ("menu-bar", "bg:#aaaaaa #000000"), + ("menu-bar.selected-item", "bg:#ffffff #000000"), + ("menu", "bg:#888888 #ffffff"), + ("menu.border", "#aaaaaa"), + ("menu.border shadow", "#444444"), + # Shadows. + ("dialog shadow", "bg:#000088"), + ("dialog.body shadow", "bg:#aaaaaa"), + ("progress-bar", "bg:#000088"), + ("progress-bar.used", "bg:#ff0000"), +] + + +# The default Pygments style, include this by default in case a Pygments lexer +# is used. +PYGMENTS_DEFAULT_STYLE = { + "pygments.whitespace": "#bbbbbb", + "pygments.comment": "italic #408080", + "pygments.comment.preproc": "noitalic #bc7a00", + "pygments.keyword": "bold #008000", + "pygments.keyword.pseudo": "nobold", + "pygments.keyword.type": "nobold #b00040", + "pygments.operator": "#666666", + "pygments.operator.word": "bold #aa22ff", + "pygments.name.builtin": "#008000", + "pygments.name.function": "#0000ff", + "pygments.name.class": "bold #0000ff", + "pygments.name.namespace": "bold #0000ff", + "pygments.name.exception": "bold #d2413a", + "pygments.name.variable": "#19177c", + "pygments.name.constant": "#880000", + "pygments.name.label": "#a0a000", + "pygments.name.entity": "bold #999999", + "pygments.name.attribute": "#7d9029", + "pygments.name.tag": "bold #008000", + "pygments.name.decorator": "#aa22ff", + # Note: In Pygments, Token.String is an alias for Token.Literal.String, + # and Token.Number as an alias for Token.Literal.Number. + "pygments.literal.string": "#ba2121", + "pygments.literal.string.doc": "italic", + "pygments.literal.string.interpol": "bold #bb6688", + "pygments.literal.string.escape": "bold #bb6622", + "pygments.literal.string.regex": "#bb6688", + "pygments.literal.string.symbol": "#19177c", + "pygments.literal.string.other": "#008000", + "pygments.literal.number": "#666666", + "pygments.generic.heading": "bold #000080", + "pygments.generic.subheading": "bold #800080", + "pygments.generic.deleted": "#a00000", + "pygments.generic.inserted": "#00a000", + "pygments.generic.error": "#ff0000", + "pygments.generic.emph": "italic", + "pygments.generic.strong": "bold", + "pygments.generic.prompt": "bold #000080", + "pygments.generic.output": "#888", + "pygments.generic.traceback": "#04d", + "pygments.error": "border:#ff0000", +} + + +@memoized() +def default_ui_style() -> BaseStyle: + """ + Create a default `Style` object. + """ + return merge_styles( + [ + Style(PROMPT_TOOLKIT_STYLE), + Style(COLORS_STYLE), + Style(WIDGETS_STYLE), + ] + ) + + +@memoized() +def default_pygments_style() -> Style: + """ + Create a `Style` object that contains the default Pygments style. + """ + return Style.from_dict(PYGMENTS_DEFAULT_STYLE) diff --git a/lib/prompt_toolkit/styles/named_colors.py b/lib/prompt_toolkit/styles/named_colors.py new file mode 100644 index 0000000..b6290a4 --- /dev/null +++ b/lib/prompt_toolkit/styles/named_colors.py @@ -0,0 +1,162 @@ +""" +All modern web browsers support these 140 color names. +Taken from: https://www.w3schools.com/colors/colors_names.asp +""" + +from __future__ import annotations + +__all__ = [ + "NAMED_COLORS", +] + + +NAMED_COLORS: dict[str, str] = { + "AliceBlue": "#f0f8ff", + "AntiqueWhite": "#faebd7", + "Aqua": "#00ffff", + "Aquamarine": "#7fffd4", + "Azure": "#f0ffff", + "Beige": "#f5f5dc", + "Bisque": "#ffe4c4", + "Black": "#000000", + "BlanchedAlmond": "#ffebcd", + "Blue": "#0000ff", + "BlueViolet": "#8a2be2", + "Brown": "#a52a2a", + "BurlyWood": "#deb887", + "CadetBlue": "#5f9ea0", + "Chartreuse": "#7fff00", + "Chocolate": "#d2691e", + "Coral": "#ff7f50", + "CornflowerBlue": "#6495ed", + "Cornsilk": "#fff8dc", + "Crimson": "#dc143c", + "Cyan": "#00ffff", + "DarkBlue": "#00008b", + "DarkCyan": "#008b8b", + "DarkGoldenRod": "#b8860b", + "DarkGray": "#a9a9a9", + "DarkGreen": "#006400", + "DarkGrey": "#a9a9a9", + "DarkKhaki": "#bdb76b", + "DarkMagenta": "#8b008b", + "DarkOliveGreen": "#556b2f", + "DarkOrange": "#ff8c00", + "DarkOrchid": "#9932cc", + "DarkRed": "#8b0000", + "DarkSalmon": "#e9967a", + "DarkSeaGreen": "#8fbc8f", + "DarkSlateBlue": "#483d8b", + "DarkSlateGray": "#2f4f4f", + "DarkSlateGrey": "#2f4f4f", + "DarkTurquoise": "#00ced1", + "DarkViolet": "#9400d3", + "DeepPink": "#ff1493", + "DeepSkyBlue": "#00bfff", + "DimGray": "#696969", + "DimGrey": "#696969", + "DodgerBlue": "#1e90ff", + "FireBrick": "#b22222", + "FloralWhite": "#fffaf0", + "ForestGreen": "#228b22", + "Fuchsia": "#ff00ff", + "Gainsboro": "#dcdcdc", + "GhostWhite": "#f8f8ff", + "Gold": "#ffd700", + "GoldenRod": "#daa520", + "Gray": "#808080", + "Green": "#008000", + "GreenYellow": "#adff2f", + "Grey": "#808080", + "HoneyDew": "#f0fff0", + "HotPink": "#ff69b4", + "IndianRed": "#cd5c5c", + "Indigo": "#4b0082", + "Ivory": "#fffff0", + "Khaki": "#f0e68c", + "Lavender": "#e6e6fa", + "LavenderBlush": "#fff0f5", + "LawnGreen": "#7cfc00", + "LemonChiffon": "#fffacd", + "LightBlue": "#add8e6", + "LightCoral": "#f08080", + "LightCyan": "#e0ffff", + "LightGoldenRodYellow": "#fafad2", + "LightGray": "#d3d3d3", + "LightGreen": "#90ee90", + "LightGrey": "#d3d3d3", + "LightPink": "#ffb6c1", + "LightSalmon": "#ffa07a", + "LightSeaGreen": "#20b2aa", + "LightSkyBlue": "#87cefa", + "LightSlateGray": "#778899", + "LightSlateGrey": "#778899", + "LightSteelBlue": "#b0c4de", + "LightYellow": "#ffffe0", + "Lime": "#00ff00", + "LimeGreen": "#32cd32", + "Linen": "#faf0e6", + "Magenta": "#ff00ff", + "Maroon": "#800000", + "MediumAquaMarine": "#66cdaa", + "MediumBlue": "#0000cd", + "MediumOrchid": "#ba55d3", + "MediumPurple": "#9370db", + "MediumSeaGreen": "#3cb371", + "MediumSlateBlue": "#7b68ee", + "MediumSpringGreen": "#00fa9a", + "MediumTurquoise": "#48d1cc", + "MediumVioletRed": "#c71585", + "MidnightBlue": "#191970", + "MintCream": "#f5fffa", + "MistyRose": "#ffe4e1", + "Moccasin": "#ffe4b5", + "NavajoWhite": "#ffdead", + "Navy": "#000080", + "OldLace": "#fdf5e6", + "Olive": "#808000", + "OliveDrab": "#6b8e23", + "Orange": "#ffa500", + "OrangeRed": "#ff4500", + "Orchid": "#da70d6", + "PaleGoldenRod": "#eee8aa", + "PaleGreen": "#98fb98", + "PaleTurquoise": "#afeeee", + "PaleVioletRed": "#db7093", + "PapayaWhip": "#ffefd5", + "PeachPuff": "#ffdab9", + "Peru": "#cd853f", + "Pink": "#ffc0cb", + "Plum": "#dda0dd", + "PowderBlue": "#b0e0e6", + "Purple": "#800080", + "RebeccaPurple": "#663399", + "Red": "#ff0000", + "RosyBrown": "#bc8f8f", + "RoyalBlue": "#4169e1", + "SaddleBrown": "#8b4513", + "Salmon": "#fa8072", + "SandyBrown": "#f4a460", + "SeaGreen": "#2e8b57", + "SeaShell": "#fff5ee", + "Sienna": "#a0522d", + "Silver": "#c0c0c0", + "SkyBlue": "#87ceeb", + "SlateBlue": "#6a5acd", + "SlateGray": "#708090", + "SlateGrey": "#708090", + "Snow": "#fffafa", + "SpringGreen": "#00ff7f", + "SteelBlue": "#4682b4", + "Tan": "#d2b48c", + "Teal": "#008080", + "Thistle": "#d8bfd8", + "Tomato": "#ff6347", + "Turquoise": "#40e0d0", + "Violet": "#ee82ee", + "Wheat": "#f5deb3", + "White": "#ffffff", + "WhiteSmoke": "#f5f5f5", + "Yellow": "#ffff00", + "YellowGreen": "#9acd32", +} diff --git a/lib/prompt_toolkit/styles/pygments.py b/lib/prompt_toolkit/styles/pygments.py new file mode 100644 index 0000000..c0f6031 --- /dev/null +++ b/lib/prompt_toolkit/styles/pygments.py @@ -0,0 +1,70 @@ +""" +Adaptor for building prompt_toolkit styles, starting from a Pygments style. + +Usage:: + + from pygments.styles.tango import TangoStyle + style = style_from_pygments_cls(pygments_style_cls=TangoStyle) +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .style import Style + +if TYPE_CHECKING: + from pygments.style import Style as PygmentsStyle + from pygments.token import Token + + +__all__ = [ + "style_from_pygments_cls", + "style_from_pygments_dict", + "pygments_token_to_classname", +] + + +def style_from_pygments_cls(pygments_style_cls: type[PygmentsStyle]) -> Style: + """ + Shortcut to create a :class:`.Style` instance from a Pygments style class + and a style dictionary. + + Example:: + + from prompt_toolkit.styles.from_pygments import style_from_pygments_cls + from pygments.styles import get_style_by_name + style = style_from_pygments_cls(get_style_by_name('monokai')) + + :param pygments_style_cls: Pygments style class to start from. + """ + # Import inline. + from pygments.style import Style as PygmentsStyle + + assert issubclass(pygments_style_cls, PygmentsStyle) + + return style_from_pygments_dict(pygments_style_cls.styles) + + +def style_from_pygments_dict(pygments_dict: dict[Token, str]) -> Style: + """ + Create a :class:`.Style` instance from a Pygments style dictionary. + (One that maps Token objects to style strings.) + """ + pygments_style = [] + + for token, style in pygments_dict.items(): + pygments_style.append((pygments_token_to_classname(token), style)) + + return Style(pygments_style) + + +def pygments_token_to_classname(token: Token) -> str: + """ + Turn e.g. `Token.Name.Exception` into `'pygments.name.exception'`. + + (Our Pygments lexer will also turn the tokens that pygments produces in a + prompt_toolkit list of fragments that match these styling rules.) + """ + parts = ("pygments",) + token + return ".".join(parts).lower() diff --git a/lib/prompt_toolkit/styles/style.py b/lib/prompt_toolkit/styles/style.py new file mode 100644 index 0000000..8aab8e1 --- /dev/null +++ b/lib/prompt_toolkit/styles/style.py @@ -0,0 +1,407 @@ +""" +Tool for creating styles from a dictionary. +""" + +from __future__ import annotations + +import itertools +import re +from enum import Enum +from typing import Hashable, TypeVar + +from prompt_toolkit.cache import SimpleCache + +from .base import ( + ANSI_COLOR_NAMES, + ANSI_COLOR_NAMES_ALIASES, + DEFAULT_ATTRS, + Attrs, + BaseStyle, +) +from .named_colors import NAMED_COLORS + +__all__ = [ + "Style", + "parse_color", + "Priority", + "merge_styles", +] + +_named_colors_lowercase = {k.lower(): v.lstrip("#") for k, v in NAMED_COLORS.items()} + + +def parse_color(text: str) -> str: + """ + Parse/validate color format. + + Like in Pygments, but also support the ANSI color names. + (These will map to the colors of the 16 color palette.) + """ + # ANSI color names. + if text in ANSI_COLOR_NAMES: + return text + if text in ANSI_COLOR_NAMES_ALIASES: + return ANSI_COLOR_NAMES_ALIASES[text] + + # 140 named colors. + try: + # Replace by 'hex' value. + return _named_colors_lowercase[text.lower()] + except KeyError: + pass + + # Hex codes. + if text[0:1] == "#": + col = text[1:] + + # Keep this for backwards-compatibility (Pygments does it). + # I don't like the '#' prefix for named colors. + if col in ANSI_COLOR_NAMES: + return col + elif col in ANSI_COLOR_NAMES_ALIASES: + return ANSI_COLOR_NAMES_ALIASES[col] + + # 6 digit hex color. + elif len(col) == 6: + return col + + # 3 digit hex color. + elif len(col) == 3: + return col[0] * 2 + col[1] * 2 + col[2] * 2 + + # Default. + elif text in ("", "default"): + return text + + raise ValueError(f"Wrong color format {text!r}") + + +# Attributes, when they are not filled in by a style. None means that we take +# the value from the parent. +_EMPTY_ATTRS = Attrs( + color=None, + bgcolor=None, + bold=None, + underline=None, + strike=None, + italic=None, + blink=None, + reverse=None, + hidden=None, + dim=None, +) + + +def _expand_classname(classname: str) -> list[str]: + """ + Split a single class name at the `.` operator, and build a list of classes. + + E.g. 'a.b.c' becomes ['a', 'a.b', 'a.b.c'] + """ + result = [] + parts = classname.split(".") + + for i in range(1, len(parts) + 1): + result.append(".".join(parts[:i]).lower()) + + return result + + +def _parse_style_str(style_str: str) -> Attrs: + """ + Take a style string, e.g. 'bg:red #88ff00 class:title' + and return a `Attrs` instance. + """ + # Start from default Attrs. + if "noinherit" in style_str: + attrs = DEFAULT_ATTRS + else: + attrs = _EMPTY_ATTRS + + # Now update with the given attributes. + for part in style_str.split(): + if part == "noinherit": + pass + elif part == "bold": + attrs = attrs._replace(bold=True) + elif part == "nobold": + attrs = attrs._replace(bold=False) + elif part == "italic": + attrs = attrs._replace(italic=True) + elif part == "noitalic": + attrs = attrs._replace(italic=False) + elif part == "underline": + attrs = attrs._replace(underline=True) + elif part == "nounderline": + attrs = attrs._replace(underline=False) + elif part == "strike": + attrs = attrs._replace(strike=True) + elif part == "nostrike": + attrs = attrs._replace(strike=False) + + # prompt_toolkit extensions. Not in Pygments. + elif part == "blink": + attrs = attrs._replace(blink=True) + elif part == "noblink": + attrs = attrs._replace(blink=False) + elif part == "reverse": + attrs = attrs._replace(reverse=True) + elif part == "noreverse": + attrs = attrs._replace(reverse=False) + elif part == "hidden": + attrs = attrs._replace(hidden=True) + elif part == "nohidden": + attrs = attrs._replace(hidden=False) + elif part == "dim": + attrs = attrs._replace(dim=True) + elif part == "nodim": + attrs = attrs._replace(dim=False) + + # Pygments properties that we ignore. + elif part in ("roman", "sans", "mono"): + pass + elif part.startswith("border:"): + pass + + # Ignore pieces in between square brackets. This is internal stuff. + # Like '[transparent]' or '[set-cursor-position]'. + elif part.startswith("[") and part.endswith("]"): + pass + + # Colors. + elif part.startswith("bg:"): + attrs = attrs._replace(bgcolor=parse_color(part[3:])) + elif part.startswith("fg:"): # The 'fg:' prefix is optional. + attrs = attrs._replace(color=parse_color(part[3:])) + else: + attrs = attrs._replace(color=parse_color(part)) + + return attrs + + +CLASS_NAMES_RE = re.compile(r"^[a-z0-9.\s_-]*$") # This one can't contain a comma! + + +class Priority(Enum): + """ + The priority of the rules, when a style is created from a dictionary. + + In a `Style`, rules that are defined later will always override previous + defined rules, however in a dictionary, the key order was arbitrary before + Python 3.6. This means that the style could change at random between rules. + + We have two options: + + - `DICT_KEY_ORDER`: This means, iterate through the dictionary, and take + the key/value pairs in order as they come. This is a good option if you + have Python >3.6. Rules at the end will override rules at the beginning. + - `MOST_PRECISE`: keys that are defined with most precision will get higher + priority. (More precise means: more elements.) + """ + + DICT_KEY_ORDER = "KEY_ORDER" + MOST_PRECISE = "MOST_PRECISE" + + +# We don't support Python versions older than 3.6 anymore, so we can always +# depend on dictionary ordering. This is the default. +default_priority = Priority.DICT_KEY_ORDER + + +class Style(BaseStyle): + """ + Create a ``Style`` instance from a list of style rules. + + The `style_rules` is supposed to be a list of ('classnames', 'style') tuples. + The classnames are a whitespace separated string of class names and the + style string is just like a Pygments style definition, but with a few + additions: it supports 'reverse' and 'blink'. + + Later rules always override previous rules. + + Usage:: + + Style([ + ('title', '#ff0000 bold underline'), + ('something-else', 'reverse'), + ('class1 class2', 'reverse'), + ]) + + The ``from_dict`` classmethod is similar, but takes a dictionary as input. + """ + + def __init__(self, style_rules: list[tuple[str, str]]) -> None: + class_names_and_attrs = [] + + # Loop through the rules in the order they were defined. + # Rules that are defined later get priority. + for class_names, style_str in style_rules: + assert CLASS_NAMES_RE.match(class_names), repr(class_names) + + # The order of the class names doesn't matter. + # (But the order of rules does matter.) + class_names_set = frozenset(class_names.lower().split()) + attrs = _parse_style_str(style_str) + + class_names_and_attrs.append((class_names_set, attrs)) + + self._style_rules = style_rules + self.class_names_and_attrs = class_names_and_attrs + + @property + def style_rules(self) -> list[tuple[str, str]]: + return self._style_rules + + @classmethod + def from_dict( + cls, style_dict: dict[str, str], priority: Priority = default_priority + ) -> Style: + """ + :param style_dict: Style dictionary. + :param priority: `Priority` value. + """ + if priority == Priority.MOST_PRECISE: + + def key(item: tuple[str, str]) -> int: + # Split on '.' and whitespace. Count elements. + return sum(len(i.split(".")) for i in item[0].split()) + + return cls(sorted(style_dict.items(), key=key)) + else: + return cls(list(style_dict.items())) + + def get_attrs_for_style_str( + self, style_str: str, default: Attrs = DEFAULT_ATTRS + ) -> Attrs: + """ + Get `Attrs` for the given style string. + """ + list_of_attrs = [default] + class_names: set[str] = set() + + # Apply default styling. + for names, attr in self.class_names_and_attrs: + if not names: + list_of_attrs.append(attr) + + # Go from left to right through the style string. Things on the right + # take precedence. + for part in style_str.split(): + # This part represents a class. + # Do lookup of this class name in the style definition, as well + # as all class combinations that we have so far. + if part.startswith("class:"): + # Expand all class names (comma separated list). + new_class_names = [] + for p in part[6:].lower().split(","): + new_class_names.extend(_expand_classname(p)) + + for new_name in new_class_names: + # Build a set of all possible class combinations to be applied. + combos = set() + combos.add(frozenset([new_name])) + + for count in range(1, len(class_names) + 1): + for c2 in itertools.combinations(class_names, count): + combos.add(frozenset(c2 + (new_name,))) + + # Apply the styles that match these class names. + for names, attr in self.class_names_and_attrs: + if names in combos: + list_of_attrs.append(attr) + + class_names.add(new_name) + + # Process inline style. + else: + inline_attrs = _parse_style_str(part) + list_of_attrs.append(inline_attrs) + + return _merge_attrs(list_of_attrs) + + def invalidation_hash(self) -> Hashable: + return id(self.class_names_and_attrs) + + +_T = TypeVar("_T") + + +def _merge_attrs(list_of_attrs: list[Attrs]) -> Attrs: + """ + Take a list of :class:`.Attrs` instances and merge them into one. + Every `Attr` in the list can override the styling of the previous one. So, + the last one has highest priority. + """ + + def _or(*values: _T) -> _T: + "Take first not-None value, starting at the end." + for v in values[::-1]: + if v is not None: + return v + raise ValueError # Should not happen, there's always one non-null value. + + return Attrs( + color=_or("", *[a.color for a in list_of_attrs]), + bgcolor=_or("", *[a.bgcolor for a in list_of_attrs]), + bold=_or(False, *[a.bold for a in list_of_attrs]), + underline=_or(False, *[a.underline for a in list_of_attrs]), + strike=_or(False, *[a.strike for a in list_of_attrs]), + italic=_or(False, *[a.italic for a in list_of_attrs]), + blink=_or(False, *[a.blink for a in list_of_attrs]), + reverse=_or(False, *[a.reverse for a in list_of_attrs]), + hidden=_or(False, *[a.hidden for a in list_of_attrs]), + dim=_or(False, *[a.dim for a in list_of_attrs]), + ) + + +def merge_styles(styles: list[BaseStyle]) -> _MergedStyle: + """ + Merge multiple `Style` objects. + """ + styles = [s for s in styles if s is not None] + return _MergedStyle(styles) + + +class _MergedStyle(BaseStyle): + """ + Merge multiple `Style` objects into one. + This is supposed to ensure consistency: if any of the given styles changes, + then this style will be updated. + """ + + # NOTE: previously, we used an algorithm where we did not generate the + # combined style. Instead this was a proxy that called one style + # after the other, passing the outcome of the previous style as the + # default for the next one. This did not work, because that way, the + # priorities like described in the `Style` class don't work. + # 'class:aborted' was for instance never displayed in gray, because + # the next style specified a default color for any text. (The + # explicit styling of class:aborted should have taken priority, + # because it was more precise.) + def __init__(self, styles: list[BaseStyle]) -> None: + self.styles = styles + self._style: SimpleCache[Hashable, Style] = SimpleCache(maxsize=1) + + @property + def _merged_style(self) -> Style: + "The `Style` object that has the other styles merged together." + + def get() -> Style: + return Style(self.style_rules) + + return self._style.get(self.invalidation_hash(), get) + + @property + def style_rules(self) -> list[tuple[str, str]]: + style_rules = [] + for s in self.styles: + style_rules.extend(s.style_rules) + return style_rules + + def get_attrs_for_style_str( + self, style_str: str, default: Attrs = DEFAULT_ATTRS + ) -> Attrs: + return self._merged_style.get_attrs_for_style_str(style_str, default) + + def invalidation_hash(self) -> Hashable: + return tuple(s.invalidation_hash() for s in self.styles) diff --git a/lib/prompt_toolkit/styles/style_transformation.py b/lib/prompt_toolkit/styles/style_transformation.py new file mode 100644 index 0000000..e8d5b0a --- /dev/null +++ b/lib/prompt_toolkit/styles/style_transformation.py @@ -0,0 +1,374 @@ +""" +Collection of style transformations. + +Think of it as a kind of color post processing after the rendering is done. +This could be used for instance to change the contrast/saturation; swap light +and dark colors or even change certain colors for other colors. + +When the UI is rendered, these transformations can be applied right after the +style strings are turned into `Attrs` objects that represent the actual +formatting. +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from colorsys import hls_to_rgb, rgb_to_hls +from typing import Callable, Hashable, Sequence + +from prompt_toolkit.cache import memoized +from prompt_toolkit.filters import FilterOrBool, to_filter +from prompt_toolkit.utils import AnyFloat, to_float, to_str + +from .base import ANSI_COLOR_NAMES, Attrs +from .style import parse_color + +__all__ = [ + "StyleTransformation", + "SwapLightAndDarkStyleTransformation", + "ReverseStyleTransformation", + "SetDefaultColorStyleTransformation", + "AdjustBrightnessStyleTransformation", + "DummyStyleTransformation", + "ConditionalStyleTransformation", + "DynamicStyleTransformation", + "merge_style_transformations", +] + + +class StyleTransformation(metaclass=ABCMeta): + """ + Base class for any style transformation. + """ + + @abstractmethod + def transform_attrs(self, attrs: Attrs) -> Attrs: + """ + Take an `Attrs` object and return a new `Attrs` object. + + Remember that the color formats can be either "ansi..." or a 6 digit + lowercase hexadecimal color (without '#' prefix). + """ + + def invalidation_hash(self) -> Hashable: + """ + When this changes, the cache should be invalidated. + """ + return f"{self.__class__.__name__}-{id(self)}" + + +class SwapLightAndDarkStyleTransformation(StyleTransformation): + """ + Turn dark colors into light colors and the other way around. + + This is meant to make color schemes that work on a dark background usable + on a light background (and the other way around). + + Notice that this doesn't swap foreground and background like "reverse" + does. It turns light green into dark green and the other way around. + Foreground and background colors are considered individually. + + Also notice that when is used somewhere and no colors are given + in particular (like what is the default for the bottom toolbar), then this + doesn't change anything. This is what makes sense, because when the + 'default' color is chosen, it's what works best for the terminal, and + reverse works good with that. + """ + + def transform_attrs(self, attrs: Attrs) -> Attrs: + """ + Return the `Attrs` used when opposite luminosity should be used. + """ + # Reverse colors. + attrs = attrs._replace(color=get_opposite_color(attrs.color)) + attrs = attrs._replace(bgcolor=get_opposite_color(attrs.bgcolor)) + + return attrs + + +class ReverseStyleTransformation(StyleTransformation): + """ + Swap the 'reverse' attribute. + + (This is still experimental.) + """ + + def transform_attrs(self, attrs: Attrs) -> Attrs: + return attrs._replace(reverse=not attrs.reverse) + + +class SetDefaultColorStyleTransformation(StyleTransformation): + """ + Set default foreground/background color for output that doesn't specify + anything. This is useful for overriding the terminal default colors. + + :param fg: Color string or callable that returns a color string for the + foreground. + :param bg: Like `fg`, but for the background. + """ + + def __init__( + self, fg: str | Callable[[], str], bg: str | Callable[[], str] + ) -> None: + self.fg = fg + self.bg = bg + + def transform_attrs(self, attrs: Attrs) -> Attrs: + if attrs.bgcolor in ("", "default"): + attrs = attrs._replace(bgcolor=parse_color(to_str(self.bg))) + + if attrs.color in ("", "default"): + attrs = attrs._replace(color=parse_color(to_str(self.fg))) + + return attrs + + def invalidation_hash(self) -> Hashable: + return ( + "set-default-color", + to_str(self.fg), + to_str(self.bg), + ) + + +class AdjustBrightnessStyleTransformation(StyleTransformation): + """ + Adjust the brightness to improve the rendering on either dark or light + backgrounds. + + For dark backgrounds, it's best to increase `min_brightness`. For light + backgrounds it's best to decrease `max_brightness`. Usually, only one + setting is adjusted. + + This will only change the brightness for text that has a foreground color + defined, but no background color. It works best for 256 or true color + output. + + .. note:: Notice that there is no universal way to detect whether the + application is running in a light or dark terminal. As a + developer of an command line application, you'll have to make + this configurable for the user. + + :param min_brightness: Float between 0.0 and 1.0 or a callable that returns + a float. + :param max_brightness: Float between 0.0 and 1.0 or a callable that returns + a float. + """ + + def __init__( + self, min_brightness: AnyFloat = 0.0, max_brightness: AnyFloat = 1.0 + ) -> None: + self.min_brightness = min_brightness + self.max_brightness = max_brightness + + def transform_attrs(self, attrs: Attrs) -> Attrs: + min_brightness = to_float(self.min_brightness) + max_brightness = to_float(self.max_brightness) + assert 0 <= min_brightness <= 1 + assert 0 <= max_brightness <= 1 + + # Don't do anything if the whole brightness range is acceptable. + # This also avoids turning ansi colors into RGB sequences. + if min_brightness == 0.0 and max_brightness == 1.0: + return attrs + + # If a foreground color is given without a background color. + no_background = not attrs.bgcolor or attrs.bgcolor == "default" + has_fgcolor = attrs.color and attrs.color != "ansidefault" + + if has_fgcolor and no_background: + # Calculate new RGB values. + r, g, b = self._color_to_rgb(attrs.color or "") + hue, brightness, saturation = rgb_to_hls(r, g, b) + brightness = self._interpolate_brightness( + brightness, min_brightness, max_brightness + ) + r, g, b = hls_to_rgb(hue, brightness, saturation) + new_color = f"{int(r * 255):02x}{int(g * 255):02x}{int(b * 255):02x}" + + attrs = attrs._replace(color=new_color) + + return attrs + + def _color_to_rgb(self, color: str) -> tuple[float, float, float]: + """ + Parse `style.Attrs` color into RGB tuple. + """ + # Do RGB lookup for ANSI colors. + try: + from prompt_toolkit.output.vt100 import ANSI_COLORS_TO_RGB + + r, g, b = ANSI_COLORS_TO_RGB[color] + return r / 255.0, g / 255.0, b / 255.0 + except KeyError: + pass + + # Parse RRGGBB format. + return ( + int(color[0:2], 16) / 255.0, + int(color[2:4], 16) / 255.0, + int(color[4:6], 16) / 255.0, + ) + + # NOTE: we don't have to support named colors here. They are already + # transformed into RGB values in `style.parse_color`. + + def _interpolate_brightness( + self, value: float, min_brightness: float, max_brightness: float + ) -> float: + """ + Map the brightness to the (min_brightness..max_brightness) range. + """ + return min_brightness + (max_brightness - min_brightness) * value + + def invalidation_hash(self) -> Hashable: + return ( + "adjust-brightness", + to_float(self.min_brightness), + to_float(self.max_brightness), + ) + + +class DummyStyleTransformation(StyleTransformation): + """ + Don't transform anything at all. + """ + + def transform_attrs(self, attrs: Attrs) -> Attrs: + return attrs + + def invalidation_hash(self) -> Hashable: + # Always return the same hash for these dummy instances. + return "dummy-style-transformation" + + +class DynamicStyleTransformation(StyleTransformation): + """ + StyleTransformation class that can dynamically returns any + `StyleTransformation`. + + :param get_style_transformation: Callable that returns a + :class:`.StyleTransformation` instance. + """ + + def __init__( + self, get_style_transformation: Callable[[], StyleTransformation | None] + ) -> None: + self.get_style_transformation = get_style_transformation + + def transform_attrs(self, attrs: Attrs) -> Attrs: + style_transformation = ( + self.get_style_transformation() or DummyStyleTransformation() + ) + return style_transformation.transform_attrs(attrs) + + def invalidation_hash(self) -> Hashable: + style_transformation = ( + self.get_style_transformation() or DummyStyleTransformation() + ) + return style_transformation.invalidation_hash() + + +class ConditionalStyleTransformation(StyleTransformation): + """ + Apply the style transformation depending on a condition. + """ + + def __init__( + self, style_transformation: StyleTransformation, filter: FilterOrBool + ) -> None: + self.style_transformation = style_transformation + self.filter = to_filter(filter) + + def transform_attrs(self, attrs: Attrs) -> Attrs: + if self.filter(): + return self.style_transformation.transform_attrs(attrs) + return attrs + + def invalidation_hash(self) -> Hashable: + return (self.filter(), self.style_transformation.invalidation_hash()) + + +class _MergedStyleTransformation(StyleTransformation): + def __init__(self, style_transformations: Sequence[StyleTransformation]) -> None: + self.style_transformations = style_transformations + + def transform_attrs(self, attrs: Attrs) -> Attrs: + for transformation in self.style_transformations: + attrs = transformation.transform_attrs(attrs) + return attrs + + def invalidation_hash(self) -> Hashable: + return tuple(t.invalidation_hash() for t in self.style_transformations) + + +def merge_style_transformations( + style_transformations: Sequence[StyleTransformation], +) -> StyleTransformation: + """ + Merge multiple transformations together. + """ + return _MergedStyleTransformation(style_transformations) + + +# Dictionary that maps ANSI color names to their opposite. This is useful for +# turning color schemes that are optimized for a black background usable for a +# white background. +OPPOSITE_ANSI_COLOR_NAMES = { + "ansidefault": "ansidefault", + "ansiblack": "ansiwhite", + "ansired": "ansibrightred", + "ansigreen": "ansibrightgreen", + "ansiyellow": "ansibrightyellow", + "ansiblue": "ansibrightblue", + "ansimagenta": "ansibrightmagenta", + "ansicyan": "ansibrightcyan", + "ansigray": "ansibrightblack", + "ansiwhite": "ansiblack", + "ansibrightred": "ansired", + "ansibrightgreen": "ansigreen", + "ansibrightyellow": "ansiyellow", + "ansibrightblue": "ansiblue", + "ansibrightmagenta": "ansimagenta", + "ansibrightcyan": "ansicyan", + "ansibrightblack": "ansigray", +} +assert set(OPPOSITE_ANSI_COLOR_NAMES.keys()) == set(ANSI_COLOR_NAMES) +assert set(OPPOSITE_ANSI_COLOR_NAMES.values()) == set(ANSI_COLOR_NAMES) + + +@memoized() +def get_opposite_color(colorname: str | None) -> str | None: + """ + Take a color name in either 'ansi...' format or 6 digit RGB, return the + color of opposite luminosity (same hue/saturation). + + This is used for turning color schemes that work on a light background + usable on a dark background. + """ + if colorname is None: # Because color/bgcolor can be None in `Attrs`. + return None + + # Special values. + if colorname in ("", "default"): + return colorname + + # Try ANSI color names. + try: + return OPPOSITE_ANSI_COLOR_NAMES[colorname] + except KeyError: + # Try 6 digit RGB colors. + r = int(colorname[:2], 16) / 255.0 + g = int(colorname[2:4], 16) / 255.0 + b = int(colorname[4:6], 16) / 255.0 + + h, l, s = rgb_to_hls(r, g, b) + + l = 1 - l + + r, g, b = hls_to_rgb(h, l, s) + + r = int(r * 255) + g = int(g * 255) + b = int(b * 255) + + return f"{r:02x}{g:02x}{b:02x}" diff --git a/lib/prompt_toolkit/token.py b/lib/prompt_toolkit/token.py new file mode 100644 index 0000000..e97893d --- /dev/null +++ b/lib/prompt_toolkit/token.py @@ -0,0 +1,9 @@ +""" """ + +from __future__ import annotations + +__all__ = [ + "ZeroWidthEscape", +] + +ZeroWidthEscape = "[ZeroWidthEscape]" diff --git a/lib/prompt_toolkit/utils.py b/lib/prompt_toolkit/utils.py new file mode 100644 index 0000000..1a99a28 --- /dev/null +++ b/lib/prompt_toolkit/utils.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import os +import signal +import sys +import threading +from collections import deque +from typing import ( + Callable, + ContextManager, + Dict, + Generator, + Generic, + TypeVar, + Union, +) + +from wcwidth import wcwidth + +__all__ = [ + "Event", + "DummyContext", + "get_cwidth", + "suspend_to_background_supported", + "is_conemu_ansi", + "is_windows", + "in_main_thread", + "get_bell_environment_variable", + "get_term_environment_variable", + "take_using_weights", + "to_str", + "to_int", + "AnyFloat", + "to_float", + "is_dumb_terminal", +] + +# Used to ensure sphinx autodoc does not try to import platform-specific +# stuff when documenting win32.py modules. +SPHINX_AUTODOC_RUNNING = "sphinx.ext.autodoc" in sys.modules + +_Sender = TypeVar("_Sender", covariant=True) + + +class Event(Generic[_Sender]): + """ + Simple event to which event handlers can be attached. For instance:: + + class Cls: + def __init__(self): + # Define event. The first parameter is the sender. + self.event = Event(self) + + obj = Cls() + + def handler(sender): + pass + + # Add event handler by using the += operator. + obj.event += handler + + # Fire event. + obj.event() + """ + + def __init__( + self, sender: _Sender, handler: Callable[[_Sender], None] | None = None + ) -> None: + self.sender = sender + self._handlers: list[Callable[[_Sender], None]] = [] + + if handler is not None: + self += handler + + def __call__(self) -> None: + "Fire event." + for handler in self._handlers: + handler(self.sender) + + def fire(self) -> None: + "Alias for just calling the event." + self() + + def add_handler(self, handler: Callable[[_Sender], None]) -> None: + """ + Add another handler to this callback. + (Handler should be a callable that takes exactly one parameter: the + sender object.) + """ + # Add to list of event handlers. + self._handlers.append(handler) + + def remove_handler(self, handler: Callable[[_Sender], None]) -> None: + """ + Remove a handler from this callback. + """ + if handler in self._handlers: + self._handlers.remove(handler) + + def __iadd__(self, handler: Callable[[_Sender], None]) -> Event[_Sender]: + """ + `event += handler` notation for adding a handler. + """ + self.add_handler(handler) + return self + + def __isub__(self, handler: Callable[[_Sender], None]) -> Event[_Sender]: + """ + `event -= handler` notation for removing a handler. + """ + self.remove_handler(handler) + return self + + +class DummyContext(ContextManager[None]): + """ + (contextlib.nested is not available on Py3) + """ + + def __enter__(self) -> None: + pass + + def __exit__(self, *a: object) -> None: + pass + + +class _CharSizesCache(Dict[str, int]): + """ + Cache for wcwidth sizes. + """ + + LONG_STRING_MIN_LEN = 64 # Minimum string length for considering it long. + MAX_LONG_STRINGS = 16 # Maximum number of long strings to remember. + + def __init__(self) -> None: + super().__init__() + # Keep track of the "long" strings in this cache. + self._long_strings: deque[str] = deque() + + def __missing__(self, string: str) -> int: + # Note: We use the `max(0, ...` because some non printable control + # characters, like e.g. Ctrl-underscore get a -1 wcwidth value. + # It can be possible that these characters end up in the input + # text. + result: int + if len(string) == 1: + result = max(0, wcwidth(string)) + else: + result = sum(self[c] for c in string) + + # Store in cache. + self[string] = result + + # Rotate long strings. + # (It's hard to tell what we can consider short...) + if len(string) > self.LONG_STRING_MIN_LEN: + long_strings = self._long_strings + long_strings.append(string) + + if len(long_strings) > self.MAX_LONG_STRINGS: + key_to_remove = long_strings.popleft() + if key_to_remove in self: + del self[key_to_remove] + + return result + + +_CHAR_SIZES_CACHE = _CharSizesCache() + + +def get_cwidth(string: str) -> int: + """ + Return width of a string. Wrapper around ``wcwidth``. + """ + return _CHAR_SIZES_CACHE[string] + + +def suspend_to_background_supported() -> bool: + """ + Returns `True` when the Python implementation supports + suspend-to-background. This is typically `False' on Windows systems. + """ + return hasattr(signal, "SIGTSTP") + + +def is_windows() -> bool: + """ + True when we are using Windows. + """ + return sys.platform == "win32" # Not 'darwin' or 'linux2' + + +def is_windows_vt100_supported() -> bool: + """ + True when we are using Windows, but VT100 escape sequences are supported. + """ + if sys.platform == "win32": + # Import needs to be inline. Windows libraries are not always available. + from prompt_toolkit.output.windows10 import is_win_vt100_enabled + + return is_win_vt100_enabled() + + return False + + +def is_conemu_ansi() -> bool: + """ + True when the ConEmu Windows console is used. + """ + return sys.platform == "win32" and os.environ.get("ConEmuANSI", "OFF") == "ON" + + +def in_main_thread() -> bool: + """ + True when the current thread is the main thread. + """ + return threading.current_thread().__class__.__name__ == "_MainThread" + + +def get_bell_environment_variable() -> bool: + """ + True if env variable is set to true (true, TRUE, True, 1). + """ + value = os.environ.get("PROMPT_TOOLKIT_BELL", "true") + return value.lower() in ("1", "true") + + +def get_term_environment_variable() -> str: + "Return the $TERM environment variable." + return os.environ.get("TERM", "") + + +_T = TypeVar("_T") + + +def take_using_weights( + items: list[_T], weights: list[int] +) -> Generator[_T, None, None]: + """ + Generator that keeps yielding items from the items list, in proportion to + their weight. For instance:: + + # Getting the first 70 items from this generator should have yielded 10 + # times A, 20 times B and 40 times C, all distributed equally.. + take_using_weights(['A', 'B', 'C'], [5, 10, 20]) + + :param items: List of items to take from. + :param weights: Integers representing the weight. (Numbers have to be + integers, not floats.) + """ + assert len(items) == len(weights) + assert len(items) > 0 + + # Remove items with zero-weight. + items2 = [] + weights2 = [] + for item, w in zip(items, weights): + if w > 0: + items2.append(item) + weights2.append(w) + + items = items2 + weights = weights2 + + # Make sure that we have some items left. + if not items: + raise ValueError("Did't got any items with a positive weight.") + + # + already_taken = [0 for i in items] + item_count = len(items) + max_weight = max(weights) + + i = 0 + while True: + # Each iteration of this loop, we fill up until by (total_weight/max_weight). + adding = True + while adding: + adding = False + + for item_i, item, weight in zip(range(item_count), items, weights): + if already_taken[item_i] < i * weight / float(max_weight): + yield item + already_taken[item_i] += 1 + adding = True + + i += 1 + + +def to_str(value: Callable[[], str] | str) -> str: + "Turn callable or string into string." + if callable(value): + return to_str(value()) + else: + return str(value) + + +def to_int(value: Callable[[], int] | int) -> int: + "Turn callable or int into int." + if callable(value): + return to_int(value()) + else: + return int(value) + + +AnyFloat = Union[Callable[[], float], float] + + +def to_float(value: AnyFloat) -> float: + "Turn callable or float into float." + if callable(value): + return to_float(value()) + else: + return float(value) + + +def is_dumb_terminal(term: str | None = None) -> bool: + """ + True if this terminal type is considered "dumb". + + If so, we should fall back to the simplest possible form of line editing, + without cursor positioning and color support. + """ + if term is None: + return is_dumb_terminal(os.environ.get("TERM", "")) + + return term.lower() in ["dumb", "unknown"] diff --git a/lib/prompt_toolkit/validation.py b/lib/prompt_toolkit/validation.py new file mode 100644 index 0000000..2b35d1f --- /dev/null +++ b/lib/prompt_toolkit/validation.py @@ -0,0 +1,192 @@ +""" +Input validation for a `Buffer`. +(Validators will be called before accepting input.) +""" + +from __future__ import annotations + +from abc import ABCMeta, abstractmethod +from typing import Callable + +from prompt_toolkit.eventloop import run_in_executor_with_context + +from .document import Document +from .filters import FilterOrBool, to_filter + +__all__ = [ + "ConditionalValidator", + "ValidationError", + "Validator", + "ThreadedValidator", + "DummyValidator", + "DynamicValidator", +] + + +class ValidationError(Exception): + """ + Error raised by :meth:`.Validator.validate`. + + :param cursor_position: The cursor position where the error occurred. + :param message: Text. + """ + + def __init__(self, cursor_position: int = 0, message: str = "") -> None: + super().__init__(message) + self.cursor_position = cursor_position + self.message = message + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(cursor_position={self.cursor_position!r}, message={self.message!r})" + + +class Validator(metaclass=ABCMeta): + """ + Abstract base class for an input validator. + + A validator is typically created in one of the following two ways: + + - Either by overriding this class and implementing the `validate` method. + - Or by passing a callable to `Validator.from_callable`. + + If the validation takes some time and needs to happen in a background + thread, this can be wrapped in a :class:`.ThreadedValidator`. + """ + + @abstractmethod + def validate(self, document: Document) -> None: + """ + Validate the input. + If invalid, this should raise a :class:`.ValidationError`. + + :param document: :class:`~prompt_toolkit.document.Document` instance. + """ + pass + + async def validate_async(self, document: Document) -> None: + """ + Return a `Future` which is set when the validation is ready. + This function can be overloaded in order to provide an asynchronous + implementation. + """ + try: + self.validate(document) + except ValidationError: + raise + + @classmethod + def from_callable( + cls, + validate_func: Callable[[str], bool], + error_message: str = "Invalid input", + move_cursor_to_end: bool = False, + ) -> Validator: + """ + Create a validator from a simple validate callable. E.g.: + + .. code:: python + + def is_valid(text): + return text in ['hello', 'world'] + Validator.from_callable(is_valid, error_message='Invalid input') + + :param validate_func: Callable that takes the input string, and returns + `True` if the input is valid input. + :param error_message: Message to be displayed if the input is invalid. + :param move_cursor_to_end: Move the cursor to the end of the input, if + the input is invalid. + """ + return _ValidatorFromCallable(validate_func, error_message, move_cursor_to_end) + + +class _ValidatorFromCallable(Validator): + """ + Validate input from a simple callable. + """ + + def __init__( + self, func: Callable[[str], bool], error_message: str, move_cursor_to_end: bool + ) -> None: + self.func = func + self.error_message = error_message + self.move_cursor_to_end = move_cursor_to_end + + def __repr__(self) -> str: + return f"Validator.from_callable({self.func!r})" + + def validate(self, document: Document) -> None: + if not self.func(document.text): + if self.move_cursor_to_end: + index = len(document.text) + else: + index = 0 + + raise ValidationError(cursor_position=index, message=self.error_message) + + +class ThreadedValidator(Validator): + """ + Wrapper that runs input validation in a thread. + (Use this to prevent the user interface from becoming unresponsive if the + input validation takes too much time.) + """ + + def __init__(self, validator: Validator) -> None: + self.validator = validator + + def validate(self, document: Document) -> None: + self.validator.validate(document) + + async def validate_async(self, document: Document) -> None: + """ + Run the `validate` function in a thread. + """ + + def run_validation_thread() -> None: + return self.validate(document) + + await run_in_executor_with_context(run_validation_thread) + + +class DummyValidator(Validator): + """ + Validator class that accepts any input. + """ + + def validate(self, document: Document) -> None: + pass # Don't raise any exception. + + +class ConditionalValidator(Validator): + """ + Validator that can be switched on/off according to + a filter. (This wraps around another validator.) + """ + + def __init__(self, validator: Validator, filter: FilterOrBool) -> None: + self.validator = validator + self.filter = to_filter(filter) + + def validate(self, document: Document) -> None: + # Call the validator only if the filter is active. + if self.filter(): + self.validator.validate(document) + + +class DynamicValidator(Validator): + """ + Validator class that can dynamically returns any Validator. + + :param get_validator: Callable that returns a :class:`.Validator` instance. + """ + + def __init__(self, get_validator: Callable[[], Validator | None]) -> None: + self.get_validator = get_validator + + def validate(self, document: Document) -> None: + validator = self.get_validator() or DummyValidator() + validator.validate(document) + + async def validate_async(self, document: Document) -> None: + validator = self.get_validator() or DummyValidator() + await validator.validate_async(document) diff --git a/lib/prompt_toolkit/widgets/__init__.py b/lib/prompt_toolkit/widgets/__init__.py new file mode 100644 index 0000000..53cc3e1 --- /dev/null +++ b/lib/prompt_toolkit/widgets/__init__.py @@ -0,0 +1,63 @@ +""" +Collection of reusable components for building full screen applications. +These are higher level abstractions on top of the `prompt_toolkit.layout` +module. + +Most of these widgets implement the ``__pt_container__`` method, which makes it +possible to embed these in the layout like any other container. +""" + +from __future__ import annotations + +from .base import ( + Box, + Button, + Checkbox, + CheckboxList, + Frame, + HorizontalLine, + Label, + ProgressBar, + RadioList, + Shadow, + TextArea, + VerticalLine, +) +from .dialogs import Dialog +from .menus import MenuContainer, MenuItem +from .toolbars import ( + ArgToolbar, + CompletionsToolbar, + FormattedTextToolbar, + SearchToolbar, + SystemToolbar, + ValidationToolbar, +) + +__all__ = [ + # Base. + "TextArea", + "Label", + "Button", + "Frame", + "Shadow", + "Box", + "VerticalLine", + "HorizontalLine", + "CheckboxList", + "RadioList", + "Checkbox", + "ProgressBar", + # Toolbars. + "ArgToolbar", + "CompletionsToolbar", + "FormattedTextToolbar", + "SearchToolbar", + "SystemToolbar", + "ValidationToolbar", + # Dialogs. + "Dialog", + # Menus. + "MenuContainer", + "MenuItem", +] diff --git a/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc b/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..365c8ab Binary files /dev/null and b/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc b/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000..efceae1 Binary files /dev/null and b/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc b/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc new file mode 100644 index 0000000..d46d253 Binary files /dev/null and b/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc b/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc new file mode 100644 index 0000000..cb8ff00 Binary files /dev/null and b/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc b/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc new file mode 100644 index 0000000..97acdbe Binary files /dev/null and b/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc differ diff --git a/lib/prompt_toolkit/widgets/base.py b/lib/prompt_toolkit/widgets/base.py new file mode 100644 index 0000000..b8c2d67 --- /dev/null +++ b/lib/prompt_toolkit/widgets/base.py @@ -0,0 +1,1080 @@ +""" +Collection of reusable components for building full screen applications. + +All of these widgets implement the ``__pt_container__`` method, which makes +them usable in any situation where we are expecting a `prompt_toolkit` +container object. + +.. warning:: + + At this point, the API for these widgets is considered unstable, and can + potentially change between minor releases (we try not too, but no + guarantees are made yet). The public API in + `prompt_toolkit.shortcuts.dialogs` on the other hand is considered stable. +""" + +from __future__ import annotations + +from functools import partial +from typing import Callable, Generic, Sequence, TypeVar + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.auto_suggest import AutoSuggest, DynamicAutoSuggest +from prompt_toolkit.buffer import Buffer, BufferAcceptHandler +from prompt_toolkit.completion import Completer, DynamicCompleter +from prompt_toolkit.document import Document +from prompt_toolkit.filters import ( + Condition, + FilterOrBool, + has_focus, + is_done, + is_true, + to_filter, +) +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + Template, + to_formatted_text, +) +from prompt_toolkit.formatted_text.utils import fragment_list_to_text +from prompt_toolkit.history import History +from prompt_toolkit.key_binding.key_bindings import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.layout.containers import ( + AnyContainer, + ConditionalContainer, + Container, + DynamicContainer, + Float, + FloatContainer, + HSplit, + VSplit, + Window, + WindowAlign, +) +from prompt_toolkit.layout.controls import ( + BufferControl, + FormattedTextControl, + GetLinePrefixCallable, +) +from prompt_toolkit.layout.dimension import AnyDimension +from prompt_toolkit.layout.dimension import Dimension as D +from prompt_toolkit.layout.margins import ( + ConditionalMargin, + NumberedMargin, + ScrollbarMargin, +) +from prompt_toolkit.layout.processors import ( + AppendAutoSuggestion, + BeforeInput, + ConditionalProcessor, + PasswordProcessor, + Processor, +) +from prompt_toolkit.lexers import DynamicLexer, Lexer +from prompt_toolkit.mouse_events import MouseEvent, MouseEventType +from prompt_toolkit.utils import get_cwidth +from prompt_toolkit.validation import DynamicValidator, Validator + +from .toolbars import SearchToolbar + +__all__ = [ + "TextArea", + "Label", + "Button", + "Frame", + "Shadow", + "Box", + "VerticalLine", + "HorizontalLine", + "RadioList", + "CheckboxList", + "Checkbox", # backward compatibility + "ProgressBar", +] + +E = KeyPressEvent + + +class Border: + "Box drawing characters. (Thin)" + + HORIZONTAL = "\u2500" + VERTICAL = "\u2502" + TOP_LEFT = "\u250c" + TOP_RIGHT = "\u2510" + BOTTOM_LEFT = "\u2514" + BOTTOM_RIGHT = "\u2518" + + +class TextArea: + """ + A simple input field. + + This is a higher level abstraction on top of several other classes with + sane defaults. + + This widget does have the most common options, but it does not intend to + cover every single use case. For more configurations options, you can + always build a text area manually, using a + :class:`~prompt_toolkit.buffer.Buffer`, + :class:`~prompt_toolkit.layout.BufferControl` and + :class:`~prompt_toolkit.layout.Window`. + + Buffer attributes: + + :param text: The initial text. + :param multiline: If True, allow multiline input. + :param completer: :class:`~prompt_toolkit.completion.Completer` instance + for auto completion. + :param complete_while_typing: Boolean. + :param accept_handler: Called when `Enter` is pressed (This should be a + callable that takes a buffer as input). + :param history: :class:`~prompt_toolkit.history.History` instance. + :param auto_suggest: :class:`~prompt_toolkit.auto_suggest.AutoSuggest` + instance for input suggestions. + + BufferControl attributes: + + :param password: When `True`, display using asterisks. + :param focusable: When `True`, allow this widget to receive the focus. + :param focus_on_click: When `True`, focus after mouse click. + :param input_processors: `None` or a list of + :class:`~prompt_toolkit.layout.Processor` objects. + :param validator: `None` or a :class:`~prompt_toolkit.validation.Validator` + object. + + Window attributes: + + :param lexer: :class:`~prompt_toolkit.lexers.Lexer` instance for syntax + highlighting. + :param wrap_lines: When `True`, don't scroll horizontally, but wrap lines. + :param width: Window width. (:class:`~prompt_toolkit.layout.Dimension` object.) + :param height: Window height. (:class:`~prompt_toolkit.layout.Dimension` object.) + :param scrollbar: When `True`, display a scroll bar. + :param style: A style string. + :param dont_extend_width: When `True`, don't take up more width then the + preferred width reported by the control. + :param dont_extend_height: When `True`, don't take up more width then the + preferred height reported by the control. + :param get_line_prefix: None or a callable that returns formatted text to + be inserted before a line. It takes a line number (int) and a + wrap_count and returns formatted text. This can be used for + implementation of line continuations, things like Vim "breakindent" and + so on. + + Other attributes: + + :param search_field: An optional `SearchToolbar` object. + """ + + def __init__( + self, + text: str = "", + multiline: FilterOrBool = True, + password: FilterOrBool = False, + lexer: Lexer | None = None, + auto_suggest: AutoSuggest | None = None, + completer: Completer | None = None, + complete_while_typing: FilterOrBool = True, + validator: Validator | None = None, + accept_handler: BufferAcceptHandler | None = None, + history: History | None = None, + focusable: FilterOrBool = True, + focus_on_click: FilterOrBool = False, + wrap_lines: FilterOrBool = True, + read_only: FilterOrBool = False, + width: AnyDimension = None, + height: AnyDimension = None, + dont_extend_height: FilterOrBool = False, + dont_extend_width: FilterOrBool = False, + line_numbers: bool = False, + get_line_prefix: GetLinePrefixCallable | None = None, + scrollbar: bool = False, + style: str = "", + search_field: SearchToolbar | None = None, + preview_search: FilterOrBool = True, + prompt: AnyFormattedText = "", + input_processors: list[Processor] | None = None, + name: str = "", + ) -> None: + if search_field is None: + search_control = None + elif isinstance(search_field, SearchToolbar): + search_control = search_field.control + + if input_processors is None: + input_processors = [] + + # Writeable attributes. + self.completer = completer + self.complete_while_typing = complete_while_typing + self.lexer = lexer + self.auto_suggest = auto_suggest + self.read_only = read_only + self.wrap_lines = wrap_lines + self.validator = validator + + self.buffer = Buffer( + document=Document(text, 0), + multiline=multiline, + read_only=Condition(lambda: is_true(self.read_only)), + completer=DynamicCompleter(lambda: self.completer), + complete_while_typing=Condition( + lambda: is_true(self.complete_while_typing) + ), + validator=DynamicValidator(lambda: self.validator), + auto_suggest=DynamicAutoSuggest(lambda: self.auto_suggest), + accept_handler=accept_handler, + history=history, + name=name, + ) + + self.control = BufferControl( + buffer=self.buffer, + lexer=DynamicLexer(lambda: self.lexer), + input_processors=[ + ConditionalProcessor( + AppendAutoSuggestion(), has_focus(self.buffer) & ~is_done + ), + ConditionalProcessor( + processor=PasswordProcessor(), filter=to_filter(password) + ), + BeforeInput(prompt, style="class:text-area.prompt"), + ] + + input_processors, + search_buffer_control=search_control, + preview_search=preview_search, + focusable=focusable, + focus_on_click=focus_on_click, + ) + + if multiline: + if scrollbar: + right_margins = [ScrollbarMargin(display_arrows=True)] + else: + right_margins = [] + if line_numbers: + left_margins = [NumberedMargin()] + else: + left_margins = [] + else: + height = D.exact(1) + left_margins = [] + right_margins = [] + + style = "class:text-area " + style + + # If no height was given, guarantee height of at least 1. + if height is None: + height = D(min=1) + + self.window = Window( + height=height, + width=width, + dont_extend_height=dont_extend_height, + dont_extend_width=dont_extend_width, + content=self.control, + style=style, + wrap_lines=Condition(lambda: is_true(self.wrap_lines)), + left_margins=left_margins, + right_margins=right_margins, + get_line_prefix=get_line_prefix, + ) + + @property + def text(self) -> str: + """ + The `Buffer` text. + """ + return self.buffer.text + + @text.setter + def text(self, value: str) -> None: + self.document = Document(value, 0) + + @property + def document(self) -> Document: + """ + The `Buffer` document (text + cursor position). + """ + return self.buffer.document + + @document.setter + def document(self, value: Document) -> None: + self.buffer.set_document(value, bypass_readonly=True) + + @property + def accept_handler(self) -> BufferAcceptHandler | None: + """ + The accept handler. Called when the user accepts the input. + """ + return self.buffer.accept_handler + + @accept_handler.setter + def accept_handler(self, value: BufferAcceptHandler) -> None: + self.buffer.accept_handler = value + + def __pt_container__(self) -> Container: + return self.window + + +class Label: + """ + Widget that displays the given text. It is not editable or focusable. + + :param text: Text to display. Can be multiline. All value types accepted by + :class:`prompt_toolkit.layout.FormattedTextControl` are allowed, + including a callable. + :param style: A style string. + :param width: When given, use this width, rather than calculating it from + the text size. + :param dont_extend_width: When `True`, don't take up more width than + preferred, i.e. the length of the longest line of + the text, or value of `width` parameter, if + given. `True` by default + :param dont_extend_height: When `True`, don't take up more width than the + preferred height, i.e. the number of lines of + the text. `False` by default. + """ + + def __init__( + self, + text: AnyFormattedText, + style: str = "", + width: AnyDimension = None, + dont_extend_height: bool = True, + dont_extend_width: bool = False, + align: WindowAlign | Callable[[], WindowAlign] = WindowAlign.LEFT, + # There is no cursor navigation in a label, so it makes sense to always + # wrap lines by default. + wrap_lines: FilterOrBool = True, + ) -> None: + self.text = text + + def get_width() -> AnyDimension: + if width is None: + text_fragments = to_formatted_text(self.text) + text = fragment_list_to_text(text_fragments) + if text: + longest_line = max(get_cwidth(line) for line in text.splitlines()) + else: + return D(preferred=0) + return D(preferred=longest_line) + else: + return width + + self.formatted_text_control = FormattedTextControl(text=lambda: self.text) + + self.window = Window( + content=self.formatted_text_control, + width=get_width, + height=D(min=1), + style="class:label " + style, + dont_extend_height=dont_extend_height, + dont_extend_width=dont_extend_width, + align=align, + wrap_lines=wrap_lines, + ) + + def __pt_container__(self) -> Container: + return self.window + + +class Button: + """ + Clickable button. + + :param text: The caption for the button. + :param handler: `None` or callable. Called when the button is clicked. No + parameters are passed to this callable. Use for instance Python's + `functools.partial` to pass parameters to this callable if needed. + :param width: Width of the button. + """ + + def __init__( + self, + text: str, + handler: Callable[[], None] | None = None, + width: int = 12, + left_symbol: str = "<", + right_symbol: str = ">", + ) -> None: + self.text = text + self.left_symbol = left_symbol + self.right_symbol = right_symbol + self.handler = handler + self.width = width + self.control = FormattedTextControl( + self._get_text_fragments, + key_bindings=self._get_key_bindings(), + focusable=True, + ) + + def get_style() -> str: + if get_app().layout.has_focus(self): + return "class:button.focused" + else: + return "class:button" + + # Note: `dont_extend_width` is False, because we want to allow buttons + # to take more space if the parent container provides more space. + # Otherwise, we will also truncate the text. + # Probably we need a better way here to adjust to width of the + # button to the text. + + self.window = Window( + self.control, + align=WindowAlign.CENTER, + height=1, + width=width, + style=get_style, + dont_extend_width=False, + dont_extend_height=True, + ) + + def _get_text_fragments(self) -> StyleAndTextTuples: + width = ( + self.width + - (get_cwidth(self.left_symbol) + get_cwidth(self.right_symbol)) + + (len(self.text) - get_cwidth(self.text)) + ) + text = (f"{{:^{max(0, width)}}}").format(self.text) + + def handler(mouse_event: MouseEvent) -> None: + if ( + self.handler is not None + and mouse_event.event_type == MouseEventType.MOUSE_UP + ): + self.handler() + + return [ + ("class:button.arrow", self.left_symbol, handler), + ("[SetCursorPosition]", ""), + ("class:button.text", text, handler), + ("class:button.arrow", self.right_symbol, handler), + ] + + def _get_key_bindings(self) -> KeyBindings: + "Key bindings for the Button." + kb = KeyBindings() + + @kb.add(" ") + @kb.add("enter") + def _(event: E) -> None: + if self.handler is not None: + self.handler() + + return kb + + def __pt_container__(self) -> Container: + return self.window + + +class Frame: + """ + Draw a border around any container, optionally with a title text. + + Changing the title and body of the frame is possible at runtime by + assigning to the `body` and `title` attributes of this class. + + :param body: Another container object. + :param title: Text to be displayed in the top of the frame (can be formatted text). + :param style: Style string to be applied to this widget. + """ + + def __init__( + self, + body: AnyContainer, + title: AnyFormattedText = "", + style: str = "", + width: AnyDimension = None, + height: AnyDimension = None, + key_bindings: KeyBindings | None = None, + modal: bool = False, + ) -> None: + self.title = title + self.body = body + + fill = partial(Window, style="class:frame.border") + style = "class:frame " + style + + top_row_with_title = VSplit( + [ + fill(width=1, height=1, char=Border.TOP_LEFT), + fill(char=Border.HORIZONTAL), + fill(width=1, height=1, char="|"), + # Notice: we use `Template` here, because `self.title` can be an + # `HTML` object for instance. + Label( + lambda: Template(" {} ").format(self.title), + style="class:frame.label", + dont_extend_width=True, + ), + fill(width=1, height=1, char="|"), + fill(char=Border.HORIZONTAL), + fill(width=1, height=1, char=Border.TOP_RIGHT), + ], + height=1, + ) + + top_row_without_title = VSplit( + [ + fill(width=1, height=1, char=Border.TOP_LEFT), + fill(char=Border.HORIZONTAL), + fill(width=1, height=1, char=Border.TOP_RIGHT), + ], + height=1, + ) + + @Condition + def has_title() -> bool: + return bool(self.title) + + self.container = HSplit( + [ + ConditionalContainer( + content=top_row_with_title, + filter=has_title, + alternative_content=top_row_without_title, + ), + VSplit( + [ + fill(width=1, char=Border.VERTICAL), + DynamicContainer(lambda: self.body), + fill(width=1, char=Border.VERTICAL), + # Padding is required to make sure that if the content is + # too small, the right frame border is still aligned. + ], + padding=0, + ), + VSplit( + [ + fill(width=1, height=1, char=Border.BOTTOM_LEFT), + fill(char=Border.HORIZONTAL), + fill(width=1, height=1, char=Border.BOTTOM_RIGHT), + ], + # specifying height here will increase the rendering speed. + height=1, + ), + ], + width=width, + height=height, + style=style, + key_bindings=key_bindings, + modal=modal, + ) + + def __pt_container__(self) -> Container: + return self.container + + +class Shadow: + """ + Draw a shadow underneath/behind this container. + (This applies `class:shadow` the the cells under the shadow. The Style + should define the colors for the shadow.) + + :param body: Another container object. + """ + + def __init__(self, body: AnyContainer) -> None: + self.container = FloatContainer( + content=body, + floats=[ + Float( + bottom=-1, + height=1, + left=1, + right=-1, + transparent=True, + content=Window(style="class:shadow"), + ), + Float( + bottom=-1, + top=1, + width=1, + right=-1, + transparent=True, + content=Window(style="class:shadow"), + ), + ], + ) + + def __pt_container__(self) -> Container: + return self.container + + +class Box: + """ + Add padding around a container. + + This also makes sure that the parent can provide more space than required by + the child. This is very useful when wrapping a small element with a fixed + size into a ``VSplit`` or ``HSplit`` object. The ``HSplit`` and ``VSplit`` + try to make sure to adapt respectively the width and height, possibly + shrinking other elements. Wrapping something in a ``Box`` makes it flexible. + + :param body: Another container object. + :param padding: The margin to be used around the body. This can be + overridden by `padding_left`, padding_right`, `padding_top` and + `padding_bottom`. + :param style: A style string. + :param char: Character to be used for filling the space around the body. + (This is supposed to be a character with a terminal width of 1.) + """ + + def __init__( + self, + body: AnyContainer, + padding: AnyDimension = None, + padding_left: AnyDimension = None, + padding_right: AnyDimension = None, + padding_top: AnyDimension = None, + padding_bottom: AnyDimension = None, + width: AnyDimension = None, + height: AnyDimension = None, + style: str = "", + char: None | str | Callable[[], str] = None, + modal: bool = False, + key_bindings: KeyBindings | None = None, + ) -> None: + self.padding = padding + self.padding_left = padding_left + self.padding_right = padding_right + self.padding_top = padding_top + self.padding_bottom = padding_bottom + self.body = body + + def left() -> AnyDimension: + if self.padding_left is None: + return self.padding + return self.padding_left + + def right() -> AnyDimension: + if self.padding_right is None: + return self.padding + return self.padding_right + + def top() -> AnyDimension: + if self.padding_top is None: + return self.padding + return self.padding_top + + def bottom() -> AnyDimension: + if self.padding_bottom is None: + return self.padding + return self.padding_bottom + + self.container = HSplit( + [ + Window(height=top, char=char), + VSplit( + [ + Window(width=left, char=char), + body, + Window(width=right, char=char), + ] + ), + Window(height=bottom, char=char), + ], + width=width, + height=height, + style=style, + modal=modal, + key_bindings=None, + ) + + def __pt_container__(self) -> Container: + return self.container + + +_T = TypeVar("_T") + + +class _DialogList(Generic[_T]): + """ + Common code for `RadioList` and `CheckboxList`. + """ + + def __init__( + self, + values: Sequence[tuple[_T, AnyFormattedText]], + default_values: Sequence[_T] | None = None, + select_on_focus: bool = False, + open_character: str = "", + select_character: str = "*", + close_character: str = "", + container_style: str = "", + default_style: str = "", + number_style: str = "", + selected_style: str = "", + checked_style: str = "", + multiple_selection: bool = False, + show_scrollbar: bool = True, + show_cursor: bool = True, + show_numbers: bool = False, + ) -> None: + assert len(values) > 0 + default_values = default_values or [] + + self.values = values + self.show_numbers = show_numbers + + self.open_character = open_character + self.select_character = select_character + self.close_character = close_character + self.container_style = container_style + self.default_style = default_style + self.number_style = number_style + self.selected_style = selected_style + self.checked_style = checked_style + self.multiple_selection = multiple_selection + self.show_scrollbar = show_scrollbar + + # current_values will be used in multiple_selection, + # current_value will be used otherwise. + keys: list[_T] = [value for (value, _) in values] + self.current_values: list[_T] = [ + value for value in default_values if value in keys + ] + self.current_value: _T = ( + default_values[0] + if len(default_values) and default_values[0] in keys + else values[0][0] + ) + + # Cursor index: take first selected item or first item otherwise. + if len(self.current_values) > 0: + self._selected_index = keys.index(self.current_values[0]) + else: + self._selected_index = 0 + + # Key bindings. + kb = KeyBindings() + + @kb.add("up") + @kb.add("k") # Vi-like. + def _up(event: E) -> None: + self._selected_index = max(0, self._selected_index - 1) + if select_on_focus: + self._handle_enter() + + @kb.add("down") + @kb.add("j") # Vi-like. + def _down(event: E) -> None: + self._selected_index = min(len(self.values) - 1, self._selected_index + 1) + if select_on_focus: + self._handle_enter() + + @kb.add("pageup") + def _pageup(event: E) -> None: + w = event.app.layout.current_window + if w.render_info: + self._selected_index = max( + 0, self._selected_index - len(w.render_info.displayed_lines) + ) + + @kb.add("pagedown") + def _pagedown(event: E) -> None: + w = event.app.layout.current_window + if w.render_info: + self._selected_index = min( + len(self.values) - 1, + self._selected_index + len(w.render_info.displayed_lines), + ) + + @kb.add("enter") + @kb.add(" ") + def _click(event: E) -> None: + self._handle_enter() + + @kb.add(Keys.Any) + def _find(event: E) -> None: + # We first check values after the selected value, then all values. + values = list(self.values) + for value in values[self._selected_index + 1 :] + values: + text = fragment_list_to_text(to_formatted_text(value[1])).lower() + + if text.startswith(event.data.lower()): + self._selected_index = self.values.index(value) + return + + numbers_visible = Condition(lambda: self.show_numbers) + + for i in range(1, 10): + + @kb.add(str(i), filter=numbers_visible) + def _select_i(event: E, index: int = i) -> None: + self._selected_index = min(len(self.values) - 1, index - 1) + if select_on_focus: + self._handle_enter() + + # Control and window. + self.control = FormattedTextControl( + self._get_text_fragments, + key_bindings=kb, + focusable=True, + show_cursor=show_cursor, + ) + + self.window = Window( + content=self.control, + style=self.container_style, + right_margins=[ + ConditionalMargin( + margin=ScrollbarMargin(display_arrows=True), + filter=Condition(lambda: self.show_scrollbar), + ), + ], + dont_extend_height=True, + ) + + def _handle_enter(self) -> None: + if self.multiple_selection: + val = self.values[self._selected_index][0] + if val in self.current_values: + self.current_values.remove(val) + else: + self.current_values.append(val) + else: + self.current_value = self.values[self._selected_index][0] + + def _get_text_fragments(self) -> StyleAndTextTuples: + def mouse_handler(mouse_event: MouseEvent) -> None: + """ + Set `_selected_index` and `current_value` according to the y + position of the mouse click event. + """ + if mouse_event.event_type == MouseEventType.MOUSE_UP: + self._selected_index = mouse_event.position.y + self._handle_enter() + + result: StyleAndTextTuples = [] + for i, value in enumerate(self.values): + if self.multiple_selection: + checked = value[0] in self.current_values + else: + checked = value[0] == self.current_value + selected = i == self._selected_index + + style = "" + if checked: + style += " " + self.checked_style + if selected: + style += " " + self.selected_style + + result.append((style, self.open_character)) + + if selected: + result.append(("[SetCursorPosition]", "")) + + if checked: + result.append((style, self.select_character)) + else: + result.append((style, " ")) + + result.append((style, self.close_character)) + result.append((f"{style} {self.default_style}", " ")) + + if self.show_numbers: + result.append((f"{style} {self.number_style}", f"{i + 1:2d}. ")) + + result.extend( + to_formatted_text(value[1], style=f"{style} {self.default_style}") + ) + result.append(("", "\n")) + + # Add mouse handler to all fragments. + for i in range(len(result)): + result[i] = (result[i][0], result[i][1], mouse_handler) + + result.pop() # Remove last newline. + return result + + def __pt_container__(self) -> Container: + return self.window + + +class RadioList(_DialogList[_T]): + """ + List of radio buttons. Only one can be checked at the same time. + + :param values: List of (value, label) tuples. + """ + + def __init__( + self, + values: Sequence[tuple[_T, AnyFormattedText]], + default: _T | None = None, + show_numbers: bool = False, + select_on_focus: bool = False, + open_character: str = "(", + select_character: str = "*", + close_character: str = ")", + container_style: str = "class:radio-list", + default_style: str = "class:radio", + selected_style: str = "class:radio-selected", + checked_style: str = "class:radio-checked", + number_style: str = "class:radio-number", + multiple_selection: bool = False, + show_cursor: bool = True, + show_scrollbar: bool = True, + ) -> None: + if default is None: + default_values = None + else: + default_values = [default] + + super().__init__( + values, + default_values=default_values, + select_on_focus=select_on_focus, + show_numbers=show_numbers, + open_character=open_character, + select_character=select_character, + close_character=close_character, + container_style=container_style, + default_style=default_style, + selected_style=selected_style, + checked_style=checked_style, + number_style=number_style, + multiple_selection=False, + show_cursor=show_cursor, + show_scrollbar=show_scrollbar, + ) + + +class CheckboxList(_DialogList[_T]): + """ + List of checkbox buttons. Several can be checked at the same time. + + :param values: List of (value, label) tuples. + """ + + def __init__( + self, + values: Sequence[tuple[_T, AnyFormattedText]], + default_values: Sequence[_T] | None = None, + open_character: str = "[", + select_character: str = "*", + close_character: str = "]", + container_style: str = "class:checkbox-list", + default_style: str = "class:checkbox", + selected_style: str = "class:checkbox-selected", + checked_style: str = "class:checkbox-checked", + ) -> None: + super().__init__( + values, + default_values=default_values, + open_character=open_character, + select_character=select_character, + close_character=close_character, + container_style=container_style, + default_style=default_style, + selected_style=selected_style, + checked_style=checked_style, + multiple_selection=True, + ) + + +class Checkbox(CheckboxList[str]): + """Backward compatibility util: creates a 1-sized CheckboxList + + :param text: the text + """ + + show_scrollbar = False + + def __init__(self, text: AnyFormattedText = "", checked: bool = False) -> None: + values = [("value", text)] + super().__init__(values=values) + self.checked = checked + + @property + def checked(self) -> bool: + return "value" in self.current_values + + @checked.setter + def checked(self, value: bool) -> None: + if value: + self.current_values = ["value"] + else: + self.current_values = [] + + +class VerticalLine: + """ + A simple vertical line with a width of 1. + """ + + def __init__(self) -> None: + self.window = Window( + char=Border.VERTICAL, style="class:line,vertical-line", width=1 + ) + + def __pt_container__(self) -> Container: + return self.window + + +class HorizontalLine: + """ + A simple horizontal line with a height of 1. + """ + + def __init__(self) -> None: + self.window = Window( + char=Border.HORIZONTAL, style="class:line,horizontal-line", height=1 + ) + + def __pt_container__(self) -> Container: + return self.window + + +class ProgressBar: + def __init__(self) -> None: + self._percentage = 60 + + self.label = Label("60%") + self.container = FloatContainer( + content=Window(height=1), + floats=[ + # We first draw the label, then the actual progress bar. Right + # now, this is the only way to have the colors of the progress + # bar appear on top of the label. The problem is that our label + # can't be part of any `Window` below. + Float(content=self.label, top=0, bottom=0), + Float( + left=0, + top=0, + right=0, + bottom=0, + content=VSplit( + [ + Window( + style="class:progress-bar.used", + width=lambda: D(weight=int(self._percentage)), + ), + Window( + style="class:progress-bar", + width=lambda: D(weight=int(100 - self._percentage)), + ), + ] + ), + ), + ], + ) + + @property + def percentage(self) -> int: + return self._percentage + + @percentage.setter + def percentage(self, value: int) -> None: + self._percentage = value + self.label.text = f"{value}%" + + def __pt_container__(self) -> Container: + return self.container diff --git a/lib/prompt_toolkit/widgets/dialogs.py b/lib/prompt_toolkit/widgets/dialogs.py new file mode 100644 index 0000000..5f5f170 --- /dev/null +++ b/lib/prompt_toolkit/widgets/dialogs.py @@ -0,0 +1,108 @@ +""" +Collection of reusable components for building full screen applications. +""" + +from __future__ import annotations + +from typing import Sequence + +from prompt_toolkit.filters import has_completions, has_focus +from prompt_toolkit.formatted_text import AnyFormattedText +from prompt_toolkit.key_binding.bindings.focus import focus_next, focus_previous +from prompt_toolkit.key_binding.key_bindings import KeyBindings +from prompt_toolkit.layout.containers import ( + AnyContainer, + DynamicContainer, + HSplit, + VSplit, +) +from prompt_toolkit.layout.dimension import AnyDimension +from prompt_toolkit.layout.dimension import Dimension as D + +from .base import Box, Button, Frame, Shadow + +__all__ = [ + "Dialog", +] + + +class Dialog: + """ + Simple dialog window. This is the base for input dialogs, message dialogs + and confirmation dialogs. + + Changing the title and body of the dialog is possible at runtime by + assigning to the `body` and `title` attributes of this class. + + :param body: Child container object. + :param title: Text to be displayed in the heading of the dialog. + :param buttons: A list of `Button` widgets, displayed at the bottom. + """ + + def __init__( + self, + body: AnyContainer, + title: AnyFormattedText = "", + buttons: Sequence[Button] | None = None, + modal: bool = True, + width: AnyDimension = None, + with_background: bool = False, + ) -> None: + self.body = body + self.title = title + + buttons = buttons or [] + + # When a button is selected, handle left/right key bindings. + buttons_kb = KeyBindings() + if len(buttons) > 1: + first_selected = has_focus(buttons[0]) + last_selected = has_focus(buttons[-1]) + + buttons_kb.add("left", filter=~first_selected)(focus_previous) + buttons_kb.add("right", filter=~last_selected)(focus_next) + + frame_body: AnyContainer + if buttons: + frame_body = HSplit( + [ + # Add optional padding around the body. + Box( + body=DynamicContainer(lambda: self.body), + padding=D(preferred=1, max=1), + padding_bottom=0, + ), + # The buttons. + Box( + body=VSplit(buttons, padding=1, key_bindings=buttons_kb), + height=D(min=1, max=3, preferred=3), + ), + ] + ) + else: + frame_body = body + + # Key bindings for whole dialog. + kb = KeyBindings() + kb.add("tab", filter=~has_completions)(focus_next) + kb.add("s-tab", filter=~has_completions)(focus_previous) + + frame = Shadow( + body=Frame( + title=lambda: self.title, + body=frame_body, + style="class:dialog.body", + width=(None if with_background is None else width), + key_bindings=kb, + modal=modal, + ) + ) + + self.container: Box | Shadow + if with_background: + self.container = Box(body=frame, style="class:dialog", width=width) + else: + self.container = frame + + def __pt_container__(self) -> AnyContainer: + return self.container diff --git a/lib/prompt_toolkit/widgets/menus.py b/lib/prompt_toolkit/widgets/menus.py new file mode 100644 index 0000000..c574c06 --- /dev/null +++ b/lib/prompt_toolkit/widgets/menus.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +from typing import Callable, Iterable, Sequence + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.filters import Condition +from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple, StyleAndTextTuples +from prompt_toolkit.key_binding.key_bindings import KeyBindings, KeyBindingsBase +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.keys import Keys +from prompt_toolkit.layout.containers import ( + AnyContainer, + ConditionalContainer, + Container, + Float, + FloatContainer, + HSplit, + Window, +) +from prompt_toolkit.layout.controls import FormattedTextControl +from prompt_toolkit.mouse_events import MouseEvent, MouseEventType +from prompt_toolkit.utils import get_cwidth +from prompt_toolkit.widgets import Shadow + +from .base import Border + +__all__ = [ + "MenuContainer", + "MenuItem", +] + +E = KeyPressEvent + + +class MenuContainer: + """ + :param floats: List of extra Float objects to display. + :param menu_items: List of `MenuItem` objects. + """ + + def __init__( + self, + body: AnyContainer, + menu_items: list[MenuItem], + floats: list[Float] | None = None, + key_bindings: KeyBindingsBase | None = None, + ) -> None: + self.body = body + self.menu_items = menu_items + self.selected_menu = [0] + + # Key bindings. + kb = KeyBindings() + + @Condition + def in_main_menu() -> bool: + return len(self.selected_menu) == 1 + + @Condition + def in_sub_menu() -> bool: + return len(self.selected_menu) > 1 + + # Navigation through the main menu. + + @kb.add("left", filter=in_main_menu) + def _left(event: E) -> None: + self.selected_menu[0] = max(0, self.selected_menu[0] - 1) + + @kb.add("right", filter=in_main_menu) + def _right(event: E) -> None: + self.selected_menu[0] = min( + len(self.menu_items) - 1, self.selected_menu[0] + 1 + ) + + @kb.add("down", filter=in_main_menu) + def _down(event: E) -> None: + self.selected_menu.append(0) + + @kb.add("c-c", filter=in_main_menu) + @kb.add("c-g", filter=in_main_menu) + def _cancel(event: E) -> None: + "Leave menu." + event.app.layout.focus_last() + + # Sub menu navigation. + + @kb.add("left", filter=in_sub_menu) + @kb.add("c-g", filter=in_sub_menu) + @kb.add("c-c", filter=in_sub_menu) + def _back(event: E) -> None: + "Go back to parent menu." + if len(self.selected_menu) > 1: + self.selected_menu.pop() + + @kb.add("right", filter=in_sub_menu) + def _submenu(event: E) -> None: + "go into sub menu." + if self._get_menu(len(self.selected_menu) - 1).children: + self.selected_menu.append(0) + + # If This item does not have a sub menu. Go up in the parent menu. + elif ( + len(self.selected_menu) == 2 + and self.selected_menu[0] < len(self.menu_items) - 1 + ): + self.selected_menu = [ + min(len(self.menu_items) - 1, self.selected_menu[0] + 1) + ] + if self.menu_items[self.selected_menu[0]].children: + self.selected_menu.append(0) + + @kb.add("up", filter=in_sub_menu) + def _up_in_submenu(event: E) -> None: + "Select previous (enabled) menu item or return to main menu." + # Look for previous enabled items in this sub menu. + menu = self._get_menu(len(self.selected_menu) - 2) + index = self.selected_menu[-1] + + previous_indexes = [ + i + for i, item in enumerate(menu.children) + if i < index and not item.disabled + ] + + if previous_indexes: + self.selected_menu[-1] = previous_indexes[-1] + elif len(self.selected_menu) == 2: + # Return to main menu. + self.selected_menu.pop() + + @kb.add("down", filter=in_sub_menu) + def _down_in_submenu(event: E) -> None: + "Select next (enabled) menu item." + menu = self._get_menu(len(self.selected_menu) - 2) + index = self.selected_menu[-1] + + next_indexes = [ + i + for i, item in enumerate(menu.children) + if i > index and not item.disabled + ] + + if next_indexes: + self.selected_menu[-1] = next_indexes[0] + + @kb.add("enter") + def _click(event: E) -> None: + "Click the selected menu item." + item = self._get_menu(len(self.selected_menu) - 1) + if item.handler: + event.app.layout.focus_last() + item.handler() + + # Controls. + self.control = FormattedTextControl( + self._get_menu_fragments, key_bindings=kb, focusable=True, show_cursor=False + ) + + self.window = Window(height=1, content=self.control, style="class:menu-bar") + + submenu = self._submenu(0) + submenu2 = self._submenu(1) + submenu3 = self._submenu(2) + + @Condition + def has_focus() -> bool: + return get_app().layout.current_window == self.window + + self.container = FloatContainer( + content=HSplit( + [ + # The titlebar. + self.window, + # The 'body', like defined above. + body, + ] + ), + floats=[ + Float( + xcursor=True, + ycursor=True, + content=ConditionalContainer( + content=Shadow(body=submenu), filter=has_focus + ), + ), + Float( + attach_to_window=submenu, + xcursor=True, + ycursor=True, + allow_cover_cursor=True, + content=ConditionalContainer( + content=Shadow(body=submenu2), + filter=has_focus + & Condition(lambda: len(self.selected_menu) >= 1), + ), + ), + Float( + attach_to_window=submenu2, + xcursor=True, + ycursor=True, + allow_cover_cursor=True, + content=ConditionalContainer( + content=Shadow(body=submenu3), + filter=has_focus + & Condition(lambda: len(self.selected_menu) >= 2), + ), + ), + # -- + ] + + (floats or []), + key_bindings=key_bindings, + ) + + def _get_menu(self, level: int) -> MenuItem: + menu = self.menu_items[self.selected_menu[0]] + + for i, index in enumerate(self.selected_menu[1:]): + if i < level: + try: + menu = menu.children[index] + except IndexError: + return MenuItem("debug") + + return menu + + def _get_menu_fragments(self) -> StyleAndTextTuples: + focused = get_app().layout.has_focus(self.window) + + # This is called during the rendering. When we discover that this + # widget doesn't have the focus anymore. Reset menu state. + if not focused: + self.selected_menu = [0] + + # Generate text fragments for the main menu. + def one_item(i: int, item: MenuItem) -> Iterable[OneStyleAndTextTuple]: + def mouse_handler(mouse_event: MouseEvent) -> None: + hover = mouse_event.event_type == MouseEventType.MOUSE_MOVE + if ( + mouse_event.event_type == MouseEventType.MOUSE_DOWN + or hover + and focused + ): + # Toggle focus. + app = get_app() + if not hover: + if app.layout.has_focus(self.window): + if self.selected_menu == [i]: + app.layout.focus_last() + else: + app.layout.focus(self.window) + self.selected_menu = [i] + + yield ("class:menu-bar", " ", mouse_handler) + if i == self.selected_menu[0] and focused: + yield ("[SetMenuPosition]", "", mouse_handler) + style = "class:menu-bar.selected-item" + else: + style = "class:menu-bar" + yield style, item.text, mouse_handler + + result: StyleAndTextTuples = [] + for i, item in enumerate(self.menu_items): + result.extend(one_item(i, item)) + + return result + + def _submenu(self, level: int = 0) -> Window: + def get_text_fragments() -> StyleAndTextTuples: + result: StyleAndTextTuples = [] + if level < len(self.selected_menu): + menu = self._get_menu(level) + if menu.children: + result.append(("class:menu", Border.TOP_LEFT)) + result.append(("class:menu", Border.HORIZONTAL * (menu.width + 4))) + result.append(("class:menu", Border.TOP_RIGHT)) + result.append(("", "\n")) + try: + selected_item = self.selected_menu[level + 1] + except IndexError: + selected_item = -1 + + def one_item( + i: int, item: MenuItem + ) -> Iterable[OneStyleAndTextTuple]: + def mouse_handler(mouse_event: MouseEvent) -> None: + if item.disabled: + # The arrow keys can't interact with menu items that are disabled. + # The mouse shouldn't be able to either. + return + hover = mouse_event.event_type == MouseEventType.MOUSE_MOVE + if ( + mouse_event.event_type == MouseEventType.MOUSE_UP + or hover + ): + app = get_app() + if not hover and item.handler: + app.layout.focus_last() + item.handler() + else: + self.selected_menu = self.selected_menu[ + : level + 1 + ] + [i] + + if i == selected_item: + yield ("[SetCursorPosition]", "") + style = "class:menu-bar.selected-item" + else: + style = "" + + yield ("class:menu", Border.VERTICAL) + if item.text == "-": + yield ( + style + "class:menu-border", + f"{Border.HORIZONTAL * (menu.width + 3)}", + mouse_handler, + ) + else: + yield ( + style, + f" {item.text}".ljust(menu.width + 3), + mouse_handler, + ) + + if item.children: + yield (style, ">", mouse_handler) + else: + yield (style, " ", mouse_handler) + + if i == selected_item: + yield ("[SetMenuPosition]", "") + yield ("class:menu", Border.VERTICAL) + + yield ("", "\n") + + for i, item in enumerate(menu.children): + result.extend(one_item(i, item)) + + result.append(("class:menu", Border.BOTTOM_LEFT)) + result.append(("class:menu", Border.HORIZONTAL * (menu.width + 4))) + result.append(("class:menu", Border.BOTTOM_RIGHT)) + return result + + return Window(FormattedTextControl(get_text_fragments), style="class:menu") + + @property + def floats(self) -> list[Float] | None: + return self.container.floats + + def __pt_container__(self) -> Container: + return self.container + + +class MenuItem: + def __init__( + self, + text: str = "", + handler: Callable[[], None] | None = None, + children: list[MenuItem] | None = None, + shortcut: Sequence[Keys | str] | None = None, + disabled: bool = False, + ) -> None: + self.text = text + self.handler = handler + self.children = children or [] + self.shortcut = shortcut + self.disabled = disabled + self.selected_item = 0 + + @property + def width(self) -> int: + if self.children: + return max(get_cwidth(c.text) for c in self.children) + else: + return 0 diff --git a/lib/prompt_toolkit/widgets/toolbars.py b/lib/prompt_toolkit/widgets/toolbars.py new file mode 100644 index 0000000..c5deffc --- /dev/null +++ b/lib/prompt_toolkit/widgets/toolbars.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +from typing import Any + +from prompt_toolkit.application.current import get_app +from prompt_toolkit.buffer import Buffer +from prompt_toolkit.enums import SYSTEM_BUFFER +from prompt_toolkit.filters import ( + Condition, + FilterOrBool, + emacs_mode, + has_arg, + has_completions, + has_focus, + has_validation_error, + to_filter, + vi_mode, + vi_navigation_mode, +) +from prompt_toolkit.formatted_text import ( + AnyFormattedText, + StyleAndTextTuples, + fragment_list_len, + to_formatted_text, +) +from prompt_toolkit.key_binding.key_bindings import ( + ConditionalKeyBindings, + KeyBindings, + KeyBindingsBase, + merge_key_bindings, +) +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.key_binding.vi_state import InputMode +from prompt_toolkit.keys import Keys +from prompt_toolkit.layout.containers import ConditionalContainer, Container, Window +from prompt_toolkit.layout.controls import ( + BufferControl, + FormattedTextControl, + SearchBufferControl, + UIContent, + UIControl, +) +from prompt_toolkit.layout.dimension import Dimension +from prompt_toolkit.layout.processors import BeforeInput +from prompt_toolkit.lexers import SimpleLexer +from prompt_toolkit.search import SearchDirection + +__all__ = [ + "ArgToolbar", + "CompletionsToolbar", + "FormattedTextToolbar", + "SearchToolbar", + "SystemToolbar", + "ValidationToolbar", +] + +E = KeyPressEvent + + +class FormattedTextToolbar(Window): + def __init__(self, text: AnyFormattedText, style: str = "", **kw: Any) -> None: + # Note: The style needs to be applied to the toolbar as a whole, not + # just the `FormattedTextControl`. + super().__init__( + FormattedTextControl(text, **kw), + style=style, + dont_extend_height=True, + height=Dimension(min=1), + ) + + +class SystemToolbar: + """ + Toolbar for a system prompt. + + :param prompt: Prompt to be displayed to the user. + """ + + def __init__( + self, + prompt: AnyFormattedText = "Shell command: ", + enable_global_bindings: FilterOrBool = True, + ) -> None: + self.prompt = prompt + self.enable_global_bindings = to_filter(enable_global_bindings) + + self.system_buffer = Buffer(name=SYSTEM_BUFFER) + + self._bindings = self._build_key_bindings() + + self.buffer_control = BufferControl( + buffer=self.system_buffer, + lexer=SimpleLexer(style="class:system-toolbar.text"), + input_processors=[ + BeforeInput(lambda: self.prompt, style="class:system-toolbar") + ], + key_bindings=self._bindings, + ) + + self.window = Window( + self.buffer_control, height=1, style="class:system-toolbar" + ) + + self.container = ConditionalContainer( + content=self.window, filter=has_focus(self.system_buffer) + ) + + def _get_display_before_text(self) -> StyleAndTextTuples: + return [ + ("class:system-toolbar", "Shell command: "), + ("class:system-toolbar.text", self.system_buffer.text), + ("", "\n"), + ] + + def _build_key_bindings(self) -> KeyBindingsBase: + focused = has_focus(self.system_buffer) + + # Emacs + emacs_bindings = KeyBindings() + handle = emacs_bindings.add + + @handle("escape", filter=focused) + @handle("c-g", filter=focused) + @handle("c-c", filter=focused) + def _cancel(event: E) -> None: + "Hide system prompt." + self.system_buffer.reset() + event.app.layout.focus_last() + + @handle("enter", filter=focused) + async def _accept(event: E) -> None: + "Run system command." + await event.app.run_system_command( + self.system_buffer.text, + display_before_text=self._get_display_before_text(), + ) + self.system_buffer.reset(append_to_history=True) + event.app.layout.focus_last() + + # Vi. + vi_bindings = KeyBindings() + handle = vi_bindings.add + + @handle("escape", filter=focused) + @handle("c-c", filter=focused) + def _cancel_vi(event: E) -> None: + "Hide system prompt." + event.app.vi_state.input_mode = InputMode.NAVIGATION + self.system_buffer.reset() + event.app.layout.focus_last() + + @handle("enter", filter=focused) + async def _accept_vi(event: E) -> None: + "Run system command." + event.app.vi_state.input_mode = InputMode.NAVIGATION + await event.app.run_system_command( + self.system_buffer.text, + display_before_text=self._get_display_before_text(), + ) + self.system_buffer.reset(append_to_history=True) + event.app.layout.focus_last() + + # Global bindings. (Listen to these bindings, even when this widget is + # not focussed.) + global_bindings = KeyBindings() + handle = global_bindings.add + + @handle(Keys.Escape, "!", filter=~focused & emacs_mode, is_global=True) + def _focus_me(event: E) -> None: + "M-'!' will focus this user control." + event.app.layout.focus(self.window) + + @handle("!", filter=~focused & vi_mode & vi_navigation_mode, is_global=True) + def _focus_me_vi(event: E) -> None: + "Focus." + event.app.vi_state.input_mode = InputMode.INSERT + event.app.layout.focus(self.window) + + return merge_key_bindings( + [ + ConditionalKeyBindings(emacs_bindings, emacs_mode), + ConditionalKeyBindings(vi_bindings, vi_mode), + ConditionalKeyBindings(global_bindings, self.enable_global_bindings), + ] + ) + + def __pt_container__(self) -> Container: + return self.container + + +class ArgToolbar: + def __init__(self) -> None: + def get_formatted_text() -> StyleAndTextTuples: + arg = get_app().key_processor.arg or "" + if arg == "-": + arg = "-1" + + return [ + ("class:arg-toolbar", "Repeat: "), + ("class:arg-toolbar.text", arg), + ] + + self.window = Window(FormattedTextControl(get_formatted_text), height=1) + + self.container = ConditionalContainer(content=self.window, filter=has_arg) + + def __pt_container__(self) -> Container: + return self.container + + +class SearchToolbar: + """ + :param vi_mode: Display '/' and '?' instead of I-search. + :param ignore_case: Search case insensitive. + """ + + def __init__( + self, + search_buffer: Buffer | None = None, + vi_mode: bool = False, + text_if_not_searching: AnyFormattedText = "", + forward_search_prompt: AnyFormattedText = "I-search: ", + backward_search_prompt: AnyFormattedText = "I-search backward: ", + ignore_case: FilterOrBool = False, + ) -> None: + if search_buffer is None: + search_buffer = Buffer() + + @Condition + def is_searching() -> bool: + return self.control in get_app().layout.search_links + + def get_before_input() -> AnyFormattedText: + if not is_searching(): + return text_if_not_searching + elif ( + self.control.searcher_search_state.direction == SearchDirection.BACKWARD + ): + return "?" if vi_mode else backward_search_prompt + else: + return "/" if vi_mode else forward_search_prompt + + self.search_buffer = search_buffer + + self.control = SearchBufferControl( + buffer=search_buffer, + input_processors=[ + BeforeInput(get_before_input, style="class:search-toolbar.prompt") + ], + lexer=SimpleLexer(style="class:search-toolbar.text"), + ignore_case=ignore_case, + ) + + self.container = ConditionalContainer( + content=Window(self.control, height=1, style="class:search-toolbar"), + filter=is_searching, + ) + + def __pt_container__(self) -> Container: + return self.container + + +class _CompletionsToolbarControl(UIControl): + def create_content(self, width: int, height: int) -> UIContent: + all_fragments: StyleAndTextTuples = [] + + complete_state = get_app().current_buffer.complete_state + if complete_state: + completions = complete_state.completions + index = complete_state.complete_index # Can be None! + + # Width of the completions without the left/right arrows in the margins. + content_width = width - 6 + + # Booleans indicating whether we stripped from the left/right + cut_left = False + cut_right = False + + # Create Menu content. + fragments: StyleAndTextTuples = [] + + for i, c in enumerate(completions): + # When there is no more place for the next completion + if fragment_list_len(fragments) + len(c.display_text) >= content_width: + # If the current one was not yet displayed, page to the next sequence. + if i <= (index or 0): + fragments = [] + cut_left = True + # If the current one is visible, stop here. + else: + cut_right = True + break + + fragments.extend( + to_formatted_text( + c.display_text, + style=( + "class:completion-toolbar.completion.current" + if i == index + else "class:completion-toolbar.completion" + ), + ) + ) + fragments.append(("", " ")) + + # Extend/strip until the content width. + fragments.append(("", " " * (content_width - fragment_list_len(fragments)))) + fragments = fragments[:content_width] + + # Return fragments + all_fragments.append(("", " ")) + all_fragments.append( + ("class:completion-toolbar.arrow", "<" if cut_left else " ") + ) + all_fragments.append(("", " ")) + + all_fragments.extend(fragments) + + all_fragments.append(("", " ")) + all_fragments.append( + ("class:completion-toolbar.arrow", ">" if cut_right else " ") + ) + all_fragments.append(("", " ")) + + def get_line(i: int) -> StyleAndTextTuples: + return all_fragments + + return UIContent(get_line=get_line, line_count=1) + + +class CompletionsToolbar: + def __init__(self) -> None: + self.container = ConditionalContainer( + content=Window( + _CompletionsToolbarControl(), height=1, style="class:completion-toolbar" + ), + filter=has_completions, + ) + + def __pt_container__(self) -> Container: + return self.container + + +class ValidationToolbar: + def __init__(self, show_position: bool = False) -> None: + def get_formatted_text() -> StyleAndTextTuples: + buff = get_app().current_buffer + + if buff.validation_error: + row, column = buff.document.translate_index_to_position( + buff.validation_error.cursor_position + ) + + if show_position: + text = f"{buff.validation_error.message} (line={row + 1} column={column + 1})" + else: + text = buff.validation_error.message + + return [("class:validation-toolbar", text)] + else: + return [] + + self.control = FormattedTextControl(get_formatted_text) + + self.container = ConditionalContainer( + content=Window(self.control, height=1), filter=has_validation_error + ) + + def __pt_container__(self) -> Container: + return self.container diff --git a/lib/prompt_toolkit/win32_types.py b/lib/prompt_toolkit/win32_types.py new file mode 100644 index 0000000..79283b8 --- /dev/null +++ b/lib/prompt_toolkit/win32_types.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +from ctypes import Structure, Union, c_char, c_long, c_short, c_ulong +from ctypes.wintypes import BOOL, DWORD, LPVOID, WCHAR, WORD +from typing import TYPE_CHECKING + +# Input/Output standard device numbers. Note that these are not handle objects. +# It's the `windll.kernel32.GetStdHandle` system call that turns them into a +# real handle object. +STD_INPUT_HANDLE = c_ulong(-10) +STD_OUTPUT_HANDLE = c_ulong(-11) +STD_ERROR_HANDLE = c_ulong(-12) + + +class COORD(Structure): + """ + Struct in wincon.h + http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx + """ + + if TYPE_CHECKING: + X: int + Y: int + + _fields_ = [ + ("X", c_short), # Short + ("Y", c_short), # Short + ] + + def __repr__(self) -> str: + return "{}(X={!r}, Y={!r}, type_x={!r}, type_y={!r})".format( + self.__class__.__name__, + self.X, + self.Y, + type(self.X), + type(self.Y), + ) + + +class UNICODE_OR_ASCII(Union): + if TYPE_CHECKING: + AsciiChar: bytes + UnicodeChar: str + + _fields_ = [ + ("AsciiChar", c_char), + ("UnicodeChar", WCHAR), + ] + + +class KEY_EVENT_RECORD(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684166(v=vs.85).aspx + """ + + if TYPE_CHECKING: + KeyDown: int + RepeatCount: int + VirtualKeyCode: int + VirtualScanCode: int + uChar: UNICODE_OR_ASCII + ControlKeyState: int + + _fields_ = [ + ("KeyDown", c_long), # bool + ("RepeatCount", c_short), # word + ("VirtualKeyCode", c_short), # word + ("VirtualScanCode", c_short), # word + ("uChar", UNICODE_OR_ASCII), # Unicode or ASCII. + ("ControlKeyState", c_long), # double word + ] + + +class MOUSE_EVENT_RECORD(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684239(v=vs.85).aspx + """ + + if TYPE_CHECKING: + MousePosition: COORD + ButtonState: int + ControlKeyState: int + EventFlags: int + + _fields_ = [ + ("MousePosition", COORD), + ("ButtonState", c_long), # dword + ("ControlKeyState", c_long), # dword + ("EventFlags", c_long), # dword + ] + + +class WINDOW_BUFFER_SIZE_RECORD(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/ms687093(v=vs.85).aspx + """ + + if TYPE_CHECKING: + Size: COORD + + _fields_ = [("Size", COORD)] + + +class MENU_EVENT_RECORD(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/ms684213(v=vs.85).aspx + """ + + if TYPE_CHECKING: + CommandId: int + + _fields_ = [("CommandId", c_long)] # uint + + +class FOCUS_EVENT_RECORD(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/ms683149(v=vs.85).aspx + """ + + if TYPE_CHECKING: + SetFocus: int + + _fields_ = [("SetFocus", c_long)] # bool + + +class EVENT_RECORD(Union): + if TYPE_CHECKING: + KeyEvent: KEY_EVENT_RECORD + MouseEvent: MOUSE_EVENT_RECORD + WindowBufferSizeEvent: WINDOW_BUFFER_SIZE_RECORD + MenuEvent: MENU_EVENT_RECORD + FocusEvent: FOCUS_EVENT_RECORD + + _fields_ = [ + ("KeyEvent", KEY_EVENT_RECORD), + ("MouseEvent", MOUSE_EVENT_RECORD), + ("WindowBufferSizeEvent", WINDOW_BUFFER_SIZE_RECORD), + ("MenuEvent", MENU_EVENT_RECORD), + ("FocusEvent", FOCUS_EVENT_RECORD), + ] + + +class INPUT_RECORD(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/ms683499(v=vs.85).aspx + """ + + if TYPE_CHECKING: + EventType: int + Event: EVENT_RECORD + + _fields_ = [("EventType", c_short), ("Event", EVENT_RECORD)] # word # Union. + + +EventTypes = { + 1: "KeyEvent", + 2: "MouseEvent", + 4: "WindowBufferSizeEvent", + 8: "MenuEvent", + 16: "FocusEvent", +} + + +class SMALL_RECT(Structure): + """struct in wincon.h.""" + + if TYPE_CHECKING: + Left: int + Top: int + Right: int + Bottom: int + + _fields_ = [ + ("Left", c_short), + ("Top", c_short), + ("Right", c_short), + ("Bottom", c_short), + ] + + +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + """struct in wincon.h.""" + + if TYPE_CHECKING: + dwSize: COORD + dwCursorPosition: COORD + wAttributes: int + srWindow: SMALL_RECT + dwMaximumWindowSize: COORD + + _fields_ = [ + ("dwSize", COORD), + ("dwCursorPosition", COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", COORD), + ] + + def __repr__(self) -> str: + return "CONSOLE_SCREEN_BUFFER_INFO({!r},{!r},{!r},{!r},{!r},{!r},{!r},{!r},{!r},{!r},{!r})".format( + self.dwSize.Y, + self.dwSize.X, + self.dwCursorPosition.Y, + self.dwCursorPosition.X, + self.wAttributes, + self.srWindow.Top, + self.srWindow.Left, + self.srWindow.Bottom, + self.srWindow.Right, + self.dwMaximumWindowSize.Y, + self.dwMaximumWindowSize.X, + ) + + +class SECURITY_ATTRIBUTES(Structure): + """ + http://msdn.microsoft.com/en-us/library/windows/desktop/aa379560(v=vs.85).aspx + """ + + if TYPE_CHECKING: + nLength: int + lpSecurityDescriptor: int + bInheritHandle: int # BOOL comes back as 'int'. + + _fields_ = [ + ("nLength", DWORD), + ("lpSecurityDescriptor", LPVOID), + ("bInheritHandle", BOOL), + ] diff --git a/lib/pycparser-3.0.dist-info/INSTALLER b/lib/pycparser-3.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/pycparser-3.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/pycparser-3.0.dist-info/METADATA b/lib/pycparser-3.0.dist-info/METADATA new file mode 100644 index 0000000..b94a88b --- /dev/null +++ b/lib/pycparser-3.0.dist-info/METADATA @@ -0,0 +1,244 @@ +Metadata-Version: 2.4 +Name: pycparser +Version: 3.0 +Summary: C parser in Python +Author-email: Eli Bendersky +Maintainer-email: Eli Bendersky +License-Expression: BSD-3-Clause +Project-URL: Homepage, https://github.com/eliben/pycparser +Classifier: Development Status :: 5 - Production/Stable +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE +Dynamic: license-file + +=============== +pycparser v3.00 +=============== + + +.. image:: https://github.com/eliben/pycparser/workflows/pycparser-tests/badge.svg + :align: center + :target: https://github.com/eliben/pycparser/actions + +---- + +.. contents:: + :backlinks: none + +.. sectnum:: + +Introduction +============ + +What is pycparser? +------------------ + +**pycparser** is a parser for the C language, written in pure Python. It is a +module designed to be easily integrated into applications that need to parse +C source code. + +What is it good for? +-------------------- + +Anything that needs C code to be parsed. The following are some uses for +**pycparser**, taken from real user reports: + +* C code obfuscator +* Front-end for various specialized C compilers +* Static code checker +* Automatic unit-test discovery +* Adding specialized extensions to the C language + +One of the most popular uses of **pycparser** is in the `cffi +`_ library, which uses it to parse the +declarations of C functions and types in order to auto-generate FFIs. + +**pycparser** is unique in the sense that it's written in pure Python - a very +high level language that's easy to experiment with and tweak. To people familiar +with Lex and Yacc, **pycparser**'s code will be simple to understand. It also +has no external dependencies (except for a Python interpreter), making it very +simple to install and deploy. + +Which version of C does pycparser support? +------------------------------------------ + +**pycparser** aims to support the full C99 language (according to the standard +ISO/IEC 9899). Some features from C11 are also supported, and patches to support +more are welcome. + +**pycparser** supports very few GCC extensions, but it's fairly easy to set +things up so that it parses code with a lot of GCC-isms successfully. See the +`FAQ `_ for more details. + +What grammar does pycparser follow? +----------------------------------- + +**pycparser** very closely follows the C grammar provided in Annex A of the C99 +standard (ISO/IEC 9899). + +How is pycparser licensed? +-------------------------- + +`BSD license `_. + +Contact details +--------------- + +For reporting problems with **pycparser** or submitting feature requests, please +open an `issue `_, or submit a +pull request. + + +Installing +========== + +Prerequisites +------------- + +**pycparser** is being tested with modern versions of Python on +Linux, macOS and Windows. See `the CI dashboard `__ +for details. + +**pycparser** has no external dependencies. + +Installation process +-------------------- + +The recommended way to install **pycparser** is with ``pip``:: + + > pip install pycparser + +Using +===== + +Interaction with the C preprocessor +----------------------------------- + +In order to be compilable, C code must be preprocessed by the C preprocessor - +``cpp``. A compatible ``cpp`` handles preprocessing directives like ``#include`` and +``#define``, removes comments, and performs other minor tasks that prepare the C +code for compilation. + +For all but the most trivial snippets of C code **pycparser**, like a C +compiler, must receive preprocessed C code in order to function correctly. If +you import the top-level ``parse_file`` function from the **pycparser** package, +it will interact with ``cpp`` for you, as long as it's in your PATH, or you +provide a path to it. + +Note also that you can use ``gcc -E`` or ``clang -E`` instead of ``cpp``. See +the ``using_gcc_E_libc.py`` example for more details. Windows users can download +and install a binary build of Clang for Windows `from this website +`_. + +What about the standard C library headers? +------------------------------------------ + +C code almost always ``#include``\s various header files from the standard C +library, like ``stdio.h``. While (with some effort) **pycparser** can be made to +parse the standard headers from any C compiler, it's much simpler to use the +provided "fake" standard includes for C11 in ``utils/fake_libc_include``. These +are standard C header files that contain only the bare necessities to allow +valid parsing of the files that use them. As a bonus, since they're minimal, it +can significantly improve the performance of parsing large C files. + +The key point to understand here is that **pycparser** doesn't really care about +the semantics of types. It only needs to know whether some token encountered in +the source is a previously defined type. This is essential in order to be able +to parse C correctly. + +See `this blog post +`_ +for more details. + +Note that the fake headers are not included in the ``pip`` package nor installed +via the package build (`#224 `_). + +Basic usage +----------- + +Take a look at the |examples|_ directory of the distribution for a few examples +of using **pycparser**. These should be enough to get you started. Please note +that most realistic C code samples would require running the C preprocessor +before passing the code to **pycparser**; see the previous sections for more +details. + +.. |examples| replace:: ``examples`` +.. _examples: examples + + +Advanced usage +-------------- + +The public interface of **pycparser** is well documented with comments in +``pycparser/c_parser.py``. For a detailed overview of the various AST nodes +created by the parser, see ``pycparser/_c_ast.cfg``. + +There's also a `FAQ available here `_. +In any case, you can always drop me an `email `_ for help. + + +Modifying +========= + +There are a few points to keep in mind when modifying **pycparser**: + +* The code for **pycparser**'s AST nodes is automatically generated from a + configuration file - ``_c_ast.cfg``, by ``_ast_gen.py``. If you modify the AST + configuration, make sure to re-generate the code. This can be done by running + the ``_ast_gen.py`` script (from the repository root or the + ``pycparser`` directory). +* Read the docstring in the constructor of the ``CParser`` class for details + on configuration and compatibility arguments. + + +Package contents +================ + +Once you unzip the ``pycparser`` package, you'll see the following files and +directories: + +README.rst: + This README file. + +LICENSE: + The pycparser license + +setup.py: + Legacy installation script (build metadata lives in ``pyproject.toml``). + +pyproject.toml: + Package metadata and build configuration. + +examples/: + A directory with some examples of using **pycparser** + +pycparser/: + The **pycparser** module source code. + +tests/: + Unit tests. + +utils/fake_libc_include: + Minimal standard C library include files that should allow to parse any C code. + Note that these headers now include C11 code, so they may not work when the + preprocessor is configured to an earlier C standard (like ``-std=c99``). + +utils/internal/: + Internal utilities for my own use. You probably don't need them. + + +Contributors +============ + +Some people have contributed to **pycparser** by opening issues on bugs they've +found and/or submitting patches. The list of contributors is in the CONTRIBUTORS +file in the source distribution. After **pycparser** moved to Github I stopped +updating this list because Github does a much better job at tracking +contributions. diff --git a/lib/pycparser-3.0.dist-info/RECORD b/lib/pycparser-3.0.dist-info/RECORD new file mode 100644 index 0000000..9f7f900 --- /dev/null +++ b/lib/pycparser-3.0.dist-info/RECORD @@ -0,0 +1,21 @@ +pycparser-3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pycparser-3.0.dist-info/METADATA,sha256=9UemCwq1TMLyQiE9H4eWCtgN991d_rmNF6RE1Iv7a5M,8229 +pycparser-3.0.dist-info/RECORD,, +pycparser-3.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92 +pycparser-3.0.dist-info/licenses/LICENSE,sha256=DIRjmTaep23de1xE_m0WSXQV_PAV9cu1CMJL-YuBxbE,1543 +pycparser-3.0.dist-info/top_level.txt,sha256=c-lPcS74L_8KoH7IE6PQF5ofyirRQNV4VhkbSFIPeWM,10 +pycparser/__init__.py,sha256=phViRyAuUmgqE4kNmaCqpm5WVEBIvzUSFapBv4XX3xo,2829 +pycparser/__pycache__/__init__.cpython-314.pyc,, +pycparser/__pycache__/_ast_gen.cpython-314.pyc,, +pycparser/__pycache__/ast_transforms.cpython-314.pyc,, +pycparser/__pycache__/c_ast.cpython-314.pyc,, +pycparser/__pycache__/c_generator.cpython-314.pyc,, +pycparser/__pycache__/c_lexer.cpython-314.pyc,, +pycparser/__pycache__/c_parser.cpython-314.pyc,, +pycparser/_ast_gen.py,sha256=ExH5Ym4pk7dQPEIkQr9RJim5feztdBQwSBPvpvE-5BM,11292 +pycparser/_c_ast.cfg,sha256=ld5ezE9yzIJFIVAUfw7ezJSlMi4nXKNCzfmqjOyQTNo,4255 +pycparser/ast_transforms.py,sha256=XwMsarc5aDddNWgIiKm4-jOWMRYib96yNQUo0_u28WA,5899 +pycparser/c_ast.py,sha256=uwkcZWHfXDQIw6WDvCL17iWM_-0R-URDqEMmPjXLOAc,32954 +pycparser/c_generator.py,sha256=RVKJPguv2CvovHHSfQkqimckUr9wGU9PofxCGj251QA,20661 +pycparser/c_lexer.py,sha256=B1VoqbYhPWkOJJWCem4OY4zj0IxrPktBZZFe2Y87kUg,25155 +pycparser/c_parser.py,sha256=3FBKGLjjlC3v8afwD_cnMR67ImoYIy73a4WKQR6hX7g,89798 diff --git a/lib/pycparser-3.0.dist-info/WHEEL b/lib/pycparser-3.0.dist-info/WHEEL new file mode 100644 index 0000000..fbbd86c --- /dev/null +++ b/lib/pycparser-3.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.10.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/pycparser-3.0.dist-info/licenses/LICENSE b/lib/pycparser-3.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000..bee14a4 --- /dev/null +++ b/lib/pycparser-3.0.dist-info/licenses/LICENSE @@ -0,0 +1,27 @@ +pycparser -- A C parser in Python + +Copyright (c) 2008-2022, Eli Bendersky +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the copyright holder nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/pycparser-3.0.dist-info/top_level.txt b/lib/pycparser-3.0.dist-info/top_level.txt new file mode 100644 index 0000000..dc1c9e1 --- /dev/null +++ b/lib/pycparser-3.0.dist-info/top_level.txt @@ -0,0 +1 @@ +pycparser diff --git a/lib/pycparser/__init__.py b/lib/pycparser/__init__.py new file mode 100644 index 0000000..0606c03 --- /dev/null +++ b/lib/pycparser/__init__.py @@ -0,0 +1,99 @@ +# ----------------------------------------------------------------- +# pycparser: __init__.py +# +# This package file exports some convenience functions for +# interacting with pycparser +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ----------------------------------------------------------------- +__all__ = ["c_lexer", "c_parser", "c_ast"] +__version__ = "3.00" + +import io +from subprocess import check_output + +from . import c_parser + +CParser = c_parser.CParser + + +def preprocess_file(filename, cpp_path="cpp", cpp_args=""): + """Preprocess a file using cpp. + + filename: + Name of the file you want to preprocess. + + cpp_path: + cpp_args: + Refer to the documentation of parse_file for the meaning of these + arguments. + + When successful, returns the preprocessed file's contents. + Errors from cpp will be printed out. + """ + path_list = [cpp_path] + if isinstance(cpp_args, list): + path_list += cpp_args + elif cpp_args != "": + path_list += [cpp_args] + path_list += [filename] + + try: + # Note the use of universal_newlines to treat all newlines + # as \n for Python's purpose + text = check_output(path_list, universal_newlines=True) + except OSError as e: + raise RuntimeError( + "Unable to invoke 'cpp'. " + + "Make sure its path was passed correctly\n" + + f"Original error: {e}" + ) + + return text + + +def parse_file( + filename, use_cpp=False, cpp_path="cpp", cpp_args="", parser=None, encoding=None +): + """Parse a C file using pycparser. + + filename: + Name of the file you want to parse. + + use_cpp: + Set to True if you want to execute the C pre-processor + on the file prior to parsing it. + + cpp_path: + If use_cpp is True, this is the path to 'cpp' on your + system. If no path is provided, it attempts to just + execute 'cpp', so it must be in your PATH. + + cpp_args: + If use_cpp is True, set this to the command line arguments strings + to cpp. Be careful with quotes - it's best to pass a raw string + (r'') here. For example: + r'-I../utils/fake_libc_include' + If several arguments are required, pass a list of strings. + + encoding: + Encoding to use for the file to parse + + parser: + Optional parser object to be used instead of the default CParser + + When successful, an AST is returned. ParseError can be + thrown if the file doesn't parse successfully. + + Errors from cpp will be printed out. + """ + if use_cpp: + text = preprocess_file(filename, cpp_path, cpp_args) + else: + with io.open(filename, encoding=encoding) as f: + text = f.read() + + if parser is None: + parser = CParser() + return parser.parse(text, filename) diff --git a/lib/pycparser/__pycache__/__init__.cpython-314.pyc b/lib/pycparser/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..4a671d3 Binary files /dev/null and b/lib/pycparser/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/pycparser/__pycache__/_ast_gen.cpython-314.pyc b/lib/pycparser/__pycache__/_ast_gen.cpython-314.pyc new file mode 100644 index 0000000..4fed975 Binary files /dev/null and b/lib/pycparser/__pycache__/_ast_gen.cpython-314.pyc differ diff --git a/lib/pycparser/__pycache__/ast_transforms.cpython-314.pyc b/lib/pycparser/__pycache__/ast_transforms.cpython-314.pyc new file mode 100644 index 0000000..fe4ed14 Binary files /dev/null and b/lib/pycparser/__pycache__/ast_transforms.cpython-314.pyc differ diff --git a/lib/pycparser/__pycache__/c_ast.cpython-314.pyc b/lib/pycparser/__pycache__/c_ast.cpython-314.pyc new file mode 100644 index 0000000..7fe43b4 Binary files /dev/null and b/lib/pycparser/__pycache__/c_ast.cpython-314.pyc differ diff --git a/lib/pycparser/__pycache__/c_generator.cpython-314.pyc b/lib/pycparser/__pycache__/c_generator.cpython-314.pyc new file mode 100644 index 0000000..2993984 Binary files /dev/null and b/lib/pycparser/__pycache__/c_generator.cpython-314.pyc differ diff --git a/lib/pycparser/__pycache__/c_lexer.cpython-314.pyc b/lib/pycparser/__pycache__/c_lexer.cpython-314.pyc new file mode 100644 index 0000000..930ed9d Binary files /dev/null and b/lib/pycparser/__pycache__/c_lexer.cpython-314.pyc differ diff --git a/lib/pycparser/__pycache__/c_parser.cpython-314.pyc b/lib/pycparser/__pycache__/c_parser.cpython-314.pyc new file mode 100644 index 0000000..bb544ec Binary files /dev/null and b/lib/pycparser/__pycache__/c_parser.cpython-314.pyc differ diff --git a/lib/pycparser/_ast_gen.py b/lib/pycparser/_ast_gen.py new file mode 100644 index 0000000..6df0b6a --- /dev/null +++ b/lib/pycparser/_ast_gen.py @@ -0,0 +1,355 @@ +# ----------------------------------------------------------------- +# _ast_gen.py +# +# Generates the AST Node classes from a specification given in +# a configuration file. This module can also be run as a script to +# regenerate c_ast.py from _c_ast.cfg (from the repo root or the +# pycparser/ directory). Use 'make check' to reformat the generated +# file after running this script. +# +# The design of this module was inspired by astgen.py from the +# Python 2.5 code-base. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ----------------------------------------------------------------- +from string import Template +import os +from typing import IO + + +class ASTCodeGenerator: + def __init__(self, cfg_filename="_c_ast.cfg"): + """Initialize the code generator from a configuration + file. + """ + self.cfg_filename = cfg_filename + self.node_cfg = [ + NodeCfg(name, contents) + for (name, contents) in self.parse_cfgfile(cfg_filename) + ] + + def generate(self, file: IO[str]) -> None: + """Generates the code into file, an open file buffer.""" + src = Template(_PROLOGUE_COMMENT).substitute(cfg_filename=self.cfg_filename) + + src += _PROLOGUE_CODE + for node_cfg in self.node_cfg: + src += node_cfg.generate_source() + "\n\n" + + file.write(src) + + def parse_cfgfile(self, filename): + """Parse the configuration file and yield pairs of + (name, contents) for each node. + """ + with open(filename, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + colon_i = line.find(":") + lbracket_i = line.find("[") + rbracket_i = line.find("]") + if colon_i < 1 or lbracket_i <= colon_i or rbracket_i <= lbracket_i: + raise RuntimeError(f"Invalid line in {filename}:\n{line}\n") + + name = line[:colon_i] + val = line[lbracket_i + 1 : rbracket_i] + vallist = [v.strip() for v in val.split(",")] if val else [] + yield name, vallist + + +class NodeCfg: + """Node configuration. + + name: node name + contents: a list of contents - attributes and child nodes + See comment at the top of the configuration file for details. + """ + + def __init__(self, name, contents): + self.name = name + self.all_entries = [] + self.attr = [] + self.child = [] + self.seq_child = [] + + for entry in contents: + clean_entry = entry.rstrip("*") + self.all_entries.append(clean_entry) + + if entry.endswith("**"): + self.seq_child.append(clean_entry) + elif entry.endswith("*"): + self.child.append(clean_entry) + else: + self.attr.append(entry) + + def generate_source(self): + src = self._gen_init() + src += "\n" + self._gen_children() + src += "\n" + self._gen_iter() + src += "\n" + self._gen_attr_names() + return src + + def _gen_init(self): + src = f"class {self.name}(Node):\n" + + if self.all_entries: + args = ", ".join(self.all_entries) + slots = ", ".join(f"'{e}'" for e in self.all_entries) + slots += ", 'coord', '__weakref__'" + arglist = f"(self, {args}, coord=None)" + else: + slots = "'coord', '__weakref__'" + arglist = "(self, coord=None)" + + src += f" __slots__ = ({slots})\n" + src += f" def __init__{arglist}:\n" + + for name in self.all_entries + ["coord"]: + src += f" self.{name} = {name}\n" + + return src + + def _gen_children(self): + src = " def children(self):\n" + + if self.all_entries: + src += " nodelist = []\n" + + for child in self.child: + src += f" if self.{child} is not None:\n" + src += f' nodelist.append(("{child}", self.{child}))\n' + + for seq_child in self.seq_child: + src += f" for i, child in enumerate(self.{seq_child} or []):\n" + src += f' nodelist.append((f"{seq_child}[{{i}}]", child))\n' + + src += " return tuple(nodelist)\n" + else: + src += " return ()\n" + + return src + + def _gen_iter(self): + src = " def __iter__(self):\n" + + if self.all_entries: + for child in self.child: + src += f" if self.{child} is not None:\n" + src += f" yield self.{child}\n" + + for seq_child in self.seq_child: + src += f" for child in (self.{seq_child} or []):\n" + src += " yield child\n" + + if not (self.child or self.seq_child): + # Empty generator + src += " return\n" + " yield\n" + else: + # Empty generator + src += " return\n" + " yield\n" + + return src + + def _gen_attr_names(self): + src = " attr_names = (" + "".join(f"{nm!r}, " for nm in self.attr) + ")" + return src + + +_PROLOGUE_COMMENT = r"""#----------------------------------------------------------------- +# ** ATTENTION ** +# This code was automatically generated from _c_ast.cfg +# +# Do not modify it directly. Modify the configuration file and +# run the generator again. +# ** ** *** ** ** +# +# pycparser: c_ast.py +# +# AST Node classes. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +#----------------------------------------------------------------- + +""" +_PROLOGUE_CODE = r''' +import sys +from typing import Any, ClassVar, IO, Optional + +def _repr(obj): + """ + Get the representation of an object, with dedicated pprint-like format for lists. + """ + if isinstance(obj, list): + return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]' + else: + return repr(obj) + +class Node: + __slots__ = () + """ Abstract base class for AST nodes. + """ + attr_names: ClassVar[tuple[str, ...]] = () + coord: Optional[Any] + def __repr__(self): + """ Generates a python representation of the current node + """ + result = self.__class__.__name__ + '(' + + indent = '' + separator = '' + for name in self.__slots__[:-2]: + result += separator + result += indent + result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n ' + (' ' * (len(name) + len(self.__class__.__name__))))) + + separator = ',' + indent = '\n ' + (' ' * len(self.__class__.__name__)) + + result += indent + ')' + + return result + + def children(self): + """ A sequence of all children that are Nodes + """ + pass + + def show( + self, + buf: IO[str] = sys.stdout, + offset: int = 0, + attrnames: bool = False, + showemptyattrs: bool = True, + nodenames: bool = False, + showcoord: bool = False, + _my_node_name: Optional[str] = None, + ): + """ Pretty print the Node and all its attributes and + children (recursively) to a buffer. + + buf: + Open IO buffer into which the Node is printed. + + offset: + Initial offset (amount of leading spaces) + + attrnames: + True if you want to see the attribute names in + name=value pairs. False to only see the values. + + showemptyattrs: + False if you want to suppress printing empty attributes. + + nodenames: + True if you want to see the actual node names + within their parents. + + showcoord: + Do you want the coordinates of each Node to be + displayed. + """ + lead = ' ' * offset + if nodenames and _my_node_name is not None: + buf.write(lead + self.__class__.__name__+ ' <' + _my_node_name + '>: ') + else: + buf.write(lead + self.__class__.__name__+ ': ') + + if self.attr_names: + def is_empty(v): + v is None or (hasattr(v, '__len__') and len(v) == 0) + nvlist = [(n, getattr(self,n)) for n in self.attr_names \ + if showemptyattrs or not is_empty(getattr(self,n))] + if attrnames: + attrstr = ', '.join(f'{name}={value}' for name, value in nvlist) + else: + attrstr = ', '.join(f'{value}' for _, value in nvlist) + buf.write(attrstr) + + if showcoord: + buf.write(f' (at {self.coord})') + buf.write('\n') + + for (child_name, child) in self.children(): + child.show( + buf, + offset=offset + 2, + attrnames=attrnames, + showemptyattrs=showemptyattrs, + nodenames=nodenames, + showcoord=showcoord, + _my_node_name=child_name) + + +class NodeVisitor: + """ A base NodeVisitor class for visiting c_ast nodes. + Subclass it and define your own visit_XXX methods, where + XXX is the class name you want to visit with these + methods. + + For example: + + class ConstantVisitor(NodeVisitor): + def __init__(self): + self.values = [] + + def visit_Constant(self, node): + self.values.append(node.value) + + Creates a list of values of all the constant nodes + encountered below the given node. To use it: + + cv = ConstantVisitor() + cv.visit(node) + + Notes: + + * generic_visit() will be called for AST nodes for which + no visit_XXX method was defined. + * The children of nodes for which a visit_XXX was + defined will not be visited - if you need this, call + generic_visit() on the node. + You can use: + NodeVisitor.generic_visit(self, node) + * Modeled after Python's own AST visiting facilities + (the ast module of Python 3.0) + """ + + _method_cache = None + + def visit(self, node: Node): + """ Visit a node. + """ + + if self._method_cache is None: + self._method_cache = {} + + visitor = self._method_cache.get(node.__class__.__name__, None) + if visitor is None: + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + self._method_cache[node.__class__.__name__] = visitor + + return visitor(node) + + def generic_visit(self, node: Node): + """ Called if no explicit visitor function exists for a + node. Implements preorder visiting of the node. + """ + for _, c in node.children(): + self.visit(c) + +''' + + +if __name__ == "__main__": + base_dir = os.path.dirname(os.path.abspath(__file__)) + cfg_path = os.path.join(base_dir, "_c_ast.cfg") + out_path = os.path.join(base_dir, "c_ast.py") + ast_gen = ASTCodeGenerator(cfg_path) + with open(out_path, "w") as out: + ast_gen.generate(out) diff --git a/lib/pycparser/_c_ast.cfg b/lib/pycparser/_c_ast.cfg new file mode 100644 index 0000000..0626533 --- /dev/null +++ b/lib/pycparser/_c_ast.cfg @@ -0,0 +1,195 @@ +#----------------------------------------------------------------- +# pycparser: _c_ast.cfg +# +# Defines the AST Node classes used in pycparser. +# +# Each entry is a Node sub-class name, listing the attributes +# and child nodes of the class: +# * - a child node +# ** - a sequence of child nodes +# - an attribute +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +#----------------------------------------------------------------- + +# ArrayDecl is a nested declaration of an array with the given type. +# dim: the dimension (for example, constant 42) +# dim_quals: list of dimension qualifiers, to support C99's allowing 'const' +# and 'static' within the array dimension in function declarations. +ArrayDecl: [type*, dim*, dim_quals] + +ArrayRef: [name*, subscript*] + +# op: =, +=, /= etc. +# +Assignment: [op, lvalue*, rvalue*] + +Alignas: [alignment*] + +BinaryOp: [op, left*, right*] + +Break: [] + +Case: [expr*, stmts**] + +Cast: [to_type*, expr*] + +# Compound statement in C99 is a list of block items (declarations or +# statements). +# +Compound: [block_items**] + +# Compound literal (anonymous aggregate) for C99. +# (type-name) {initializer_list} +# type: the typename +# init: InitList for the initializer list +# +CompoundLiteral: [type*, init*] + +# type: int, char, float, string, etc. +# +Constant: [type, value] + +Continue: [] + +# name: the variable being declared +# quals: list of qualifiers (const, volatile) +# funcspec: list function specifiers (i.e. inline in C99) +# storage: list of storage specifiers (extern, register, etc.) +# type: declaration type (probably nested with all the modifiers) +# init: initialization value, or None +# bitsize: bit field size, or None +# +Decl: [name, quals, align, storage, funcspec, type*, init*, bitsize*] + +DeclList: [decls**] + +Default: [stmts**] + +DoWhile: [cond*, stmt*] + +# Represents the ellipsis (...) parameter in a function +# declaration +# +EllipsisParam: [] + +# An empty statement (a semicolon ';' on its own) +# +EmptyStatement: [] + +# Enumeration type specifier +# name: an optional ID +# values: an EnumeratorList +# +Enum: [name, values*] + +# A name/value pair for enumeration values +# +Enumerator: [name, value*] + +# A list of enumerators +# +EnumeratorList: [enumerators**] + +# A list of expressions separated by the comma operator. +# +ExprList: [exprs**] + +# This is the top of the AST, representing a single C file (a +# translation unit in K&R jargon). It contains a list of +# "external-declaration"s, which is either declarations (Decl), +# Typedef or function definitions (FuncDef). +# +FileAST: [ext**] + +# for (init; cond; next) stmt +# +For: [init*, cond*, next*, stmt*] + +# name: Id +# args: ExprList +# +FuncCall: [name*, args*] + +# type (args) +# +FuncDecl: [args*, type*] + +# Function definition: a declarator for the function name and +# a body, which is a compound statement. +# There's an optional list of parameter declarations for old +# K&R-style definitions +# +FuncDef: [decl*, param_decls**, body*] + +Goto: [name] + +ID: [name] + +# Holder for types that are a simple identifier (e.g. the built +# ins void, char etc. and typedef-defined types) +# +IdentifierType: [names] + +If: [cond*, iftrue*, iffalse*] + +# An initialization list used for compound literals. +# +InitList: [exprs**] + +Label: [name, stmt*] + +# A named initializer for C99. +# The name of a NamedInitializer is a sequence of Nodes, because +# names can be hierarchical and contain constant expressions. +# +NamedInitializer: [name**, expr*] + +# a list of comma separated function parameter declarations +# +ParamList: [params**] + +PtrDecl: [quals, type*] + +Return: [expr*] + +StaticAssert: [cond*, message*] + +# name: struct tag name +# decls: declaration of members +# +Struct: [name, decls**] + +# type: . or -> +# name.field or name->field +# +StructRef: [name*, type, field*] + +Switch: [cond*, stmt*] + +# cond ? iftrue : iffalse +# +TernaryOp: [cond*, iftrue*, iffalse*] + +# A base type declaration +# +TypeDecl: [declname, quals, align, type*] + +# A typedef declaration. +# Very similar to Decl, but without some attributes +# +Typedef: [name, quals, storage, type*] + +Typename: [name, quals, align, type*] + +UnaryOp: [op, expr*] + +# name: union tag name +# decls: declaration of members +# +Union: [name, decls**] + +While: [cond*, stmt*] + +Pragma: [string] diff --git a/lib/pycparser/ast_transforms.py b/lib/pycparser/ast_transforms.py new file mode 100644 index 0000000..1051737 --- /dev/null +++ b/lib/pycparser/ast_transforms.py @@ -0,0 +1,174 @@ +# ------------------------------------------------------------------------------ +# pycparser: ast_transforms.py +# +# Some utilities used by the parser to create a friendlier AST. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ + +from typing import Any, List, Tuple, cast + +from . import c_ast + + +def fix_switch_cases(switch_node: c_ast.Switch) -> c_ast.Switch: + """The 'case' statements in a 'switch' come out of parsing with one + child node, so subsequent statements are just tucked to the parent + Compound. Additionally, consecutive (fall-through) case statements + come out messy. This is a peculiarity of the C grammar. The following: + + switch (myvar) { + case 10: + k = 10; + p = k + 1; + return 10; + case 20: + case 30: + return 20; + default: + break; + } + + Creates this tree (pseudo-dump): + + Switch + ID: myvar + Compound: + Case 10: + k = 10 + p = k + 1 + return 10 + Case 20: + Case 30: + return 20 + Default: + break + + The goal of this transform is to fix this mess, turning it into the + following: + + Switch + ID: myvar + Compound: + Case 10: + k = 10 + p = k + 1 + return 10 + Case 20: + Case 30: + return 20 + Default: + break + + A fixed AST node is returned. The argument may be modified. + """ + assert isinstance(switch_node, c_ast.Switch) + if not isinstance(switch_node.stmt, c_ast.Compound): + return switch_node + + # The new Compound child for the Switch, which will collect children in the + # correct order + new_compound = c_ast.Compound([], switch_node.stmt.coord) + + # The last Case/Default node + last_case: c_ast.Case | c_ast.Default | None = None + + # Goes over the children of the Compound below the Switch, adding them + # either directly below new_compound or below the last Case as appropriate + # (for `switch(cond) {}`, block_items would have been None) + for child in switch_node.stmt.block_items or []: + if isinstance(child, (c_ast.Case, c_ast.Default)): + # If it's a Case/Default: + # 1. Add it to the Compound and mark as "last case" + # 2. If its immediate child is also a Case or Default, promote it + # to a sibling. + new_compound.block_items.append(child) + _extract_nested_case(child, new_compound.block_items) + last_case = new_compound.block_items[-1] + else: + # Other statements are added as children to the last case, if it + # exists. + if last_case is None: + new_compound.block_items.append(child) + else: + last_case.stmts.append(child) + + switch_node.stmt = new_compound + return switch_node + + +def _extract_nested_case( + case_node: c_ast.Case | c_ast.Default, stmts_list: List[c_ast.Node] +) -> None: + """Recursively extract consecutive Case statements that are made nested + by the parser and add them to the stmts_list. + """ + if isinstance(case_node.stmts[0], (c_ast.Case, c_ast.Default)): + nested = case_node.stmts.pop() + stmts_list.append(nested) + _extract_nested_case(cast(Any, nested), stmts_list) + + +def fix_atomic_specifiers( + decl: c_ast.Decl | c_ast.Typedef, +) -> c_ast.Decl | c_ast.Typedef: + """Atomic specifiers like _Atomic(type) are unusually structured, + conferring a qualifier upon the contained type. + + This function fixes a decl with atomic specifiers to have a sane AST + structure, by removing spurious Typename->TypeDecl pairs and attaching + the _Atomic qualifier in the right place. + """ + # There can be multiple levels of _Atomic in a decl; fix them until a + # fixed point is reached. + while True: + decl, found = _fix_atomic_specifiers_once(decl) + if not found: + break + + # Make sure to add an _Atomic qual on the topmost decl if needed. Also + # restore the declname on the innermost TypeDecl (it gets placed in the + # wrong place during construction). + typ: Any = decl + while not isinstance(typ, c_ast.TypeDecl): + try: + typ = typ.type + except AttributeError: + return decl + if "_Atomic" in typ.quals and "_Atomic" not in decl.quals: + decl.quals.append("_Atomic") + if typ.declname is None: + typ.declname = decl.name + + return decl + + +def _fix_atomic_specifiers_once( + decl: c_ast.Decl | c_ast.Typedef, +) -> Tuple[c_ast.Decl | c_ast.Typedef, bool]: + """Performs one 'fix' round of atomic specifiers. + Returns (modified_decl, found) where found is True iff a fix was made. + """ + parent: Any = decl + grandparent: Any = None + node: Any = decl.type + while node is not None: + if isinstance(node, c_ast.Typename) and "_Atomic" in node.quals: + break + try: + grandparent = parent + parent = node + node = node.type + except AttributeError: + # If we've reached a node without a `type` field, it means we won't + # find what we're looking for at this point; give up the search + # and return the original decl unmodified. + return decl, False + + assert isinstance(parent, c_ast.TypeDecl) + assert grandparent is not None + cast(Any, grandparent).type = node.type + if "_Atomic" not in node.type.quals: + node.type.quals.append("_Atomic") + return decl, True diff --git a/lib/pycparser/c_ast.py b/lib/pycparser/c_ast.py new file mode 100644 index 0000000..b6f42af --- /dev/null +++ b/lib/pycparser/c_ast.py @@ -0,0 +1,1341 @@ +# ----------------------------------------------------------------- +# ** ATTENTION ** +# This code was automatically generated from _c_ast.cfg +# +# Do not modify it directly. Modify the configuration file and +# run the generator again. +# ** ** *** ** ** +# +# pycparser: c_ast.py +# +# AST Node classes. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ----------------------------------------------------------------- + + +import sys +from typing import Any, ClassVar, IO, Optional + + +def _repr(obj): + """ + Get the representation of an object, with dedicated pprint-like format for lists. + """ + if isinstance(obj, list): + return "[" + (",\n ".join((_repr(e).replace("\n", "\n ") for e in obj))) + "\n]" + else: + return repr(obj) + + +class Node: + __slots__ = () + """ Abstract base class for AST nodes. + """ + attr_names: ClassVar[tuple[str, ...]] = () + coord: Optional[Any] + + def __repr__(self): + """Generates a python representation of the current node""" + result = self.__class__.__name__ + "(" + + indent = "" + separator = "" + for name in self.__slots__[:-2]: + result += separator + result += indent + result += ( + name + + "=" + + ( + _repr(getattr(self, name)).replace( + "\n", + "\n " + (" " * (len(name) + len(self.__class__.__name__))), + ) + ) + ) + + separator = "," + indent = "\n " + (" " * len(self.__class__.__name__)) + + result += indent + ")" + + return result + + def children(self): + """A sequence of all children that are Nodes""" + pass + + def show( + self, + buf: IO[str] = sys.stdout, + offset: int = 0, + attrnames: bool = False, + showemptyattrs: bool = True, + nodenames: bool = False, + showcoord: bool = False, + _my_node_name: Optional[str] = None, + ): + """Pretty print the Node and all its attributes and + children (recursively) to a buffer. + + buf: + Open IO buffer into which the Node is printed. + + offset: + Initial offset (amount of leading spaces) + + attrnames: + True if you want to see the attribute names in + name=value pairs. False to only see the values. + + showemptyattrs: + False if you want to suppress printing empty attributes. + + nodenames: + True if you want to see the actual node names + within their parents. + + showcoord: + Do you want the coordinates of each Node to be + displayed. + """ + lead = " " * offset + if nodenames and _my_node_name is not None: + buf.write(lead + self.__class__.__name__ + " <" + _my_node_name + ">: ") + else: + buf.write(lead + self.__class__.__name__ + ": ") + + if self.attr_names: + + def is_empty(v): + v is None or (hasattr(v, "__len__") and len(v) == 0) + + nvlist = [ + (n, getattr(self, n)) + for n in self.attr_names + if showemptyattrs or not is_empty(getattr(self, n)) + ] + if attrnames: + attrstr = ", ".join(f"{name}={value}" for name, value in nvlist) + else: + attrstr = ", ".join(f"{value}" for _, value in nvlist) + buf.write(attrstr) + + if showcoord: + buf.write(f" (at {self.coord})") + buf.write("\n") + + for child_name, child in self.children(): + child.show( + buf, + offset=offset + 2, + attrnames=attrnames, + showemptyattrs=showemptyattrs, + nodenames=nodenames, + showcoord=showcoord, + _my_node_name=child_name, + ) + + +class NodeVisitor: + """A base NodeVisitor class for visiting c_ast nodes. + Subclass it and define your own visit_XXX methods, where + XXX is the class name you want to visit with these + methods. + + For example: + + class ConstantVisitor(NodeVisitor): + def __init__(self): + self.values = [] + + def visit_Constant(self, node): + self.values.append(node.value) + + Creates a list of values of all the constant nodes + encountered below the given node. To use it: + + cv = ConstantVisitor() + cv.visit(node) + + Notes: + + * generic_visit() will be called for AST nodes for which + no visit_XXX method was defined. + * The children of nodes for which a visit_XXX was + defined will not be visited - if you need this, call + generic_visit() on the node. + You can use: + NodeVisitor.generic_visit(self, node) + * Modeled after Python's own AST visiting facilities + (the ast module of Python 3.0) + """ + + _method_cache = None + + def visit(self, node: Node): + """Visit a node.""" + + if self._method_cache is None: + self._method_cache = {} + + visitor = self._method_cache.get(node.__class__.__name__, None) + if visitor is None: + method = "visit_" + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + self._method_cache[node.__class__.__name__] = visitor + + return visitor(node) + + def generic_visit(self, node: Node): + """Called if no explicit visitor function exists for a + node. Implements preorder visiting of the node. + """ + for _, c in node.children(): + self.visit(c) + + +class ArrayDecl(Node): + __slots__ = ("type", "dim", "dim_quals", "coord", "__weakref__") + + def __init__(self, type, dim, dim_quals, coord=None): + self.type = type + self.dim = dim + self.dim_quals = dim_quals + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + if self.dim is not None: + nodelist.append(("dim", self.dim)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + if self.dim is not None: + yield self.dim + + attr_names = ("dim_quals",) + + +class ArrayRef(Node): + __slots__ = ("name", "subscript", "coord", "__weakref__") + + def __init__(self, name, subscript, coord=None): + self.name = name + self.subscript = subscript + self.coord = coord + + def children(self): + nodelist = [] + if self.name is not None: + nodelist.append(("name", self.name)) + if self.subscript is not None: + nodelist.append(("subscript", self.subscript)) + return tuple(nodelist) + + def __iter__(self): + if self.name is not None: + yield self.name + if self.subscript is not None: + yield self.subscript + + attr_names = () + + +class Assignment(Node): + __slots__ = ("op", "lvalue", "rvalue", "coord", "__weakref__") + + def __init__(self, op, lvalue, rvalue, coord=None): + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + self.coord = coord + + def children(self): + nodelist = [] + if self.lvalue is not None: + nodelist.append(("lvalue", self.lvalue)) + if self.rvalue is not None: + nodelist.append(("rvalue", self.rvalue)) + return tuple(nodelist) + + def __iter__(self): + if self.lvalue is not None: + yield self.lvalue + if self.rvalue is not None: + yield self.rvalue + + attr_names = ("op",) + + +class Alignas(Node): + __slots__ = ("alignment", "coord", "__weakref__") + + def __init__(self, alignment, coord=None): + self.alignment = alignment + self.coord = coord + + def children(self): + nodelist = [] + if self.alignment is not None: + nodelist.append(("alignment", self.alignment)) + return tuple(nodelist) + + def __iter__(self): + if self.alignment is not None: + yield self.alignment + + attr_names = () + + +class BinaryOp(Node): + __slots__ = ("op", "left", "right", "coord", "__weakref__") + + def __init__(self, op, left, right, coord=None): + self.op = op + self.left = left + self.right = right + self.coord = coord + + def children(self): + nodelist = [] + if self.left is not None: + nodelist.append(("left", self.left)) + if self.right is not None: + nodelist.append(("right", self.right)) + return tuple(nodelist) + + def __iter__(self): + if self.left is not None: + yield self.left + if self.right is not None: + yield self.right + + attr_names = ("op",) + + +class Break(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class Case(Node): + __slots__ = ("expr", "stmts", "coord", "__weakref__") + + def __init__(self, expr, stmts, coord=None): + self.expr = expr + self.stmts = stmts + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + for i, child in enumerate(self.stmts or []): + nodelist.append((f"stmts[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + for child in self.stmts or []: + yield child + + attr_names = () + + +class Cast(Node): + __slots__ = ("to_type", "expr", "coord", "__weakref__") + + def __init__(self, to_type, expr, coord=None): + self.to_type = to_type + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.to_type is not None: + nodelist.append(("to_type", self.to_type)) + if self.expr is not None: + nodelist.append(("expr", self.expr)) + return tuple(nodelist) + + def __iter__(self): + if self.to_type is not None: + yield self.to_type + if self.expr is not None: + yield self.expr + + attr_names = () + + +class Compound(Node): + __slots__ = ("block_items", "coord", "__weakref__") + + def __init__(self, block_items, coord=None): + self.block_items = block_items + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.block_items or []): + nodelist.append((f"block_items[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.block_items or []: + yield child + + attr_names = () + + +class CompoundLiteral(Node): + __slots__ = ("type", "init", "coord", "__weakref__") + + def __init__(self, type, init, coord=None): + self.type = type + self.init = init + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + if self.init is not None: + nodelist.append(("init", self.init)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + if self.init is not None: + yield self.init + + attr_names = () + + +class Constant(Node): + __slots__ = ("type", "value", "coord", "__weakref__") + + def __init__(self, type, value, coord=None): + self.type = type + self.value = value + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ( + "type", + "value", + ) + + +class Continue(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class Decl(Node): + __slots__ = ( + "name", + "quals", + "align", + "storage", + "funcspec", + "type", + "init", + "bitsize", + "coord", + "__weakref__", + ) + + def __init__( + self, name, quals, align, storage, funcspec, type, init, bitsize, coord=None + ): + self.name = name + self.quals = quals + self.align = align + self.storage = storage + self.funcspec = funcspec + self.type = type + self.init = init + self.bitsize = bitsize + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + if self.init is not None: + nodelist.append(("init", self.init)) + if self.bitsize is not None: + nodelist.append(("bitsize", self.bitsize)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + if self.init is not None: + yield self.init + if self.bitsize is not None: + yield self.bitsize + + attr_names = ( + "name", + "quals", + "align", + "storage", + "funcspec", + ) + + +class DeclList(Node): + __slots__ = ("decls", "coord", "__weakref__") + + def __init__(self, decls, coord=None): + self.decls = decls + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.decls or []): + nodelist.append((f"decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.decls or []: + yield child + + attr_names = () + + +class Default(Node): + __slots__ = ("stmts", "coord", "__weakref__") + + def __init__(self, stmts, coord=None): + self.stmts = stmts + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.stmts or []): + nodelist.append((f"stmts[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.stmts or []: + yield child + + attr_names = () + + +class DoWhile(Node): + __slots__ = ("cond", "stmt", "coord", "__weakref__") + + def __init__(self, cond, stmt, coord=None): + self.cond = cond + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class EllipsisParam(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class EmptyStatement(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class Enum(Node): + __slots__ = ("name", "values", "coord", "__weakref__") + + def __init__(self, name, values, coord=None): + self.name = name + self.values = values + self.coord = coord + + def children(self): + nodelist = [] + if self.values is not None: + nodelist.append(("values", self.values)) + return tuple(nodelist) + + def __iter__(self): + if self.values is not None: + yield self.values + + attr_names = ("name",) + + +class Enumerator(Node): + __slots__ = ("name", "value", "coord", "__weakref__") + + def __init__(self, name, value, coord=None): + self.name = name + self.value = value + self.coord = coord + + def children(self): + nodelist = [] + if self.value is not None: + nodelist.append(("value", self.value)) + return tuple(nodelist) + + def __iter__(self): + if self.value is not None: + yield self.value + + attr_names = ("name",) + + +class EnumeratorList(Node): + __slots__ = ("enumerators", "coord", "__weakref__") + + def __init__(self, enumerators, coord=None): + self.enumerators = enumerators + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.enumerators or []): + nodelist.append((f"enumerators[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.enumerators or []: + yield child + + attr_names = () + + +class ExprList(Node): + __slots__ = ("exprs", "coord", "__weakref__") + + def __init__(self, exprs, coord=None): + self.exprs = exprs + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.exprs or []): + nodelist.append((f"exprs[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.exprs or []: + yield child + + attr_names = () + + +class FileAST(Node): + __slots__ = ("ext", "coord", "__weakref__") + + def __init__(self, ext, coord=None): + self.ext = ext + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.ext or []): + nodelist.append((f"ext[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.ext or []: + yield child + + attr_names = () + + +class For(Node): + __slots__ = ("init", "cond", "next", "stmt", "coord", "__weakref__") + + def __init__(self, init, cond, next, stmt, coord=None): + self.init = init + self.cond = cond + self.next = next + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.init is not None: + nodelist.append(("init", self.init)) + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.next is not None: + nodelist.append(("next", self.next)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.init is not None: + yield self.init + if self.cond is not None: + yield self.cond + if self.next is not None: + yield self.next + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class FuncCall(Node): + __slots__ = ("name", "args", "coord", "__weakref__") + + def __init__(self, name, args, coord=None): + self.name = name + self.args = args + self.coord = coord + + def children(self): + nodelist = [] + if self.name is not None: + nodelist.append(("name", self.name)) + if self.args is not None: + nodelist.append(("args", self.args)) + return tuple(nodelist) + + def __iter__(self): + if self.name is not None: + yield self.name + if self.args is not None: + yield self.args + + attr_names = () + + +class FuncDecl(Node): + __slots__ = ("args", "type", "coord", "__weakref__") + + def __init__(self, args, type, coord=None): + self.args = args + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.args is not None: + nodelist.append(("args", self.args)) + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.args is not None: + yield self.args + if self.type is not None: + yield self.type + + attr_names = () + + +class FuncDef(Node): + __slots__ = ("decl", "param_decls", "body", "coord", "__weakref__") + + def __init__(self, decl, param_decls, body, coord=None): + self.decl = decl + self.param_decls = param_decls + self.body = body + self.coord = coord + + def children(self): + nodelist = [] + if self.decl is not None: + nodelist.append(("decl", self.decl)) + if self.body is not None: + nodelist.append(("body", self.body)) + for i, child in enumerate(self.param_decls or []): + nodelist.append((f"param_decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + if self.decl is not None: + yield self.decl + if self.body is not None: + yield self.body + for child in self.param_decls or []: + yield child + + attr_names = () + + +class Goto(Node): + __slots__ = ("name", "coord", "__weakref__") + + def __init__(self, name, coord=None): + self.name = name + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("name",) + + +class ID(Node): + __slots__ = ("name", "coord", "__weakref__") + + def __init__(self, name, coord=None): + self.name = name + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("name",) + + +class IdentifierType(Node): + __slots__ = ("names", "coord", "__weakref__") + + def __init__(self, names, coord=None): + self.names = names + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("names",) + + +class If(Node): + __slots__ = ("cond", "iftrue", "iffalse", "coord", "__weakref__") + + def __init__(self, cond, iftrue, iffalse, coord=None): + self.cond = cond + self.iftrue = iftrue + self.iffalse = iffalse + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.iftrue is not None: + nodelist.append(("iftrue", self.iftrue)) + if self.iffalse is not None: + nodelist.append(("iffalse", self.iffalse)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.iftrue is not None: + yield self.iftrue + if self.iffalse is not None: + yield self.iffalse + + attr_names = () + + +class InitList(Node): + __slots__ = ("exprs", "coord", "__weakref__") + + def __init__(self, exprs, coord=None): + self.exprs = exprs + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.exprs or []): + nodelist.append((f"exprs[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.exprs or []: + yield child + + attr_names = () + + +class Label(Node): + __slots__ = ("name", "stmt", "coord", "__weakref__") + + def __init__(self, name, stmt, coord=None): + self.name = name + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.stmt is not None: + yield self.stmt + + attr_names = ("name",) + + +class NamedInitializer(Node): + __slots__ = ("name", "expr", "coord", "__weakref__") + + def __init__(self, name, expr, coord=None): + self.name = name + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + for i, child in enumerate(self.name or []): + nodelist.append((f"name[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + for child in self.name or []: + yield child + + attr_names = () + + +class ParamList(Node): + __slots__ = ("params", "coord", "__weakref__") + + def __init__(self, params, coord=None): + self.params = params + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.params or []): + nodelist.append((f"params[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.params or []: + yield child + + attr_names = () + + +class PtrDecl(Node): + __slots__ = ("quals", "type", "coord", "__weakref__") + + def __init__(self, quals, type, coord=None): + self.quals = quals + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ("quals",) + + +class Return(Node): + __slots__ = ("expr", "coord", "__weakref__") + + def __init__(self, expr, coord=None): + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + + attr_names = () + + +class StaticAssert(Node): + __slots__ = ("cond", "message", "coord", "__weakref__") + + def __init__(self, cond, message, coord=None): + self.cond = cond + self.message = message + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.message is not None: + nodelist.append(("message", self.message)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.message is not None: + yield self.message + + attr_names = () + + +class Struct(Node): + __slots__ = ("name", "decls", "coord", "__weakref__") + + def __init__(self, name, decls, coord=None): + self.name = name + self.decls = decls + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.decls or []): + nodelist.append((f"decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.decls or []: + yield child + + attr_names = ("name",) + + +class StructRef(Node): + __slots__ = ("name", "type", "field", "coord", "__weakref__") + + def __init__(self, name, type, field, coord=None): + self.name = name + self.type = type + self.field = field + self.coord = coord + + def children(self): + nodelist = [] + if self.name is not None: + nodelist.append(("name", self.name)) + if self.field is not None: + nodelist.append(("field", self.field)) + return tuple(nodelist) + + def __iter__(self): + if self.name is not None: + yield self.name + if self.field is not None: + yield self.field + + attr_names = ("type",) + + +class Switch(Node): + __slots__ = ("cond", "stmt", "coord", "__weakref__") + + def __init__(self, cond, stmt, coord=None): + self.cond = cond + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class TernaryOp(Node): + __slots__ = ("cond", "iftrue", "iffalse", "coord", "__weakref__") + + def __init__(self, cond, iftrue, iffalse, coord=None): + self.cond = cond + self.iftrue = iftrue + self.iffalse = iffalse + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.iftrue is not None: + nodelist.append(("iftrue", self.iftrue)) + if self.iffalse is not None: + nodelist.append(("iffalse", self.iffalse)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.iftrue is not None: + yield self.iftrue + if self.iffalse is not None: + yield self.iffalse + + attr_names = () + + +class TypeDecl(Node): + __slots__ = ("declname", "quals", "align", "type", "coord", "__weakref__") + + def __init__(self, declname, quals, align, type, coord=None): + self.declname = declname + self.quals = quals + self.align = align + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ( + "declname", + "quals", + "align", + ) + + +class Typedef(Node): + __slots__ = ("name", "quals", "storage", "type", "coord", "__weakref__") + + def __init__(self, name, quals, storage, type, coord=None): + self.name = name + self.quals = quals + self.storage = storage + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ( + "name", + "quals", + "storage", + ) + + +class Typename(Node): + __slots__ = ("name", "quals", "align", "type", "coord", "__weakref__") + + def __init__(self, name, quals, align, type, coord=None): + self.name = name + self.quals = quals + self.align = align + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ( + "name", + "quals", + "align", + ) + + +class UnaryOp(Node): + __slots__ = ("op", "expr", "coord", "__weakref__") + + def __init__(self, op, expr, coord=None): + self.op = op + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + + attr_names = ("op",) + + +class Union(Node): + __slots__ = ("name", "decls", "coord", "__weakref__") + + def __init__(self, name, decls, coord=None): + self.name = name + self.decls = decls + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.decls or []): + nodelist.append((f"decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.decls or []: + yield child + + attr_names = ("name",) + + +class While(Node): + __slots__ = ("cond", "stmt", "coord", "__weakref__") + + def __init__(self, cond, stmt, coord=None): + self.cond = cond + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class Pragma(Node): + __slots__ = ("string", "coord", "__weakref__") + + def __init__(self, string, coord=None): + self.string = string + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("string",) diff --git a/lib/pycparser/c_generator.py b/lib/pycparser/c_generator.py new file mode 100644 index 0000000..424e00e --- /dev/null +++ b/lib/pycparser/c_generator.py @@ -0,0 +1,573 @@ +# ------------------------------------------------------------------------------ +# pycparser: c_generator.py +# +# C code generator from pycparser AST nodes. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ +from typing import Callable, List, Optional + +from . import c_ast + + +class CGenerator: + """Uses the same visitor pattern as c_ast.NodeVisitor, but modified to + return a value from each visit method, using string accumulation in + generic_visit. + """ + + indent_level: int + reduce_parentheses: bool + + def __init__(self, reduce_parentheses: bool = False) -> None: + """Constructs C-code generator + + reduce_parentheses: + if True, eliminates needless parentheses on binary operators + """ + # Statements start with indentation of self.indent_level spaces, using + # the _make_indent method. + self.indent_level = 0 + self.reduce_parentheses = reduce_parentheses + + def _make_indent(self) -> str: + return " " * self.indent_level + + def visit(self, node: c_ast.Node) -> str: + method = "visit_" + node.__class__.__name__ + return getattr(self, method, self.generic_visit)(node) + + def generic_visit(self, node: Optional[c_ast.Node]) -> str: + if node is None: + return "" + else: + return "".join(self.visit(c) for c_name, c in node.children()) + + def visit_Constant(self, n: c_ast.Constant) -> str: + return n.value + + def visit_ID(self, n: c_ast.ID) -> str: + return n.name + + def visit_Pragma(self, n: c_ast.Pragma) -> str: + ret = "#pragma" + if n.string: + ret += " " + n.string + return ret + + def visit_ArrayRef(self, n: c_ast.ArrayRef) -> str: + arrref = self._parenthesize_unless_simple(n.name) + return arrref + "[" + self.visit(n.subscript) + "]" + + def visit_StructRef(self, n: c_ast.StructRef) -> str: + sref = self._parenthesize_unless_simple(n.name) + return sref + n.type + self.visit(n.field) + + def visit_FuncCall(self, n: c_ast.FuncCall) -> str: + fref = self._parenthesize_unless_simple(n.name) + args = self.visit(n.args) if n.args is not None else "" + return fref + "(" + args + ")" + + def visit_UnaryOp(self, n: c_ast.UnaryOp) -> str: + match n.op: + case "sizeof": + # Always parenthesize the argument of sizeof since it can be + # a name. + return f"sizeof({self.visit(n.expr)})" + case "p++": + operand = self._parenthesize_unless_simple(n.expr) + return f"{operand}++" + case "p--": + operand = self._parenthesize_unless_simple(n.expr) + return f"{operand}--" + case _: + operand = self._parenthesize_unless_simple(n.expr) + return f"{n.op}{operand}" + + # Precedence map of binary operators: + precedence_map = { + # Should be in sync with c_parser.CParser.precedence + # Higher numbers are stronger binding + "||": 0, # weakest binding + "&&": 1, + "|": 2, + "^": 3, + "&": 4, + "==": 5, + "!=": 5, + ">": 6, + ">=": 6, + "<": 6, + "<=": 6, + ">>": 7, + "<<": 7, + "+": 8, + "-": 8, + "*": 9, + "/": 9, + "%": 9, # strongest binding + } + + def visit_BinaryOp(self, n: c_ast.BinaryOp) -> str: + # Note: all binary operators are left-to-right associative + # + # If `n.left.op` has a stronger or equally binding precedence in + # comparison to `n.op`, no parenthesis are needed for the left: + # e.g., `(a*b) + c` is equivalent to `a*b + c`, as well as + # `(a+b) - c` is equivalent to `a+b - c` (same precedence). + # If the left operator is weaker binding than the current, then + # parentheses are necessary: + # e.g., `(a+b) * c` is NOT equivalent to `a+b * c`. + lval_str = self._parenthesize_if( + n.left, + lambda d: not ( + self._is_simple_node(d) + or self.reduce_parentheses + and isinstance(d, c_ast.BinaryOp) + and self.precedence_map[d.op] >= self.precedence_map[n.op] + ), + ) + # If `n.right.op` has a stronger -but not equal- binding precedence, + # parenthesis can be omitted on the right: + # e.g., `a + (b*c)` is equivalent to `a + b*c`. + # If the right operator is weaker or equally binding, then parentheses + # are necessary: + # e.g., `a * (b+c)` is NOT equivalent to `a * b+c` and + # `a - (b+c)` is NOT equivalent to `a - b+c` (same precedence). + rval_str = self._parenthesize_if( + n.right, + lambda d: not ( + self._is_simple_node(d) + or self.reduce_parentheses + and isinstance(d, c_ast.BinaryOp) + and self.precedence_map[d.op] > self.precedence_map[n.op] + ), + ) + return f"{lval_str} {n.op} {rval_str}" + + def visit_Assignment(self, n: c_ast.Assignment) -> str: + rval_str = self._parenthesize_if( + n.rvalue, lambda n: isinstance(n, c_ast.Assignment) + ) + return f"{self.visit(n.lvalue)} {n.op} {rval_str}" + + def visit_IdentifierType(self, n: c_ast.IdentifierType) -> str: + return " ".join(n.names) + + def _visit_expr(self, n: c_ast.Node) -> str: + match n: + case c_ast.InitList(): + return "{" + self.visit(n) + "}" + case c_ast.ExprList() | c_ast.Compound(): + return "(" + self.visit(n) + ")" + case _: + return self.visit(n) + + def visit_Decl(self, n: c_ast.Decl, no_type: bool = False) -> str: + # no_type is used when a Decl is part of a DeclList, where the type is + # explicitly only for the first declaration in a list. + # + s = n.name if no_type else self._generate_decl(n) + if n.bitsize: + s += " : " + self.visit(n.bitsize) + if n.init: + s += " = " + self._visit_expr(n.init) + return s + + def visit_DeclList(self, n: c_ast.DeclList) -> str: + s = self.visit(n.decls[0]) + if len(n.decls) > 1: + s += ", " + ", ".join( + self.visit_Decl(decl, no_type=True) for decl in n.decls[1:] + ) + return s + + def visit_Typedef(self, n: c_ast.Typedef) -> str: + s = "" + if n.storage: + s += " ".join(n.storage) + " " + s += self._generate_type(n.type) + return s + + def visit_Cast(self, n: c_ast.Cast) -> str: + s = "(" + self._generate_type(n.to_type, emit_declname=False) + ")" + return s + " " + self._parenthesize_unless_simple(n.expr) + + def visit_ExprList(self, n: c_ast.ExprList) -> str: + visited_subexprs = [] + for expr in n.exprs: + visited_subexprs.append(self._visit_expr(expr)) + return ", ".join(visited_subexprs) + + def visit_InitList(self, n: c_ast.InitList) -> str: + visited_subexprs = [] + for expr in n.exprs: + visited_subexprs.append(self._visit_expr(expr)) + return ", ".join(visited_subexprs) + + def visit_Enum(self, n: c_ast.Enum) -> str: + return self._generate_struct_union_enum(n, name="enum") + + def visit_Alignas(self, n: c_ast.Alignas) -> str: + return "_Alignas({})".format(self.visit(n.alignment)) + + def visit_Enumerator(self, n: c_ast.Enumerator) -> str: + if not n.value: + return "{indent}{name},\n".format( + indent=self._make_indent(), + name=n.name, + ) + else: + return "{indent}{name} = {value},\n".format( + indent=self._make_indent(), + name=n.name, + value=self.visit(n.value), + ) + + def visit_FuncDef(self, n: c_ast.FuncDef) -> str: + decl = self.visit(n.decl) + self.indent_level = 0 + body = self.visit(n.body) + if n.param_decls: + knrdecls = ";\n".join(self.visit(p) for p in n.param_decls) + return decl + "\n" + knrdecls + ";\n" + body + "\n" + else: + return decl + "\n" + body + "\n" + + def visit_FileAST(self, n: c_ast.FileAST) -> str: + s = "" + for ext in n.ext: + match ext: + case c_ast.FuncDef(): + s += self.visit(ext) + case c_ast.Pragma(): + s += self.visit(ext) + "\n" + case _: + s += self.visit(ext) + ";\n" + return s + + def visit_Compound(self, n: c_ast.Compound) -> str: + s = self._make_indent() + "{\n" + self.indent_level += 2 + if n.block_items: + s += "".join(self._generate_stmt(stmt) for stmt in n.block_items) + self.indent_level -= 2 + s += self._make_indent() + "}\n" + return s + + def visit_CompoundLiteral(self, n: c_ast.CompoundLiteral) -> str: + return "(" + self.visit(n.type) + "){" + self.visit(n.init) + "}" + + def visit_EmptyStatement(self, n: c_ast.EmptyStatement) -> str: + return ";" + + def visit_ParamList(self, n: c_ast.ParamList) -> str: + return ", ".join(self.visit(param) for param in n.params) + + def visit_Return(self, n: c_ast.Return) -> str: + s = "return" + if n.expr: + s += " " + self.visit(n.expr) + return s + ";" + + def visit_Break(self, n: c_ast.Break) -> str: + return "break;" + + def visit_Continue(self, n: c_ast.Continue) -> str: + return "continue;" + + def visit_TernaryOp(self, n: c_ast.TernaryOp) -> str: + s = "(" + self._visit_expr(n.cond) + ") ? " + s += "(" + self._visit_expr(n.iftrue) + ") : " + s += "(" + self._visit_expr(n.iffalse) + ")" + return s + + def visit_If(self, n: c_ast.If) -> str: + s = "if (" + if n.cond: + s += self.visit(n.cond) + s += ")\n" + s += self._generate_stmt(n.iftrue, add_indent=True) + if n.iffalse: + s += self._make_indent() + "else\n" + s += self._generate_stmt(n.iffalse, add_indent=True) + return s + + def visit_For(self, n: c_ast.For) -> str: + s = "for (" + if n.init: + s += self.visit(n.init) + s += ";" + if n.cond: + s += " " + self.visit(n.cond) + s += ";" + if n.next: + s += " " + self.visit(n.next) + s += ")\n" + s += self._generate_stmt(n.stmt, add_indent=True) + return s + + def visit_While(self, n: c_ast.While) -> str: + s = "while (" + if n.cond: + s += self.visit(n.cond) + s += ")\n" + s += self._generate_stmt(n.stmt, add_indent=True) + return s + + def visit_DoWhile(self, n: c_ast.DoWhile) -> str: + s = "do\n" + s += self._generate_stmt(n.stmt, add_indent=True) + s += self._make_indent() + "while (" + if n.cond: + s += self.visit(n.cond) + s += ");" + return s + + def visit_StaticAssert(self, n: c_ast.StaticAssert) -> str: + s = "_Static_assert(" + s += self.visit(n.cond) + if n.message: + s += "," + s += self.visit(n.message) + s += ")" + return s + + def visit_Switch(self, n: c_ast.Switch) -> str: + s = "switch (" + self.visit(n.cond) + ")\n" + s += self._generate_stmt(n.stmt, add_indent=True) + return s + + def visit_Case(self, n: c_ast.Case) -> str: + s = "case " + self.visit(n.expr) + ":\n" + for stmt in n.stmts: + s += self._generate_stmt(stmt, add_indent=True) + return s + + def visit_Default(self, n: c_ast.Default) -> str: + s = "default:\n" + for stmt in n.stmts: + s += self._generate_stmt(stmt, add_indent=True) + return s + + def visit_Label(self, n: c_ast.Label) -> str: + return n.name + ":\n" + self._generate_stmt(n.stmt) + + def visit_Goto(self, n: c_ast.Goto) -> str: + return "goto " + n.name + ";" + + def visit_EllipsisParam(self, n: c_ast.EllipsisParam) -> str: + return "..." + + def visit_Struct(self, n: c_ast.Struct) -> str: + return self._generate_struct_union_enum(n, "struct") + + def visit_Typename(self, n: c_ast.Typename) -> str: + return self._generate_type(n.type) + + def visit_Union(self, n: c_ast.Union) -> str: + return self._generate_struct_union_enum(n, "union") + + def visit_NamedInitializer(self, n: c_ast.NamedInitializer) -> str: + s = "" + for name in n.name: + if isinstance(name, c_ast.ID): + s += "." + name.name + else: + s += "[" + self.visit(name) + "]" + s += " = " + self._visit_expr(n.expr) + return s + + def visit_FuncDecl(self, n: c_ast.FuncDecl) -> str: + return self._generate_type(n) + + def visit_ArrayDecl(self, n: c_ast.ArrayDecl) -> str: + return self._generate_type(n, emit_declname=False) + + def visit_TypeDecl(self, n: c_ast.TypeDecl) -> str: + return self._generate_type(n, emit_declname=False) + + def visit_PtrDecl(self, n: c_ast.PtrDecl) -> str: + return self._generate_type(n, emit_declname=False) + + def _generate_struct_union_enum( + self, n: c_ast.Struct | c_ast.Union | c_ast.Enum, name: str + ) -> str: + """Generates code for structs, unions, and enums. name should be + 'struct', 'union', or 'enum'. + """ + if name in ("struct", "union"): + assert isinstance(n, (c_ast.Struct, c_ast.Union)) + members = n.decls + body_function = self._generate_struct_union_body + else: + assert name == "enum" + assert isinstance(n, c_ast.Enum) + members = None if n.values is None else n.values.enumerators + body_function = self._generate_enum_body + s = name + " " + (n.name or "") + if members is not None: + # None means no members + # Empty sequence means an empty list of members + s += "\n" + s += self._make_indent() + self.indent_level += 2 + s += "{\n" + s += body_function(members) + self.indent_level -= 2 + s += self._make_indent() + "}" + return s + + def _generate_struct_union_body(self, members: List[c_ast.Node]) -> str: + return "".join(self._generate_stmt(decl) for decl in members) + + def _generate_enum_body(self, members: List[c_ast.Enumerator]) -> str: + # `[:-2] + '\n'` removes the final `,` from the enumerator list + return "".join(self.visit(value) for value in members)[:-2] + "\n" + + def _generate_stmt(self, n: c_ast.Node, add_indent: bool = False) -> str: + """Generation from a statement node. This method exists as a wrapper + for individual visit_* methods to handle different treatment of + some statements in this context. + """ + if add_indent: + self.indent_level += 2 + indent = self._make_indent() + if add_indent: + self.indent_level -= 2 + + match n: + case ( + c_ast.Decl() + | c_ast.Assignment() + | c_ast.Cast() + | c_ast.UnaryOp() + | c_ast.BinaryOp() + | c_ast.TernaryOp() + | c_ast.FuncCall() + | c_ast.ArrayRef() + | c_ast.StructRef() + | c_ast.Constant() + | c_ast.ID() + | c_ast.Typedef() + | c_ast.ExprList() + ): + # These can also appear in an expression context so no semicolon + # is added to them automatically + # + return indent + self.visit(n) + ";\n" + case c_ast.Compound(): + # No extra indentation required before the opening brace of a + # compound - because it consists of multiple lines it has to + # compute its own indentation. + # + return self.visit(n) + case c_ast.If(): + return indent + self.visit(n) + case _: + return indent + self.visit(n) + "\n" + + def _generate_decl(self, n: c_ast.Decl) -> str: + """Generation from a Decl node.""" + s = "" + if n.funcspec: + s = " ".join(n.funcspec) + " " + if n.storage: + s += " ".join(n.storage) + " " + if n.align: + s += self.visit(n.align[0]) + " " + s += self._generate_type(n.type) + return s + + def _generate_type( + self, + n: c_ast.Node, + modifiers: List[c_ast.Node] = [], + emit_declname: bool = True, + ) -> str: + """Recursive generation from a type node. n is the type node. + modifiers collects the PtrDecl, ArrayDecl and FuncDecl modifiers + encountered on the way down to a TypeDecl, to allow proper + generation from it. + """ + # ~ print(n, modifiers) + match n: + case c_ast.TypeDecl(): + s = "" + if n.quals: + s += " ".join(n.quals) + " " + s += self.visit(n.type) + + nstr = n.declname if n.declname and emit_declname else "" + # Resolve modifiers. + # Wrap in parens to distinguish pointer to array and pointer to + # function syntax. + # + for i, modifier in enumerate(modifiers): + match modifier: + case c_ast.ArrayDecl(): + if i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl): + nstr = "(" + nstr + ")" + nstr += "[" + if modifier.dim_quals: + nstr += " ".join(modifier.dim_quals) + " " + if modifier.dim is not None: + nstr += self.visit(modifier.dim) + nstr += "]" + case c_ast.FuncDecl(): + if i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl): + nstr = "(" + nstr + ")" + args = ( + self.visit(modifier.args) + if modifier.args is not None + else "" + ) + nstr += "(" + args + ")" + case c_ast.PtrDecl(): + if modifier.quals: + quals = " ".join(modifier.quals) + suffix = f" {nstr}" if nstr else "" + nstr = f"* {quals}{suffix}" + else: + nstr = "*" + nstr + if nstr: + s += " " + nstr + return s + case c_ast.Decl(): + return self._generate_decl(n.type) + case c_ast.Typename(): + return self._generate_type(n.type, emit_declname=emit_declname) + case c_ast.IdentifierType(): + return " ".join(n.names) + " " + case c_ast.ArrayDecl() | c_ast.PtrDecl() | c_ast.FuncDecl(): + return self._generate_type( + n.type, modifiers + [n], emit_declname=emit_declname + ) + case _: + return self.visit(n) + + def _parenthesize_if( + self, n: c_ast.Node, condition: Callable[[c_ast.Node], bool] + ) -> str: + """Visits 'n' and returns its string representation, parenthesized + if the condition function applied to the node returns True. + """ + s = self._visit_expr(n) + if condition(n): + return "(" + s + ")" + else: + return s + + def _parenthesize_unless_simple(self, n: c_ast.Node) -> str: + """Common use case for _parenthesize_if""" + return self._parenthesize_if(n, lambda d: not self._is_simple_node(d)) + + def _is_simple_node(self, n: c_ast.Node) -> bool: + """Returns True for nodes that are "simple" - i.e. nodes that always + have higher precedence than operators. + """ + return isinstance( + n, + (c_ast.Constant, c_ast.ID, c_ast.ArrayRef, c_ast.StructRef, c_ast.FuncCall), + ) diff --git a/lib/pycparser/c_lexer.py b/lib/pycparser/c_lexer.py new file mode 100644 index 0000000..ef59d69 --- /dev/null +++ b/lib/pycparser/c_lexer.py @@ -0,0 +1,706 @@ +# ------------------------------------------------------------------------------ +# pycparser: c_lexer.py +# +# CLexer class: lexer for the C language +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ +import re +from dataclasses import dataclass +from enum import Enum +from typing import Callable, Dict, List, Optional, Tuple + + +@dataclass(slots=True) +class _Token: + type: str + value: str + lineno: int + column: int + + +class CLexer: + """A standalone lexer for C. + + Parameters for construction: + error_func: + Called with (msg, line, column) on lexing errors. + on_lbrace_func: + Called when an LBRACE token is produced (used for scope tracking). + on_rbrace_func: + Called when an RBRACE token is produced (used for scope tracking). + type_lookup_func: + Called with an identifier name; expected to return True if it is + a typedef name and should be tokenized as TYPEID. + + Call input(text) to initialize lexing, and then keep calling token() to + get the next token, until it returns None (at end of input). + """ + + def __init__( + self, + error_func: Callable[[str, int, int], None], + on_lbrace_func: Callable[[], None], + on_rbrace_func: Callable[[], None], + type_lookup_func: Callable[[str], bool], + ) -> None: + self.error_func = error_func + self.on_lbrace_func = on_lbrace_func + self.on_rbrace_func = on_rbrace_func + self.type_lookup_func = type_lookup_func + self._init_state() + + def input(self, text: str, filename: str = "") -> None: + """Initialize the lexer to the given input text. + + filename is an optional name identifying the file from which the input + comes. The lexer can modify it if #line directives are encountered. + """ + self._init_state() + self._lexdata = text + self._filename = filename + + def _init_state(self) -> None: + self._lexdata = "" + self._filename = "" + self._pos = 0 + self._line_start = 0 + self._pending_tok: Optional[_Token] = None + self._lineno = 1 + + @property + def filename(self) -> str: + return self._filename + + def token(self) -> Optional[_Token]: + # Lexing strategy overview: + # + # - We maintain a current position (self._pos), line number, and the + # byte offset of the current line start. The lexer is a simple loop + # that skips whitespace/newlines and emits one token per call. + # - A small amount of logic is handled manually before regex matching: + # + # * Preprocessor-style directives: if we see '#', we check whether + # it's a #line or #pragma directive and consume it inline. #line + # updates lineno/filename and produces no tokens. #pragma can yield + # both PPPRAGMA and PPPRAGMASTR, but token() returns a single token, + # so we stash the PPPRAGMASTR as _pending_tok to return on the next + # token() call. Otherwise we return PPHASH. + # * Newlines update lineno/line-start tracking so tokens can record + # accurate columns. + # + # - The bulk of tokens are recognized in _match_token: + # + # * _regex_rules: regex patterns for identifiers, literals, and other + # complex tokens (including error-producing patterns). The lexer + # uses a combined _regex_master to scan options at the same time. + # * _fixed_tokens: exact string matches for operators and punctuation, + # resolved by longest match. + # + # - Error patterns call the error callback and advance minimally, which + # keeps lexing resilient while reporting useful diagnostics. + text = self._lexdata + n = len(text) + + if self._pending_tok is not None: + tok = self._pending_tok + self._pending_tok = None + return tok + + while self._pos < n: + match text[self._pos]: + case " " | "\t": + self._pos += 1 + case "\n": + self._lineno += 1 + self._pos += 1 + self._line_start = self._pos + case "#": + if _line_pattern.match(text, self._pos + 1): + self._pos += 1 + self._handle_ppline() + continue + if _pragma_pattern.match(text, self._pos + 1): + self._pos += 1 + toks = self._handle_pppragma() + if len(toks) > 1: + self._pending_tok = toks[1] + if len(toks) > 0: + return toks[0] + continue + tok = self._make_token("PPHASH", "#", self._pos) + self._pos += 1 + return tok + case _: + if tok := self._match_token(): + return tok + else: + continue + + def _match_token(self) -> Optional[_Token]: + """Match one token at the current position. + + Returns a Token on success, or None if no token could be matched and + an error was reported. This method always advances _pos by the matched + length, or by 1 on error/no-match. + """ + text = self._lexdata + pos = self._pos + # We pick the longest match between: + # - the master regex (identifiers, literals, error patterns, etc.) + # - fixed operator/punctuator literals from the bucket for text[pos] + # + # The longest match is required to ensure we properly lex something + # like ".123" (a floating-point constant) as a single entity (with + # FLOAT_CONST), rather than a PERIOD followed by a number. + # + # The fixed-literal buckets are already length-sorted, so within that + # bucket we can take the first match. However, we still compare its + # length to the regex match because the regex may have matched a longer + # token that should take precedence. + best = None + + if m := _regex_master.match(text, pos): + tok_type = m.lastgroup + # All master-regex alternatives are named; lastgroup shouldn't be None. + assert tok_type is not None + value = m.group(tok_type) + length = len(value) + action, msg = _regex_actions[tok_type] + best = (length, tok_type, value, action, msg) + + if bucket := _fixed_tokens_by_first.get(text[pos]): + for entry in bucket: + if text.startswith(entry.literal, pos): + length = len(entry.literal) + if best is None or length > best[0]: + best = ( + length, + entry.tok_type, + entry.literal, + _RegexAction.TOKEN, + None, + ) + break + + if best is None: + msg = f"Illegal character {repr(text[pos])}" + self._error(msg, pos) + self._pos += 1 + return None + + length, tok_type, value, action, msg = best + if action == _RegexAction.ERROR: + if tok_type == "BAD_CHAR_CONST": + msg = f"Invalid char constant {value}" + # All other ERROR rules provide a message. + assert msg is not None + self._error(msg, pos) + self._pos += max(1, length) + return None + + if action == _RegexAction.ID: + tok_type = _keyword_map.get(value, "ID") + if tok_type == "ID" and self.type_lookup_func(value): + tok_type = "TYPEID" + + tok = self._make_token(tok_type, value, pos) + self._pos += length + + if tok.type == "LBRACE": + self.on_lbrace_func() + elif tok.type == "RBRACE": + self.on_rbrace_func() + + return tok + + def _make_token(self, tok_type: str, value: str, pos: int) -> _Token: + """Create a Token at an absolute input position. + + Expects tok_type/value and the absolute byte offset pos in the current + input. Does not advance lexer state; callers manage _pos themselves. + Returns a Token with lineno/column computed from current line tracking. + """ + column = pos - self._line_start + 1 + tok = _Token(tok_type, value, self._lineno, column) + return tok + + def _error(self, msg: str, pos: int) -> None: + column = pos - self._line_start + 1 + self.error_func(msg, self._lineno, column) + + def _handle_ppline(self) -> None: + # Since #line directives aren't supposed to return tokens but should + # only affect the lexer's state (update line/filename for coords), this + # method does a bit of parsing on its own. It doesn't return anything, + # but its side effect is to update self._pos past the directive, and + # potentially update self._lineno and self._filename, based on the + # directive's contents. + # + # Accepted #line forms from preprocessors: + # - "#line 66 \"kwas\\df.h\"" + # - "# 9" + # - "#line 10 \"include/me.h\" 1 2 3" (extra numeric flags) + # - "# 1 \"file.h\" 3" + # Errors we must report: + # - "#line \"file.h\"" (filename before line number) + # - "#line df" (garbage instead of number/string) + # + # We scan the directive line once (after an optional 'line' keyword), + # validating the order: NUMBER, optional STRING, then any NUMBERs. + # The NUMBERs tail is only accepted if a filename STRING was present. + text = self._lexdata + n = len(text) + line_end = text.find("\n", self._pos) + if line_end == -1: + line_end = n + line = text[self._pos : line_end] + pos = 0 + line_len = len(line) + + def skip_ws() -> None: + nonlocal pos + while pos < line_len and line[pos] in " \t": + pos += 1 + + skip_ws() + if line.startswith("line", pos): + pos += 4 + + def success(pp_line: Optional[str], pp_filename: Optional[str]) -> None: + if pp_line is None: + self._error("line number missing in #line", self._pos + line_len) + else: + self._lineno = int(pp_line) + if pp_filename is not None: + self._filename = pp_filename + self._pos = line_end + 1 + self._line_start = self._pos + + def fail(msg: str, offset: int) -> None: + self._error(msg, self._pos + offset) + self._pos = line_end + 1 + self._line_start = self._pos + + skip_ws() + if pos >= line_len: + success(None, None) + return + if line[pos] == '"': + fail("filename before line number in #line", pos) + return + + m = re.match(_decimal_constant, line[pos:]) + if not m: + fail("invalid #line directive", pos) + return + + pp_line = m.group(0) + pos += len(pp_line) + skip_ws() + if pos >= line_len: + success(pp_line, None) + return + + if line[pos] != '"': + fail("invalid #line directive", pos) + return + + m = re.match(_string_literal, line[pos:]) + if not m: + fail("invalid #line directive", pos) + return + + pp_filename = m.group(0).lstrip('"').rstrip('"') + pos += len(m.group(0)) + + # Consume arbitrary sequence of numeric flags after the directive + while True: + skip_ws() + if pos >= line_len: + break + m = re.match(_decimal_constant, line[pos:]) + if not m: + fail("invalid #line directive", pos) + return + pos += len(m.group(0)) + + success(pp_line, pp_filename) + + def _handle_pppragma(self) -> List[_Token]: + # Parse a full #pragma line; returns a list of tokens with 1 or 2 + # tokens - PPPRAGMA and an optional PPPRAGMASTR. If an empty list is + # returned, it means an error occurred, or we're at the end of input. + # + # Examples: + # - "#pragma" -> PPPRAGMA only + # - "#pragma once" -> PPPRAGMA, PPPRAGMASTR("once") + # - "# pragma omp parallel private(th_id)" -> PPPRAGMA, PPPRAGMASTR("omp parallel private(th_id)") + # - "#\tpragma {pack: 2, smack: 3}" -> PPPRAGMA, PPPRAGMASTR("{pack: 2, smack: 3}") + text = self._lexdata + n = len(text) + pos = self._pos + + while pos < n and text[pos] in " \t": + pos += 1 + if pos >= n: + self._pos = pos + return [] + + if not text.startswith("pragma", pos): + self._error("invalid #pragma directive", pos) + self._pos = pos + 1 + return [] + + pragma_pos = pos + pos += len("pragma") + toks = [self._make_token("PPPRAGMA", "pragma", pragma_pos)] + + while pos < n and text[pos] in " \t": + pos += 1 + + start = pos + while pos < n and text[pos] != "\n": + pos += 1 + if pos > start: + toks.append(self._make_token("PPPRAGMASTR", text[start:pos], start)) + if pos < n and text[pos] == "\n": + self._lineno += 1 + pos += 1 + self._line_start = pos + self._pos = pos + return toks + + +## +## Reserved keywords +## +_keywords: Tuple[str, ...] = ( + "AUTO", + "BREAK", + "CASE", + "CHAR", + "CONST", + "CONTINUE", + "DEFAULT", + "DO", + "DOUBLE", + "ELSE", + "ENUM", + "EXTERN", + "FLOAT", + "FOR", + "GOTO", + "IF", + "INLINE", + "INT", + "LONG", + "REGISTER", + "OFFSETOF", + "RESTRICT", + "RETURN", + "SHORT", + "SIGNED", + "SIZEOF", + "STATIC", + "STRUCT", + "SWITCH", + "TYPEDEF", + "UNION", + "UNSIGNED", + "VOID", + "VOLATILE", + "WHILE", + "__INT128", + "_BOOL", + "_COMPLEX", + "_NORETURN", + "_THREAD_LOCAL", + "_STATIC_ASSERT", + "_ATOMIC", + "_ALIGNOF", + "_ALIGNAS", + "_PRAGMA", +) + +_keyword_map: Dict[str, str] = {} + +for keyword in _keywords: + # Keywords from new C standard are mixed-case, like _Bool, _Alignas, etc. + if keyword.startswith("_") and len(keyword) > 1 and keyword[1].isalpha(): + _keyword_map[keyword[:2].upper() + keyword[2:].lower()] = keyword + else: + _keyword_map[keyword.lower()] = keyword + +## +## Regexes for use in tokens +## + +# valid C identifiers (K&R2: A.2.3), plus '$' (supported by some compilers) +_identifier = r"[a-zA-Z_$][0-9a-zA-Z_$]*" + +_hex_prefix = "0[xX]" +_hex_digits = "[0-9a-fA-F]+" +_bin_prefix = "0[bB]" +_bin_digits = "[01]+" + +# integer constants (K&R2: A.2.5.1) +_integer_suffix_opt = ( + r"(([uU]ll)|([uU]LL)|(ll[uU]?)|(LL[uU]?)|([uU][lL])|([lL][uU]?)|[uU])?" +) +_decimal_constant = ( + "(0" + _integer_suffix_opt + ")|([1-9][0-9]*" + _integer_suffix_opt + ")" +) +_octal_constant = "0[0-7]*" + _integer_suffix_opt +_hex_constant = _hex_prefix + _hex_digits + _integer_suffix_opt +_bin_constant = _bin_prefix + _bin_digits + _integer_suffix_opt + +_bad_octal_constant = "0[0-7]*[89]" + +# comments are not supported +_unsupported_c_style_comment = r"\/\*" +_unsupported_cxx_style_comment = r"\/\/" + +# character constants (K&R2: A.2.5.2) +# Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line +# directives with Windows paths as filenames (..\..\dir\file) +# For the same reason, decimal_escape allows all digit sequences. We want to +# parse all correct code, even if it means to sometimes parse incorrect +# code. +# +# The original regexes were taken verbatim from the C syntax definition, +# and were later modified to avoid worst-case exponential running time. +# +# simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" +# decimal_escape = r"""(\d+)""" +# hex_escape = r"""(x[0-9a-fA-F]+)""" +# bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])""" +# +# The following modifications were made to avoid the ambiguity that allowed +# backtracking: (https://github.com/eliben/pycparser/issues/61) +# +# - \x was removed from simple_escape, unless it was not followed by a hex +# digit, to avoid ambiguity with hex_escape. +# - hex_escape allows one or more hex characters, but requires that the next +# character(if any) is not hex +# - decimal_escape allows one or more decimal characters, but requires that the +# next character(if any) is not a decimal +# - bad_escape does not allow any decimals (8-9), to avoid conflicting with the +# permissive decimal_escape. +# +# Without this change, python's `re` module would recursively try parsing each +# ambiguous escape sequence in multiple ways. e.g. `\123` could be parsed as +# `\1`+`23`, `\12`+`3`, and `\123`. + +_simple_escape = r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))""" +_decimal_escape = r"""(\d+)(?!\d)""" +_hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])""" +_bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])""" + +_escape_sequence = ( + r"""(\\(""" + _simple_escape + "|" + _decimal_escape + "|" + _hex_escape + "))" +) + +# This complicated regex with lookahead might be slow for strings, so because +# all of the valid escapes (including \x) allowed +# 0 or more non-escaped characters after the first character, +# simple_escape+decimal_escape+hex_escape got simplified to + +_escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])""" + +_cconst_char = r"""([^'\\\n]|""" + _escape_sequence + ")" +_char_const = "'" + _cconst_char + "'" +_wchar_const = "L" + _char_const +_u8char_const = "u8" + _char_const +_u16char_const = "u" + _char_const +_u32char_const = "U" + _char_const +_multicharacter_constant = "'" + _cconst_char + "{2,4}'" +_unmatched_quote = "('" + _cconst_char + "*\\n)|('" + _cconst_char + "*$)" +_bad_char_const = ( + r"""('""" + _cconst_char + """[^'\n]+')|('')|('""" + _bad_escape + r"""[^'\n]*')""" +) + +# string literals (K&R2: A.2.6) +_string_char = r"""([^"\\\n]|""" + _escape_sequence_start_in_string + ")" +_string_literal = '"' + _string_char + '*"' +_wstring_literal = "L" + _string_literal +_u8string_literal = "u8" + _string_literal +_u16string_literal = "u" + _string_literal +_u32string_literal = "U" + _string_literal +_bad_string_literal = '"' + _string_char + "*" + _bad_escape + _string_char + '*"' + +# floating constants (K&R2: A.2.5.3) +_exponent_part = r"""([eE][-+]?[0-9]+)""" +_fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" +_floating_constant = ( + "((((" + + _fractional_constant + + ")" + + _exponent_part + + "?)|([0-9]+" + + _exponent_part + + "))[FfLl]?)" +) +_binary_exponent_part = r"""([pP][+-]?[0-9]+)""" +_hex_fractional_constant = ( + "(((" + _hex_digits + r""")?\.""" + _hex_digits + ")|(" + _hex_digits + r"""\.))""" +) +_hex_floating_constant = ( + "(" + + _hex_prefix + + "(" + + _hex_digits + + "|" + + _hex_fractional_constant + + ")" + + _binary_exponent_part + + "[FfLl]?)" +) + + +class _RegexAction(Enum): + TOKEN = 0 + ID = 1 + ERROR = 2 + + +@dataclass(frozen=True) +class _RegexRule: + # tok_type: name of the token emitted for a match + # regex_pattern: the raw regex (no anchors) to match at the current position + # action: TOKEN for normal tokens, ID for identifiers, ERROR to report + # error_message: message used for ERROR entries + tok_type: str + regex_pattern: str + action: _RegexAction + error_message: Optional[str] + + +_regex_rules: List[_RegexRule] = [ + _RegexRule( + "UNSUPPORTED_C_STYLE_COMMENT", + _unsupported_c_style_comment, + _RegexAction.ERROR, + "Comments are not supported, see https://github.com/eliben/pycparser#3using.", + ), + _RegexRule( + "UNSUPPORTED_CXX_STYLE_COMMENT", + _unsupported_cxx_style_comment, + _RegexAction.ERROR, + "Comments are not supported, see https://github.com/eliben/pycparser#3using.", + ), + _RegexRule( + "BAD_STRING_LITERAL", + _bad_string_literal, + _RegexAction.ERROR, + "String contains invalid escape code", + ), + _RegexRule("WSTRING_LITERAL", _wstring_literal, _RegexAction.TOKEN, None), + _RegexRule("U8STRING_LITERAL", _u8string_literal, _RegexAction.TOKEN, None), + _RegexRule("U16STRING_LITERAL", _u16string_literal, _RegexAction.TOKEN, None), + _RegexRule("U32STRING_LITERAL", _u32string_literal, _RegexAction.TOKEN, None), + _RegexRule("STRING_LITERAL", _string_literal, _RegexAction.TOKEN, None), + _RegexRule("HEX_FLOAT_CONST", _hex_floating_constant, _RegexAction.TOKEN, None), + _RegexRule("FLOAT_CONST", _floating_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_HEX", _hex_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_BIN", _bin_constant, _RegexAction.TOKEN, None), + _RegexRule( + "BAD_CONST_OCT", + _bad_octal_constant, + _RegexAction.ERROR, + "Invalid octal constant", + ), + _RegexRule("INT_CONST_OCT", _octal_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_DEC", _decimal_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_CHAR", _multicharacter_constant, _RegexAction.TOKEN, None), + _RegexRule("CHAR_CONST", _char_const, _RegexAction.TOKEN, None), + _RegexRule("WCHAR_CONST", _wchar_const, _RegexAction.TOKEN, None), + _RegexRule("U8CHAR_CONST", _u8char_const, _RegexAction.TOKEN, None), + _RegexRule("U16CHAR_CONST", _u16char_const, _RegexAction.TOKEN, None), + _RegexRule("U32CHAR_CONST", _u32char_const, _RegexAction.TOKEN, None), + _RegexRule("UNMATCHED_QUOTE", _unmatched_quote, _RegexAction.ERROR, "Unmatched '"), + _RegexRule("BAD_CHAR_CONST", _bad_char_const, _RegexAction.ERROR, None), + _RegexRule("ID", _identifier, _RegexAction.ID, None), +] + +_regex_actions: Dict[str, Tuple[_RegexAction, Optional[str]]] = {} +_regex_pattern_parts: List[str] = [] +for _rule in _regex_rules: + _regex_actions[_rule.tok_type] = (_rule.action, _rule.error_message) + _regex_pattern_parts.append(f"(?P<{_rule.tok_type}>{_rule.regex_pattern})") +# The master regex is a single alternation of all token patterns, each wrapped +# in a named group. We match once at the current position and then use +# `lastgroup` to recover which token kind fired; this avoids iterating over all +# regexes on every character while keeping the same token-level semantics. +_regex_master: re.Pattern[str] = re.compile("|".join(_regex_pattern_parts)) + + +@dataclass(frozen=True) +class _FixedToken: + tok_type: str + literal: str + + +_fixed_tokens: List[_FixedToken] = [ + _FixedToken("ELLIPSIS", "..."), + _FixedToken("LSHIFTEQUAL", "<<="), + _FixedToken("RSHIFTEQUAL", ">>="), + _FixedToken("PLUSPLUS", "++"), + _FixedToken("MINUSMINUS", "--"), + _FixedToken("ARROW", "->"), + _FixedToken("LAND", "&&"), + _FixedToken("LOR", "||"), + _FixedToken("LSHIFT", "<<"), + _FixedToken("RSHIFT", ">>"), + _FixedToken("LE", "<="), + _FixedToken("GE", ">="), + _FixedToken("EQ", "=="), + _FixedToken("NE", "!="), + _FixedToken("TIMESEQUAL", "*="), + _FixedToken("DIVEQUAL", "/="), + _FixedToken("MODEQUAL", "%="), + _FixedToken("PLUSEQUAL", "+="), + _FixedToken("MINUSEQUAL", "-="), + _FixedToken("ANDEQUAL", "&="), + _FixedToken("OREQUAL", "|="), + _FixedToken("XOREQUAL", "^="), + _FixedToken("EQUALS", "="), + _FixedToken("PLUS", "+"), + _FixedToken("MINUS", "-"), + _FixedToken("TIMES", "*"), + _FixedToken("DIVIDE", "/"), + _FixedToken("MOD", "%"), + _FixedToken("OR", "|"), + _FixedToken("AND", "&"), + _FixedToken("NOT", "~"), + _FixedToken("XOR", "^"), + _FixedToken("LNOT", "!"), + _FixedToken("LT", "<"), + _FixedToken("GT", ">"), + _FixedToken("CONDOP", "?"), + _FixedToken("LPAREN", "("), + _FixedToken("RPAREN", ")"), + _FixedToken("LBRACKET", "["), + _FixedToken("RBRACKET", "]"), + _FixedToken("LBRACE", "{"), + _FixedToken("RBRACE", "}"), + _FixedToken("COMMA", ","), + _FixedToken("PERIOD", "."), + _FixedToken("SEMI", ";"), + _FixedToken("COLON", ":"), +] + +# To avoid scanning all fixed tokens on every character, we bucket them by the +# first character. When matching at position i, we only look at the bucket for +# text[i], and we pre-sort that bucket by token length so the first match is +# also the longest. This preserves longest-match semantics (e.g. '>>=' before +# '>>' before '>') while reducing the number of comparisons. +_fixed_tokens_by_first: Dict[str, List[_FixedToken]] = {} +for _entry in _fixed_tokens: + _fixed_tokens_by_first.setdefault(_entry.literal[0], []).append(_entry) +for _bucket in _fixed_tokens_by_first.values(): + _bucket.sort(key=lambda item: len(item.literal), reverse=True) + +_line_pattern: re.Pattern[str] = re.compile(r"([ \t]*line\W)|([ \t]*\d+)") +_pragma_pattern: re.Pattern[str] = re.compile(r"[ \t]*pragma\W") diff --git a/lib/pycparser/c_parser.py b/lib/pycparser/c_parser.py new file mode 100644 index 0000000..f980672 --- /dev/null +++ b/lib/pycparser/c_parser.py @@ -0,0 +1,2376 @@ +# ------------------------------------------------------------------------------ +# pycparser: c_parser.py +# +# Recursive-descent parser for the C language. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ +from dataclasses import dataclass +from typing import ( + Any, + Dict, + List, + Literal, + NoReturn, + Optional, + Tuple, + TypedDict, + cast, +) + +from . import c_ast +from .c_lexer import CLexer, _Token +from .ast_transforms import fix_switch_cases, fix_atomic_specifiers + + +@dataclass +class Coord: + """Coordinates of a syntactic element. Consists of: + - File name + - Line number + - Column number + """ + + file: str + line: int + column: Optional[int] = None + + def __str__(self) -> str: + text = f"{self.file}:{self.line}" + if self.column: + text += f":{self.column}" + return text + + +class ParseError(Exception): + pass + + +class CParser: + """Recursive-descent C parser. + + Usage: + parser = CParser() + ast = parser.parse(text, filename) + + The `lexer` parameter lets you inject a lexer class (defaults to CLexer). + The parameters after `lexer` are accepted for backward compatibility with + the old PLY-based parser and are otherwise unused. + """ + + def __init__( + self, + lex_optimize: bool = True, + lexer: type[CLexer] = CLexer, + lextab: str = "pycparser.lextab", + yacc_optimize: bool = True, + yacctab: str = "pycparser.yacctab", + yacc_debug: bool = False, + taboutputdir: str = "", + ) -> None: + self.clex: CLexer = lexer( + error_func=self._lex_error_func, + on_lbrace_func=self._lex_on_lbrace_func, + on_rbrace_func=self._lex_on_rbrace_func, + type_lookup_func=self._lex_type_lookup_func, + ) + + # Stack of scopes for keeping track of symbols. _scope_stack[-1] is + # the current (topmost) scope. Each scope is a dictionary that + # specifies whether a name is a type. If _scope_stack[n][name] is + # True, 'name' is currently a type in the scope. If it's False, + # 'name' is used in the scope but not as a type (for instance, if we + # saw: int name; + # If 'name' is not a key in _scope_stack[n] then 'name' was not defined + # in this scope at all. + self._scope_stack: List[Dict[str, bool]] = [dict()] + self._tokens: _TokenStream = _TokenStream(self.clex) + + def parse( + self, text: str, filename: str = "", debug: bool = False + ) -> c_ast.FileAST: + """Parses C code and returns an AST. + + text: + A string containing the C source code + + filename: + Name of the file being parsed (for meaningful + error messages) + + debug: + Deprecated debug flag (unused); for backwards compatibility. + """ + self._scope_stack = [dict()] + self.clex.input(text, filename) + self._tokens = _TokenStream(self.clex) + + ast = self._parse_translation_unit_or_empty() + tok = self._peek() + if tok is not None: + self._parse_error(f"before: {tok.value}", self._tok_coord(tok)) + return ast + + # ------------------------------------------------------------------ + # Scope and declaration helpers + # ------------------------------------------------------------------ + def _coord(self, lineno: int, column: Optional[int] = None) -> Coord: + return Coord(file=self.clex.filename, line=lineno, column=column) + + def _parse_error(self, msg: str, coord: Coord | str | None) -> NoReturn: + raise ParseError(f"{coord}: {msg}") + + def _push_scope(self) -> None: + self._scope_stack.append(dict()) + + def _pop_scope(self) -> None: + assert len(self._scope_stack) > 1 + self._scope_stack.pop() + + def _add_typedef_name(self, name: str, coord: Optional[Coord]) -> None: + """Add a new typedef name (ie a TYPEID) to the current scope""" + if not self._scope_stack[-1].get(name, True): + self._parse_error( + f"Typedef {name!r} previously declared as non-typedef in this scope", + coord, + ) + self._scope_stack[-1][name] = True + + def _add_identifier(self, name: str, coord: Optional[Coord]) -> None: + """Add a new object, function, or enum member name (ie an ID) to the + current scope + """ + if self._scope_stack[-1].get(name, False): + self._parse_error( + f"Non-typedef {name!r} previously declared as typedef in this scope", + coord, + ) + self._scope_stack[-1][name] = False + + def _is_type_in_scope(self, name: str) -> bool: + """Is *name* a typedef-name in the current scope?""" + for scope in reversed(self._scope_stack): + # If name is an identifier in this scope it shadows typedefs in + # higher scopes. + in_scope = scope.get(name) + if in_scope is not None: + return in_scope + return False + + def _lex_error_func(self, msg: str, line: int, column: int) -> None: + self._parse_error(msg, self._coord(line, column)) + + def _lex_on_lbrace_func(self) -> None: + self._push_scope() + + def _lex_on_rbrace_func(self) -> None: + self._pop_scope() + + def _lex_type_lookup_func(self, name: str) -> bool: + """Looks up types that were previously defined with + typedef. + Passed to the lexer for recognizing identifiers that + are types. + """ + return self._is_type_in_scope(name) + + # To understand what's going on here, read sections A.8.5 and + # A.8.6 of K&R2 very carefully. + # + # A C type consists of a basic type declaration, with a list + # of modifiers. For example: + # + # int *c[5]; + # + # The basic declaration here is 'int c', and the pointer and + # the array are the modifiers. + # + # Basic declarations are represented by TypeDecl (from module c_ast) and the + # modifiers are FuncDecl, PtrDecl and ArrayDecl. + # + # The standard states that whenever a new modifier is parsed, it should be + # added to the end of the list of modifiers. For example: + # + # K&R2 A.8.6.2: Array Declarators + # + # In a declaration T D where D has the form + # D1 [constant-expression-opt] + # and the type of the identifier in the declaration T D1 is + # "type-modifier T", the type of the + # identifier of D is "type-modifier array of T" + # + # This is what this method does. The declarator it receives + # can be a list of declarators ending with TypeDecl. It + # tacks the modifier to the end of this list, just before + # the TypeDecl. + # + # Additionally, the modifier may be a list itself. This is + # useful for pointers, that can come as a chain from the rule + # p_pointer. In this case, the whole modifier list is spliced + # into the new location. + def _type_modify_decl(self, decl: Any, modifier: Any) -> c_ast.Node: + """Tacks a type modifier on a declarator, and returns + the modified declarator. + + Note: the declarator and modifier may be modified + """ + modifier_head = modifier + modifier_tail = modifier + + # The modifier may be a nested list. Reach its tail. + while modifier_tail.type: + modifier_tail = modifier_tail.type + + # If the decl is a basic type, just tack the modifier onto it. + if isinstance(decl, c_ast.TypeDecl): + modifier_tail.type = decl + return modifier + else: + # Otherwise, the decl is a list of modifiers. Reach + # its tail and splice the modifier onto the tail, + # pointing to the underlying basic type. + decl_tail = decl + while not isinstance(decl_tail.type, c_ast.TypeDecl): + decl_tail = decl_tail.type + + modifier_tail.type = decl_tail.type + decl_tail.type = modifier_head + return decl + + # Due to the order in which declarators are constructed, + # they have to be fixed in order to look like a normal AST. + # + # When a declaration arrives from syntax construction, it has + # these problems: + # * The innermost TypeDecl has no type (because the basic + # type is only known at the uppermost declaration level) + # * The declaration has no variable name, since that is saved + # in the innermost TypeDecl + # * The typename of the declaration is a list of type + # specifiers, and not a node. Here, basic identifier types + # should be separated from more complex types like enums + # and structs. + # + # This method fixes these problems. + def _fix_decl_name_type( + self, + decl: c_ast.Decl | c_ast.Typedef | c_ast.Typename, + typename: List[Any], + ) -> c_ast.Decl | c_ast.Typedef | c_ast.Typename: + """Fixes a declaration. Modifies decl.""" + # Reach the underlying basic type + typ = decl + while not isinstance(typ, c_ast.TypeDecl): + typ = typ.type + + decl.name = typ.declname + typ.quals = decl.quals[:] + + # The typename is a list of types. If any type in this + # list isn't an IdentifierType, it must be the only + # type in the list (it's illegal to declare "int enum ..") + # If all the types are basic, they're collected in the + # IdentifierType holder. + for tn in typename: + if not isinstance(tn, c_ast.IdentifierType): + if len(typename) > 1: + self._parse_error("Invalid multiple types specified", tn.coord) + else: + typ.type = tn + return decl + + if not typename: + # Functions default to returning int + if not isinstance(decl.type, c_ast.FuncDecl): + self._parse_error("Missing type in declaration", decl.coord) + typ.type = c_ast.IdentifierType(["int"], coord=decl.coord) + else: + # At this point, we know that typename is a list of IdentifierType + # nodes. Concatenate all the names into a single list. + typ.type = c_ast.IdentifierType( + [name for id in typename for name in id.names], coord=typename[0].coord + ) + return decl + + def _add_declaration_specifier( + self, + declspec: Optional["_DeclSpec"], + newspec: Any, + kind: "_DeclSpecKind", + append: bool = False, + ) -> "_DeclSpec": + """See _DeclSpec for the specifier dictionary layout.""" + if declspec is None: + spec: _DeclSpec = dict( + qual=[], storage=[], type=[], function=[], alignment=[] + ) + else: + spec = declspec + + if append: + spec[kind].append(newspec) + else: + spec[kind].insert(0, newspec) + + return spec + + def _build_declarations( + self, + spec: "_DeclSpec", + decls: List["_DeclInfo"], + typedef_namespace: bool = False, + ) -> List[c_ast.Node]: + """Builds a list of declarations all sharing the given specifiers. + If typedef_namespace is true, each declared name is added + to the "typedef namespace", which also includes objects, + functions, and enum constants. + """ + is_typedef = "typedef" in spec["storage"] + declarations = [] + + # Bit-fields are allowed to be unnamed. + if decls[0].get("bitsize") is None: + # When redeclaring typedef names as identifiers in inner scopes, a + # problem can occur where the identifier gets grouped into + # spec['type'], leaving decl as None. This can only occur for the + # first declarator. + if decls[0]["decl"] is None: + if ( + len(spec["type"]) < 2 + or len(spec["type"][-1].names) != 1 + or not self._is_type_in_scope(spec["type"][-1].names[0]) + ): + coord = "?" + for t in spec["type"]: + if hasattr(t, "coord"): + coord = t.coord + break + self._parse_error("Invalid declaration", coord) + + # Make this look as if it came from "direct_declarator:ID" + decls[0]["decl"] = c_ast.TypeDecl( + declname=spec["type"][-1].names[0], + type=None, + quals=None, + align=spec["alignment"], + coord=spec["type"][-1].coord, + ) + # Remove the "new" type's name from the end of spec['type'] + del spec["type"][-1] + # A similar problem can occur where the declaration ends up + # looking like an abstract declarator. Give it a name if this is + # the case. + elif not isinstance( + decls[0]["decl"], + (c_ast.Enum, c_ast.Struct, c_ast.Union, c_ast.IdentifierType), + ): + decls_0_tail = cast(Any, decls[0]["decl"]) + while not isinstance(decls_0_tail, c_ast.TypeDecl): + decls_0_tail = decls_0_tail.type + if decls_0_tail.declname is None: + decls_0_tail.declname = spec["type"][-1].names[0] + del spec["type"][-1] + + for decl in decls: + assert decl["decl"] is not None + if is_typedef: + declaration = c_ast.Typedef( + name=None, + quals=spec["qual"], + storage=spec["storage"], + type=decl["decl"], + coord=decl["decl"].coord, + ) + else: + declaration = c_ast.Decl( + name=None, + quals=spec["qual"], + align=spec["alignment"], + storage=spec["storage"], + funcspec=spec["function"], + type=decl["decl"], + init=decl.get("init"), + bitsize=decl.get("bitsize"), + coord=decl["decl"].coord, + ) + + if isinstance( + declaration.type, + (c_ast.Enum, c_ast.Struct, c_ast.Union, c_ast.IdentifierType), + ): + fixed_decl = declaration + else: + fixed_decl = self._fix_decl_name_type(declaration, spec["type"]) + + # Add the type name defined by typedef to a + # symbol table (for usage in the lexer) + if typedef_namespace: + if is_typedef: + self._add_typedef_name(fixed_decl.name, fixed_decl.coord) + else: + self._add_identifier(fixed_decl.name, fixed_decl.coord) + + fixed_decl = fix_atomic_specifiers( + cast(c_ast.Decl | c_ast.Typedef, fixed_decl) + ) + declarations.append(fixed_decl) + + return declarations + + def _build_function_definition( + self, + spec: "_DeclSpec", + decl: c_ast.Node, + param_decls: Optional[List[c_ast.Node]], + body: c_ast.Node, + ) -> c_ast.Node: + """Builds a function definition.""" + if "typedef" in spec["storage"]: + self._parse_error("Invalid typedef", decl.coord) + + declaration = self._build_declarations( + spec=spec, + decls=[dict(decl=decl, init=None, bitsize=None)], + typedef_namespace=True, + )[0] + + return c_ast.FuncDef( + decl=declaration, param_decls=param_decls, body=body, coord=decl.coord + ) + + def _select_struct_union_class(self, token: str) -> type: + """Given a token (either STRUCT or UNION), selects the + appropriate AST class. + """ + if token == "struct": + return c_ast.Struct + else: + return c_ast.Union + + # ------------------------------------------------------------------ + # Token helpers + # ------------------------------------------------------------------ + def _peek(self, k: int = 1) -> Optional[_Token]: + """Return the k-th next token without consuming it (1-based).""" + return self._tokens.peek(k) + + def _peek_type(self, k: int = 1) -> Optional[str]: + """Return the type of the k-th next token, or None if absent (1-based).""" + tok = self._peek(k) + return tok.type if tok is not None else None + + def _advance(self) -> _Token: + tok = self._tokens.next() + if tok is None: + self._parse_error("At end of input", self.clex.filename) + else: + return tok + + def _accept(self, token_type: str) -> Optional[_Token]: + """Conditionally consume next token, only if it's of token_type. + + If it is of the expected type, consume and return it. + Otherwise, leaves the token intact and returns None. + """ + tok = self._peek() + if tok is not None and tok.type == token_type: + return self._advance() + return None + + def _expect(self, token_type: str) -> _Token: + tok = self._advance() + if tok.type != token_type: + self._parse_error(f"before: {tok.value}", self._tok_coord(tok)) + return tok + + def _mark(self) -> int: + return self._tokens.mark() + + def _reset(self, mark: int) -> None: + self._tokens.reset(mark) + + def _tok_coord(self, tok: _Token) -> Coord: + return self._coord(tok.lineno, tok.column) + + def _starts_declaration(self, tok: Optional[_Token] = None) -> bool: + tok = tok or self._peek() + if tok is None: + return False + return tok.type in _DECL_START + + def _starts_expression(self, tok: Optional[_Token] = None) -> bool: + tok = tok or self._peek() + if tok is None: + return False + return tok.type in _STARTS_EXPRESSION + + def _starts_statement(self) -> bool: + tok_type = self._peek_type() + if tok_type is None: + return False + if tok_type in _STARTS_STATEMENT: + return True + return self._starts_expression() + + def _starts_declarator(self, id_only: bool = False) -> bool: + tok_type = self._peek_type() + if tok_type is None: + return False + if tok_type in {"TIMES", "LPAREN"}: + return True + if id_only: + return tok_type == "ID" + return tok_type in {"ID", "TYPEID"} + + def _peek_declarator_name_info(self) -> Tuple[Optional[str], bool]: + mark = self._mark() + tok_type, saw_paren = self._scan_declarator_name_info() + self._reset(mark) + return tok_type, saw_paren + + def _parse_any_declarator( + self, allow_abstract: bool = False, typeid_paren_as_abstract: bool = False + ) -> Tuple[Optional[c_ast.Node], bool]: + # C declarators are ambiguous without lookahead. For example: + # int foo(int (aa)); -> aa is a name (ID) + # typedef char TT; + # int bar(int (TT)); -> TT is a type (TYPEID) in parens + name_type, saw_paren = self._peek_declarator_name_info() + if name_type is None or ( + typeid_paren_as_abstract and name_type == "TYPEID" and saw_paren + ): + if not allow_abstract: + tok = self._peek() + coord = self._tok_coord(tok) if tok is not None else self.clex.filename + self._parse_error("Invalid declarator", coord) + decl = self._parse_abstract_declarator_opt() + return decl, False + + if name_type == "TYPEID": + if typeid_paren_as_abstract: + decl = self._parse_typeid_noparen_declarator() + else: + decl = self._parse_typeid_declarator() + else: + decl = self._parse_id_declarator() + return decl, True + + def _scan_declarator_name_info(self) -> Tuple[Optional[str], bool]: + saw_paren = False + while self._accept("TIMES"): + while self._peek_type() in _TYPE_QUALIFIER: + self._advance() + + tok = self._peek() + if tok is None: + return None, saw_paren + if tok.type in {"ID", "TYPEID"}: + self._advance() + return tok.type, saw_paren + if tok.type == "LPAREN": + saw_paren = True + self._advance() + tok_type, nested_paren = self._scan_declarator_name_info() + if nested_paren: + saw_paren = True + depth = 1 + while True: + tok = self._peek() + if tok is None: + return None, saw_paren + if tok.type == "LPAREN": + depth += 1 + elif tok.type == "RPAREN": + depth -= 1 + self._advance() + if depth == 0: + break + continue + self._advance() + return tok_type, saw_paren + return None, saw_paren + + def _starts_direct_abstract_declarator(self) -> bool: + return self._peek_type() in {"LPAREN", "LBRACKET"} + + def _is_assignment_op(self) -> bool: + tok = self._peek() + return tok is not None and tok.type in _ASSIGNMENT_OPS + + def _try_parse_paren_type_name( + self, + ) -> Optional[Tuple[c_ast.Typename, int, _Token]]: + """Parse and return a parenthesized type name if present. + + Returns (typ, mark, lparen_tok) when the next tokens look like + '(' type_name ')', where typ is the parsed type name, mark is the + token-stream position before parsing, and lparen_tok is the LPAREN + token. Returns None if no parenthesized type name is present. + """ + mark = self._mark() + lparen_tok = self._accept("LPAREN") + if lparen_tok is None: + return None + if not self._starts_declaration(): + self._reset(mark) + return None + typ = self._parse_type_name() + if self._accept("RPAREN") is None: + self._reset(mark) + return None + return typ, mark, lparen_tok + + # ------------------------------------------------------------------ + # Top-level + # ------------------------------------------------------------------ + # BNF: translation_unit_or_empty : translation_unit | empty + def _parse_translation_unit_or_empty(self) -> c_ast.FileAST: + if self._peek() is None: + return c_ast.FileAST([]) + return c_ast.FileAST(self._parse_translation_unit()) + + # BNF: translation_unit : external_declaration+ + def _parse_translation_unit(self) -> List[c_ast.Node]: + ext = [] + while self._peek() is not None: + ext.extend(self._parse_external_declaration()) + return ext + + # BNF: external_declaration : function_definition + # | declaration + # | pp_directive + # | pppragma_directive + # | static_assert + # | ';' + def _parse_external_declaration(self) -> List[c_ast.Node]: + tok = self._peek() + if tok is None: + return [] + if tok.type == "PPHASH": + self._parse_pp_directive() + return [] + if tok.type in {"PPPRAGMA", "_PRAGMA"}: + return [self._parse_pppragma_directive()] + if self._accept("SEMI"): + return [] + if tok.type == "_STATIC_ASSERT": + return self._parse_static_assert() + + if not self._starts_declaration(tok): + # Special handling for old-style function definitions that have an + # implicit return type, e.g. + # + # foo() { + # return 5; + # } + # + # These get an implicit 'int' return type. + decl = self._parse_id_declarator() + param_decls = None + if self._peek_type() != "LBRACE": + self._parse_error("Invalid function definition", decl.coord) + spec: _DeclSpec = dict( + qual=[], + alignment=[], + storage=[], + type=[c_ast.IdentifierType(["int"], coord=decl.coord)], + function=[], + ) + func = self._build_function_definition( + spec=spec, + decl=decl, + param_decls=param_decls, + body=self._parse_compound_statement(), + ) + return [func] + + # From here on, parsing a standard declatation/definition. + spec, saw_type, spec_coord = self._parse_declaration_specifiers( + allow_no_type=True + ) + + name_type, _ = self._peek_declarator_name_info() + if name_type != "ID": + decls = self._parse_decl_body_with_spec(spec, saw_type) + self._expect("SEMI") + return decls + + decl = self._parse_id_declarator() + + if self._peek_type() == "LBRACE" or self._starts_declaration(): + param_decls = None + if self._starts_declaration(): + param_decls = self._parse_declaration_list() + if self._peek_type() != "LBRACE": + self._parse_error("Invalid function definition", decl.coord) + if not spec["type"]: + spec["type"] = [c_ast.IdentifierType(["int"], coord=spec_coord)] + func = self._build_function_definition( + spec=spec, + decl=decl, + param_decls=param_decls, + body=self._parse_compound_statement(), + ) + return [func] + + decl_dict: "_DeclInfo" = dict(decl=decl, init=None, bitsize=None) + if self._accept("EQUALS"): + decl_dict["init"] = self._parse_initializer() + decls = self._parse_init_declarator_list(first=decl_dict) + decls = self._build_declarations(spec=spec, decls=decls, typedef_namespace=True) + self._expect("SEMI") + return decls + + # ------------------------------------------------------------------ + # Declarations + # + # Declarations always come as lists (because they can be several in one + # line). When returning parsed declarations, a list is always returned - + # even if it contains a single element. + # ------------------------------------------------------------------ + def _parse_declaration(self) -> List[c_ast.Node]: + decls = self._parse_decl_body() + self._expect("SEMI") + return decls + + # BNF: decl_body : declaration_specifiers decl_body_with_spec + def _parse_decl_body(self) -> List[c_ast.Node]: + spec, saw_type, _ = self._parse_declaration_specifiers(allow_no_type=True) + return self._parse_decl_body_with_spec(spec, saw_type) + + # BNF: decl_body_with_spec : init_declarator_list + # | struct_or_union_or_enum_only + def _parse_decl_body_with_spec( + self, spec: "_DeclSpec", saw_type: bool + ) -> List[c_ast.Node]: + decls = None + if saw_type: + if self._starts_declarator(): + decls = self._parse_init_declarator_list() + else: + if self._starts_declarator(id_only=True): + decls = self._parse_init_declarator_list(id_only=True) + + if decls is None: + ty = spec["type"] + s_u_or_e = (c_ast.Struct, c_ast.Union, c_ast.Enum) + if len(ty) == 1 and isinstance(ty[0], s_u_or_e): + decls = [ + c_ast.Decl( + name=None, + quals=spec["qual"], + align=spec["alignment"], + storage=spec["storage"], + funcspec=spec["function"], + type=ty[0], + init=None, + bitsize=None, + coord=ty[0].coord, + ) + ] + else: + decls = self._build_declarations( + spec=spec, + decls=[dict(decl=None, init=None, bitsize=None)], + typedef_namespace=True, + ) + else: + decls = self._build_declarations( + spec=spec, decls=decls, typedef_namespace=True + ) + + return decls + + # BNF: declaration_list : declaration+ + def _parse_declaration_list(self) -> List[c_ast.Node]: + decls = [] + while self._starts_declaration(): + decls.extend(self._parse_declaration()) + return decls + + # BNF: declaration_specifiers : (storage_class_specifier + # | type_specifier + # | type_qualifier + # | function_specifier + # | alignment_specifier)+ + def _parse_declaration_specifiers( + self, allow_no_type: bool = False + ) -> Tuple["_DeclSpec", bool, Optional[Coord]]: + """Parse declaration-specifier sequence. + + allow_no_type: + If True, allow a missing type specifier without error. + + Returns: + (spec, saw_type, first_coord) where spec is a dict with + qual/storage/type/function/alignment entries, saw_type is True + if a type specifier was consumed, and first_coord is the coord + of the first specifier token (used for diagnostics). + """ + spec = None + saw_type = False + first_coord = None + + while True: + tok = self._peek() + if tok is None: + break + + if tok.type == "_ALIGNAS": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_alignment_specifier(), "alignment", append=True + ) + continue + + if tok.type == "_ATOMIC" and self._peek_type(2) == "LPAREN": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_atomic_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type in _TYPE_QUALIFIER: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "qual", append=True + ) + continue + + if tok.type in _STORAGE_CLASS: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "storage", append=True + ) + continue + + if tok.type in _FUNCTION_SPEC: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "function", append=True + ) + continue + + if tok.type in _TYPE_SPEC_SIMPLE: + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type == "TYPEID": + if saw_type: + break + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type in {"STRUCT", "UNION"}: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_struct_or_union_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type == "ENUM": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_enum_specifier(), "type", append=True + ) + saw_type = True + continue + + break + + if spec is None: + self._parse_error("Invalid declaration", self.clex.filename) + + if not saw_type and not allow_no_type: + self._parse_error("Missing type in declaration", first_coord) + + return spec, saw_type, first_coord + + # BNF: specifier_qualifier_list : (type_specifier + # | type_qualifier + # | alignment_specifier)+ + def _parse_specifier_qualifier_list(self) -> "_DeclSpec": + spec = None + saw_type = False + saw_alignment = False + first_coord = None + + while True: + tok = self._peek() + if tok is None: + break + + if tok.type == "_ALIGNAS": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_alignment_specifier(), "alignment", append=True + ) + saw_alignment = True + continue + + if tok.type == "_ATOMIC" and self._peek_type(2) == "LPAREN": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_atomic_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type in _TYPE_QUALIFIER: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "qual", append=True + ) + continue + + if tok.type in _TYPE_SPEC_SIMPLE: + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type == "TYPEID": + if saw_type: + break + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type in {"STRUCT", "UNION"}: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_struct_or_union_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type == "ENUM": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_enum_specifier(), "type", append=True + ) + saw_type = True + continue + + break + + if spec is None: + self._parse_error("Invalid specifier list", self.clex.filename) + + if not saw_type and not saw_alignment: + self._parse_error("Missing type in declaration", first_coord) + + if spec.get("storage") is None: + spec["storage"] = [] + if spec.get("function") is None: + spec["function"] = [] + + return spec + + # BNF: type_qualifier_list : type_qualifier+ + def _parse_type_qualifier_list(self) -> List[str]: + quals = [] + while self._peek_type() in _TYPE_QUALIFIER: + quals.append(self._advance().value) + return quals + + # BNF: alignment_specifier : _ALIGNAS '(' type_name | constant_expression ')' + def _parse_alignment_specifier(self) -> c_ast.Node: + tok = self._expect("_ALIGNAS") + self._expect("LPAREN") + + if self._starts_declaration(): + typ = self._parse_type_name() + self._expect("RPAREN") + return c_ast.Alignas(typ, self._tok_coord(tok)) + + expr = self._parse_constant_expression() + self._expect("RPAREN") + return c_ast.Alignas(expr, self._tok_coord(tok)) + + # BNF: atomic_specifier : _ATOMIC '(' type_name ')' + def _parse_atomic_specifier(self) -> c_ast.Node: + self._expect("_ATOMIC") + self._expect("LPAREN") + typ = self._parse_type_name() + self._expect("RPAREN") + typ.quals.append("_Atomic") + return typ + + # BNF: init_declarator_list : init_declarator (',' init_declarator)* + def _parse_init_declarator_list( + self, first: Optional["_DeclInfo"] = None, id_only: bool = False + ) -> List["_DeclInfo"]: + decls = ( + [first] + if first is not None + else [self._parse_init_declarator(id_only=id_only)] + ) + + while self._accept("COMMA"): + decls.append(self._parse_init_declarator(id_only=id_only)) + return decls + + # BNF: init_declarator : declarator ('=' initializer)? + def _parse_init_declarator(self, id_only: bool = False) -> "_DeclInfo": + decl = self._parse_id_declarator() if id_only else self._parse_declarator() + init = None + if self._accept("EQUALS"): + init = self._parse_initializer() + return dict(decl=decl, init=init, bitsize=None) + + # ------------------------------------------------------------------ + # Structs/unions/enums + # ------------------------------------------------------------------ + # BNF: struct_or_union_specifier : struct_or_union ID? '{' struct_declaration_list? '}' + # | struct_or_union ID + def _parse_struct_or_union_specifier(self) -> c_ast.Node: + tok = self._advance() + klass = self._select_struct_union_class(tok.value) + + if self._peek_type() in {"ID", "TYPEID"}: + name_tok = self._advance() + if self._peek_type() == "LBRACE": + self._advance() + if self._accept("RBRACE"): + return klass( + name=name_tok.value, decls=[], coord=self._tok_coord(name_tok) + ) + decls = self._parse_struct_declaration_list() + self._expect("RBRACE") + return klass( + name=name_tok.value, decls=decls, coord=self._tok_coord(name_tok) + ) + + return klass( + name=name_tok.value, decls=None, coord=self._tok_coord(name_tok) + ) + + if self._peek_type() == "LBRACE": + brace_tok = self._advance() + if self._accept("RBRACE"): + return klass(name=None, decls=[], coord=self._tok_coord(brace_tok)) + decls = self._parse_struct_declaration_list() + self._expect("RBRACE") + return klass(name=None, decls=decls, coord=self._tok_coord(brace_tok)) + + self._parse_error("Invalid struct/union declaration", self._tok_coord(tok)) + + # BNF: struct_declaration_list : struct_declaration+ + def _parse_struct_declaration_list(self) -> List[c_ast.Node]: + decls = [] + while self._peek_type() not in {None, "RBRACE"}: + items = self._parse_struct_declaration() + if items is None: + continue + decls.extend(items) + return decls + + # BNF: struct_declaration : specifier_qualifier_list struct_declarator_list? ';' + # | static_assert + # | pppragma_directive + def _parse_struct_declaration(self) -> Optional[List[c_ast.Node]]: + if self._peek_type() == "SEMI": + self._advance() + return None + if self._peek_type() in {"PPPRAGMA", "_PRAGMA"}: + return [self._parse_pppragma_directive()] + + spec = self._parse_specifier_qualifier_list() + assert "typedef" not in spec.get("storage", []) + + decls = None + if self._starts_declarator() or self._peek_type() == "COLON": + decls = self._parse_struct_declarator_list() + if decls is not None: + self._expect("SEMI") + return self._build_declarations(spec=spec, decls=decls) + + if len(spec["type"]) == 1: + node = spec["type"][0] + if isinstance(node, c_ast.Node): + decl_type = node + else: + decl_type = c_ast.IdentifierType(node) + self._expect("SEMI") + return self._build_declarations( + spec=spec, decls=[dict(decl=decl_type, init=None, bitsize=None)] + ) + + self._expect("SEMI") + return self._build_declarations( + spec=spec, decls=[dict(decl=None, init=None, bitsize=None)] + ) + + # BNF: struct_declarator_list : struct_declarator (',' struct_declarator)* + def _parse_struct_declarator_list(self) -> List["_DeclInfo"]: + decls = [self._parse_struct_declarator()] + while self._accept("COMMA"): + decls.append(self._parse_struct_declarator()) + return decls + + # BNF: struct_declarator : declarator? ':' constant_expression + # | declarator (':' constant_expression)? + def _parse_struct_declarator(self) -> "_DeclInfo": + if self._accept("COLON"): + bitsize = self._parse_constant_expression() + return { + "decl": c_ast.TypeDecl(None, None, None, None), + "init": None, + "bitsize": bitsize, + } + + decl = self._parse_declarator() + if self._accept("COLON"): + bitsize = self._parse_constant_expression() + return {"decl": decl, "init": None, "bitsize": bitsize} + + return {"decl": decl, "init": None, "bitsize": None} + + # BNF: enum_specifier : ENUM ID? '{' enumerator_list? '}' + # | ENUM ID + def _parse_enum_specifier(self) -> c_ast.Node: + tok = self._expect("ENUM") + if self._peek_type() in {"ID", "TYPEID"}: + name_tok = self._advance() + if self._peek_type() == "LBRACE": + self._advance() + enums = self._parse_enumerator_list() + self._expect("RBRACE") + return c_ast.Enum(name_tok.value, enums, self._tok_coord(tok)) + return c_ast.Enum(name_tok.value, None, self._tok_coord(tok)) + + self._expect("LBRACE") + enums = self._parse_enumerator_list() + self._expect("RBRACE") + return c_ast.Enum(None, enums, self._tok_coord(tok)) + + # BNF: enumerator_list : enumerator (',' enumerator)* ','? + def _parse_enumerator_list(self) -> c_ast.Node: + enum = self._parse_enumerator() + enum_list = c_ast.EnumeratorList([enum], enum.coord) + while self._accept("COMMA"): + if self._peek_type() == "RBRACE": + break + enum = self._parse_enumerator() + enum_list.enumerators.append(enum) + return enum_list + + # BNF: enumerator : ID ('=' constant_expression)? + def _parse_enumerator(self) -> c_ast.Node: + name_tok = self._expect("ID") + if self._accept("EQUALS"): + value = self._parse_constant_expression() + else: + value = None + enum = c_ast.Enumerator(name_tok.value, value, self._tok_coord(name_tok)) + self._add_identifier(enum.name, enum.coord) + return enum + + # ------------------------------------------------------------------ + # Declarators + # ------------------------------------------------------------------ + # BNF: declarator : pointer? direct_declarator + def _parse_declarator(self) -> c_ast.Node: + decl, _ = self._parse_any_declarator( + allow_abstract=False, typeid_paren_as_abstract=False + ) + assert decl is not None + return decl + + # BNF: id_declarator : declarator with ID name + def _parse_id_declarator(self) -> c_ast.Node: + return self._parse_declarator_kind(kind="id", allow_paren=True) + + # BNF: typeid_declarator : declarator with TYPEID name + def _parse_typeid_declarator(self) -> c_ast.Node: + return self._parse_declarator_kind(kind="typeid", allow_paren=True) + + # BNF: typeid_noparen_declarator : declarator without parenthesized name + def _parse_typeid_noparen_declarator(self) -> c_ast.Node: + return self._parse_declarator_kind(kind="typeid", allow_paren=False) + + # BNF: declarator_kind : pointer? direct_declarator(kind) + def _parse_declarator_kind(self, kind: str, allow_paren: bool) -> c_ast.Node: + ptr = None + if self._peek_type() == "TIMES": + ptr = self._parse_pointer() + direct = self._parse_direct_declarator(kind, allow_paren=allow_paren) + if ptr is not None: + return self._type_modify_decl(direct, ptr) + return direct + + # BNF: direct_declarator : ID | TYPEID | '(' declarator ')' + # | direct_declarator '[' ... ']' + # | direct_declarator '(' ... ')' + def _parse_direct_declarator( + self, kind: str, allow_paren: bool = True + ) -> c_ast.Node: + if allow_paren and self._accept("LPAREN"): + decl = self._parse_declarator_kind(kind, allow_paren=True) + self._expect("RPAREN") + else: + if kind == "id": + name_tok = self._expect("ID") + else: + name_tok = self._expect("TYPEID") + decl = c_ast.TypeDecl( + declname=name_tok.value, + type=None, + quals=None, + align=None, + coord=self._tok_coord(name_tok), + ) + + return self._parse_decl_suffixes(decl) + + def _parse_decl_suffixes(self, decl: c_ast.Node) -> c_ast.Node: + """Parse a chain of array/function suffixes and attach them to decl.""" + while True: + if self._peek_type() == "LBRACKET": + decl = self._type_modify_decl(decl, self._parse_array_decl(decl)) + continue + if self._peek_type() == "LPAREN": + func = self._parse_function_decl(decl) + decl = self._type_modify_decl(decl, func) + continue + break + return decl + + # BNF: array_decl : '[' array_specifiers? assignment_expression? ']' + def _parse_array_decl(self, base_decl: c_ast.Node) -> c_ast.Node: + return self._parse_array_decl_common(base_type=None, coord=base_decl.coord) + + def _parse_array_decl_common( + self, base_type: Optional[c_ast.Node], coord: Optional[Coord] = None + ) -> c_ast.Node: + """Parse an array declarator suffix and return an ArrayDecl node. + + base_type: + Base declarator node to attach (None for direct-declarator parsing, + TypeDecl for abstract declarators). + + coord: + Coordinate to use for the ArrayDecl. If None, uses the '[' token. + """ + lbrack_tok = self._expect("LBRACKET") + if coord is None: + coord = self._tok_coord(lbrack_tok) + + def make_array_decl(dim, dim_quals): + return c_ast.ArrayDecl( + type=base_type, dim=dim, dim_quals=dim_quals, coord=coord + ) + + if self._accept("STATIC"): + dim_quals = ["static"] + (self._parse_type_qualifier_list() or []) + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, dim_quals) + + if self._peek_type() in _TYPE_QUALIFIER: + dim_quals = self._parse_type_qualifier_list() or [] + if self._accept("STATIC"): + dim_quals = dim_quals + ["static"] + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, dim_quals) + times_tok = self._accept("TIMES") + if times_tok: + self._expect("RBRACKET") + dim = c_ast.ID(times_tok.value, self._tok_coord(times_tok)) + return make_array_decl(dim, dim_quals) + dim = None + if self._starts_expression(): + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, dim_quals) + + times_tok = self._accept("TIMES") + if times_tok: + self._expect("RBRACKET") + dim = c_ast.ID(times_tok.value, self._tok_coord(times_tok)) + return make_array_decl(dim, []) + + dim = None + if self._starts_expression(): + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, []) + + # BNF: function_decl : '(' parameter_type_list_opt | identifier_list_opt ')' + def _parse_function_decl(self, base_decl: c_ast.Node) -> c_ast.Node: + self._expect("LPAREN") + if self._accept("RPAREN"): + args = None + else: + args = ( + self._parse_parameter_type_list() + if self._starts_declaration() + else self._parse_identifier_list_opt() + ) + self._expect("RPAREN") + + func = c_ast.FuncDecl(args=args, type=None, coord=base_decl.coord) + + if self._peek_type() == "LBRACE": + if func.args is not None: + for param in func.args.params: + if isinstance(param, c_ast.EllipsisParam): + break + name = getattr(param, "name", None) + if name: + self._add_identifier(name, param.coord) + + return func + + # BNF: pointer : '*' type_qualifier_list? pointer? + def _parse_pointer(self) -> Optional[c_ast.Node]: + stars = [] + times_tok = self._accept("TIMES") + while times_tok: + quals = self._parse_type_qualifier_list() or [] + stars.append((quals, self._tok_coord(times_tok))) + times_tok = self._accept("TIMES") + + if not stars: + return None + + ptr = None + for quals, coord in stars: + ptr = c_ast.PtrDecl(quals=quals, type=ptr, coord=coord) + return ptr + + # BNF: parameter_type_list : parameter_list (',' ELLIPSIS)? + def _parse_parameter_type_list(self) -> c_ast.ParamList: + params = self._parse_parameter_list() + if self._peek_type() == "COMMA" and self._peek_type(2) == "ELLIPSIS": + self._advance() + ell_tok = self._advance() + params.params.append(c_ast.EllipsisParam(self._tok_coord(ell_tok))) + return params + + # BNF: parameter_list : parameter_declaration (',' parameter_declaration)* + def _parse_parameter_list(self) -> c_ast.ParamList: + first = self._parse_parameter_declaration() + params = c_ast.ParamList([first], first.coord) + while self._peek_type() == "COMMA" and self._peek_type(2) != "ELLIPSIS": + self._advance() + params.params.append(self._parse_parameter_declaration()) + return params + + # BNF: parameter_declaration : declaration_specifiers declarator? + # | declaration_specifiers abstract_declarator_opt + def _parse_parameter_declaration(self) -> c_ast.Node: + spec, _, spec_coord = self._parse_declaration_specifiers(allow_no_type=True) + + if not spec["type"]: + spec["type"] = [c_ast.IdentifierType(["int"], coord=spec_coord)] + + if self._starts_declarator(): + decl, is_named = self._parse_any_declarator( + allow_abstract=True, typeid_paren_as_abstract=True + ) + if is_named: + return self._build_declarations( + spec=spec, decls=[dict(decl=decl, init=None, bitsize=None)] + )[0] + return self._build_parameter_declaration(spec, decl, spec_coord) + + decl = self._parse_abstract_declarator_opt() + return self._build_parameter_declaration(spec, decl, spec_coord) + + def _build_parameter_declaration( + self, spec: "_DeclSpec", decl: Optional[c_ast.Node], spec_coord: Optional[Coord] + ) -> c_ast.Node: + if ( + len(spec["type"]) > 1 + and len(spec["type"][-1].names) == 1 + and self._is_type_in_scope(spec["type"][-1].names[0]) + ): + return self._build_declarations( + spec=spec, decls=[dict(decl=decl, init=None, bitsize=None)] + )[0] + + decl = c_ast.Typename( + name="", + quals=spec["qual"], + align=None, + type=decl or c_ast.TypeDecl(None, None, None, None), + coord=spec_coord, + ) + return self._fix_decl_name_type(decl, spec["type"]) + + # BNF: identifier_list_opt : identifier_list | empty + def _parse_identifier_list_opt(self) -> Optional[c_ast.Node]: + if self._peek_type() == "RPAREN": + return None + return self._parse_identifier_list() + + # BNF: identifier_list : identifier (',' identifier)* + def _parse_identifier_list(self) -> c_ast.Node: + first = self._parse_identifier() + params = c_ast.ParamList([first], first.coord) + while self._accept("COMMA"): + params.params.append(self._parse_identifier()) + return params + + # ------------------------------------------------------------------ + # Abstract declarators + # ------------------------------------------------------------------ + # BNF: type_name : specifier_qualifier_list abstract_declarator_opt + def _parse_type_name(self) -> c_ast.Typename: + spec = self._parse_specifier_qualifier_list() + decl = self._parse_abstract_declarator_opt() + + coord = None + if decl is not None: + coord = decl.coord + elif spec["type"]: + coord = spec["type"][0].coord + + typename = c_ast.Typename( + name="", + quals=spec["qual"][:], + align=None, + type=decl or c_ast.TypeDecl(None, None, None, None), + coord=coord, + ) + return cast(c_ast.Typename, self._fix_decl_name_type(typename, spec["type"])) + + # BNF: abstract_declarator_opt : pointer? direct_abstract_declarator? + def _parse_abstract_declarator_opt(self) -> Optional[c_ast.Node]: + if self._peek_type() == "TIMES": + ptr = self._parse_pointer() + if self._starts_direct_abstract_declarator(): + decl = self._parse_direct_abstract_declarator() + else: + decl = c_ast.TypeDecl(None, None, None, None) + assert ptr is not None + return self._type_modify_decl(decl, ptr) + + if self._starts_direct_abstract_declarator(): + return self._parse_direct_abstract_declarator() + + return None + + # BNF: direct_abstract_declarator : '(' parameter_type_list_opt ')' + # | '(' abstract_declarator ')' + # | '[' ... ']' + def _parse_direct_abstract_declarator(self) -> c_ast.Node: + lparen_tok = self._accept("LPAREN") + if lparen_tok: + if self._starts_declaration() or self._peek_type() == "RPAREN": + params = self._parse_parameter_type_list_opt() + self._expect("RPAREN") + decl = c_ast.FuncDecl( + args=params, + type=c_ast.TypeDecl(None, None, None, None), + coord=self._tok_coord(lparen_tok), + ) + else: + decl = self._parse_abstract_declarator_opt() + self._expect("RPAREN") + assert decl is not None + elif self._peek_type() == "LBRACKET": + decl = self._parse_abstract_array_base() + else: + self._parse_error("Invalid abstract declarator", self.clex.filename) + + return self._parse_decl_suffixes(decl) + + # BNF: parameter_type_list_opt : parameter_type_list | empty + def _parse_parameter_type_list_opt(self) -> Optional[c_ast.ParamList]: + if self._peek_type() == "RPAREN": + return None + return self._parse_parameter_type_list() + + # BNF: abstract_array_base : '[' array_specifiers? assignment_expression? ']' + def _parse_abstract_array_base(self) -> c_ast.Node: + return self._parse_array_decl_common( + base_type=c_ast.TypeDecl(None, None, None, None), coord=None + ) + + # ------------------------------------------------------------------ + # Statements + # ------------------------------------------------------------------ + # BNF: statement : labeled_statement | compound_statement + # | selection_statement | iteration_statement + # | jump_statement | expression_statement + # | static_assert | pppragma_directive + def _parse_statement(self) -> c_ast.Node | List[c_ast.Node]: + tok_type = self._peek_type() + match tok_type: + case "CASE" | "DEFAULT": + return self._parse_labeled_statement() + case "ID" if self._peek_type(2) == "COLON": + return self._parse_labeled_statement() + case "LBRACE": + return self._parse_compound_statement() + case "IF" | "SWITCH": + return self._parse_selection_statement() + case "WHILE" | "DO" | "FOR": + return self._parse_iteration_statement() + case "GOTO" | "BREAK" | "CONTINUE" | "RETURN": + return self._parse_jump_statement() + case "PPPRAGMA" | "_PRAGMA": + return self._parse_pppragma_directive() + case "_STATIC_ASSERT": + return self._parse_static_assert() + case _: + return self._parse_expression_statement() + + # BNF: pragmacomp_or_statement : pppragma_directive* statement + def _parse_pragmacomp_or_statement(self) -> c_ast.Node | List[c_ast.Node]: + if self._peek_type() in {"PPPRAGMA", "_PRAGMA"}: + pragmas = self._parse_pppragma_directive_list() + stmt = self._parse_statement() + return c_ast.Compound(block_items=pragmas + [stmt], coord=pragmas[0].coord) + return self._parse_statement() + + # BNF: block_item : declaration | statement + def _parse_block_item(self) -> c_ast.Node | List[c_ast.Node]: + if self._starts_declaration(): + return self._parse_declaration() + return self._parse_statement() + + # BNF: block_item_list : block_item+ + def _parse_block_item_list(self) -> List[c_ast.Node]: + items = [] + while self._peek_type() not in {"RBRACE", None}: + item = self._parse_block_item() + if isinstance(item, list): + if item == [None]: + continue + items.extend(item) + else: + items.append(item) + return items + + # BNF: compound_statement : '{' block_item_list? '}' + def _parse_compound_statement(self) -> c_ast.Node: + lbrace_tok = self._expect("LBRACE") + if self._accept("RBRACE"): + return c_ast.Compound(block_items=None, coord=self._tok_coord(lbrace_tok)) + block_items = self._parse_block_item_list() + self._expect("RBRACE") + return c_ast.Compound( + block_items=block_items, coord=self._tok_coord(lbrace_tok) + ) + + # BNF: labeled_statement : ID ':' statement + # | CASE constant_expression ':' statement + # | DEFAULT ':' statement + def _parse_labeled_statement(self) -> c_ast.Node: + tok_type = self._peek_type() + match tok_type: + case "ID": + name_tok = self._advance() + self._expect("COLON") + if self._starts_statement(): + stmt = self._parse_pragmacomp_or_statement() + else: + stmt = c_ast.EmptyStatement(self._tok_coord(name_tok)) + return c_ast.Label(name_tok.value, stmt, self._tok_coord(name_tok)) + case "CASE": + case_tok = self._advance() + expr = self._parse_constant_expression() + self._expect("COLON") + if self._starts_statement(): + stmt = self._parse_pragmacomp_or_statement() + else: + stmt = c_ast.EmptyStatement(self._tok_coord(case_tok)) + return c_ast.Case(expr, [stmt], self._tok_coord(case_tok)) + case "DEFAULT": + def_tok = self._advance() + self._expect("COLON") + if self._starts_statement(): + stmt = self._parse_pragmacomp_or_statement() + else: + stmt = c_ast.EmptyStatement(self._tok_coord(def_tok)) + return c_ast.Default([stmt], self._tok_coord(def_tok)) + case _: + self._parse_error("Invalid labeled statement", self.clex.filename) + + # BNF: selection_statement : IF '(' expression ')' statement (ELSE statement)? + # | SWITCH '(' expression ')' statement + def _parse_selection_statement(self) -> c_ast.Node: + tok = self._advance() + match tok.type: + case "IF": + self._expect("LPAREN") + cond = self._parse_expression() + self._expect("RPAREN") + then_stmt = self._parse_pragmacomp_or_statement() + if self._accept("ELSE"): + else_stmt = self._parse_pragmacomp_or_statement() + return c_ast.If(cond, then_stmt, else_stmt, self._tok_coord(tok)) + return c_ast.If(cond, then_stmt, None, self._tok_coord(tok)) + case "SWITCH": + self._expect("LPAREN") + expr = self._parse_expression() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return fix_switch_cases(c_ast.Switch(expr, stmt, self._tok_coord(tok))) + case _: + self._parse_error("Invalid selection statement", self._tok_coord(tok)) + + # BNF: iteration_statement : WHILE '(' expression ')' statement + # | DO statement WHILE '(' expression ')' ';' + # | FOR '(' (declaration | expression_opt) ';' + # expression_opt ';' expression_opt ')' statement + def _parse_iteration_statement(self) -> c_ast.Node: + tok = self._advance() + match tok.type: + case "WHILE": + self._expect("LPAREN") + cond = self._parse_expression() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return c_ast.While(cond, stmt, self._tok_coord(tok)) + case "DO": + stmt = self._parse_pragmacomp_or_statement() + self._expect("WHILE") + self._expect("LPAREN") + cond = self._parse_expression() + self._expect("RPAREN") + self._expect("SEMI") + return c_ast.DoWhile(cond, stmt, self._tok_coord(tok)) + case "FOR": + self._expect("LPAREN") + if self._starts_declaration(): + decls = self._parse_declaration() + init = c_ast.DeclList(decls, self._tok_coord(tok)) + cond = self._parse_expression_opt() + self._expect("SEMI") + next_expr = self._parse_expression_opt() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return c_ast.For(init, cond, next_expr, stmt, self._tok_coord(tok)) + + init = self._parse_expression_opt() + self._expect("SEMI") + cond = self._parse_expression_opt() + self._expect("SEMI") + next_expr = self._parse_expression_opt() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return c_ast.For(init, cond, next_expr, stmt, self._tok_coord(tok)) + case _: + self._parse_error("Invalid iteration statement", self._tok_coord(tok)) + + # BNF: jump_statement : GOTO ID ';' | BREAK ';' | CONTINUE ';' + # | RETURN expression? ';' + def _parse_jump_statement(self) -> c_ast.Node: + tok = self._advance() + match tok.type: + case "GOTO": + name_tok = self._expect("ID") + self._expect("SEMI") + return c_ast.Goto(name_tok.value, self._tok_coord(tok)) + case "BREAK": + self._expect("SEMI") + return c_ast.Break(self._tok_coord(tok)) + case "CONTINUE": + self._expect("SEMI") + return c_ast.Continue(self._tok_coord(tok)) + case "RETURN": + if self._accept("SEMI"): + return c_ast.Return(None, self._tok_coord(tok)) + expr = self._parse_expression() + self._expect("SEMI") + return c_ast.Return(expr, self._tok_coord(tok)) + case _: + self._parse_error("Invalid jump statement", self._tok_coord(tok)) + + # BNF: expression_statement : expression_opt ';' + def _parse_expression_statement(self) -> c_ast.Node: + expr = self._parse_expression_opt() + semi_tok = self._expect("SEMI") + if expr is None: + return c_ast.EmptyStatement(self._tok_coord(semi_tok)) + return expr + + # ------------------------------------------------------------------ + # Expressions + # ------------------------------------------------------------------ + # BNF: expression_opt : expression | empty + def _parse_expression_opt(self) -> Optional[c_ast.Node]: + if self._starts_expression(): + return self._parse_expression() + return None + + # BNF: expression : assignment_expression (',' assignment_expression)* + def _parse_expression(self) -> c_ast.Node: + expr = self._parse_assignment_expression() + if not self._accept("COMMA"): + return expr + exprs = [expr, self._parse_assignment_expression()] + while self._accept("COMMA"): + exprs.append(self._parse_assignment_expression()) + return c_ast.ExprList(exprs, expr.coord) + + # BNF: assignment_expression : conditional_expression + # | unary_expression assignment_op assignment_expression + def _parse_assignment_expression(self) -> c_ast.Node: + if self._peek_type() == "LPAREN" and self._peek_type(2) == "LBRACE": + self._advance() + comp = self._parse_compound_statement() + self._expect("RPAREN") + return comp + + expr = self._parse_conditional_expression() + if self._is_assignment_op(): + op = self._advance().value + rhs = self._parse_assignment_expression() + return c_ast.Assignment(op, expr, rhs, expr.coord) + return expr + + # BNF: conditional_expression : binary_expression + # | binary_expression '?' expression ':' conditional_expression + def _parse_conditional_expression(self) -> c_ast.Node: + expr = self._parse_binary_expression() + if self._accept("CONDOP"): + iftrue = self._parse_expression() + self._expect("COLON") + iffalse = self._parse_conditional_expression() + return c_ast.TernaryOp(expr, iftrue, iffalse, expr.coord) + return expr + + # BNF: binary_expression : cast_expression (binary_op cast_expression)* + def _parse_binary_expression( + self, min_prec: int = 0, lhs: Optional[c_ast.Node] = None + ) -> c_ast.Node: + if lhs is None: + lhs = self._parse_cast_expression() + + while True: + tok = self._peek() + if tok is None or tok.type not in _BINARY_PRECEDENCE: + break + prec = _BINARY_PRECEDENCE[tok.type] + if prec < min_prec: + break + + op = tok.value + self._advance() + rhs = self._parse_cast_expression() + + while True: + next_tok = self._peek() + if next_tok is None or next_tok.type not in _BINARY_PRECEDENCE: + break + next_prec = _BINARY_PRECEDENCE[next_tok.type] + if next_prec > prec: + rhs = self._parse_binary_expression(next_prec, rhs) + else: + break + + lhs = c_ast.BinaryOp(op, lhs, rhs, lhs.coord) + + return lhs + + # BNF: cast_expression : '(' type_name ')' cast_expression + # | unary_expression + def _parse_cast_expression(self) -> c_ast.Node: + result = self._try_parse_paren_type_name() + if result is not None: + typ, mark, lparen_tok = result + if self._peek_type() == "LBRACE": + # (type){...} is a compound literal, not a cast. Examples: + # (int){1} -> compound literal, handled in postfix + # (int) x -> cast, handled below + self._reset(mark) + else: + expr = self._parse_cast_expression() + return c_ast.Cast(typ, expr, self._tok_coord(lparen_tok)) + return self._parse_unary_expression() + + # BNF: unary_expression : postfix_expression + # | '++' unary_expression + # | '--' unary_expression + # | unary_op cast_expression + # | 'sizeof' unary_expression + # | 'sizeof' '(' type_name ')' + # | '_Alignof' '(' type_name ')' + def _parse_unary_expression(self) -> c_ast.Node: + tok_type = self._peek_type() + if tok_type in {"PLUSPLUS", "MINUSMINUS"}: + tok = self._advance() + expr = self._parse_unary_expression() + return c_ast.UnaryOp(tok.value, expr, expr.coord) + + if tok_type in {"AND", "TIMES", "PLUS", "MINUS", "NOT", "LNOT"}: + tok = self._advance() + expr = self._parse_cast_expression() + return c_ast.UnaryOp(tok.value, expr, expr.coord) + + if tok_type == "SIZEOF": + tok = self._advance() + result = self._try_parse_paren_type_name() + if result is not None: + typ, _, _ = result + return c_ast.UnaryOp(tok.value, typ, self._tok_coord(tok)) + expr = self._parse_unary_expression() + return c_ast.UnaryOp(tok.value, expr, self._tok_coord(tok)) + + if tok_type == "_ALIGNOF": + tok = self._advance() + self._expect("LPAREN") + typ = self._parse_type_name() + self._expect("RPAREN") + return c_ast.UnaryOp(tok.value, typ, self._tok_coord(tok)) + + return self._parse_postfix_expression() + + # BNF: postfix_expression : primary_expression postfix_suffix* + # | '(' type_name ')' '{' initializer_list ','? '}' + def _parse_postfix_expression(self) -> c_ast.Node: + result = self._try_parse_paren_type_name() + if result is not None: + typ, mark, _ = result + # Disambiguate between casts and compound literals: + # (int) x -> cast + # (int) {1} -> compound literal + if self._accept("LBRACE"): + init = self._parse_initializer_list() + self._accept("COMMA") + self._expect("RBRACE") + return c_ast.CompoundLiteral(typ, init) + else: + self._reset(mark) + + expr = self._parse_primary_expression() + while True: + if self._accept("LBRACKET"): + sub = self._parse_expression() + self._expect("RBRACKET") + expr = c_ast.ArrayRef(expr, sub, expr.coord) + continue + if self._accept("LPAREN"): + if self._peek_type() == "RPAREN": + self._advance() + args = None + else: + args = self._parse_argument_expression_list() + self._expect("RPAREN") + expr = c_ast.FuncCall(expr, args, expr.coord) + continue + if self._peek_type() in {"PERIOD", "ARROW"}: + op_tok = self._advance() + name_tok = self._advance() + if name_tok.type not in {"ID", "TYPEID"}: + self._parse_error( + "Invalid struct reference", self._tok_coord(name_tok) + ) + field = c_ast.ID(name_tok.value, self._tok_coord(name_tok)) + expr = c_ast.StructRef(expr, op_tok.value, field, expr.coord) + continue + if self._peek_type() in {"PLUSPLUS", "MINUSMINUS"}: + tok = self._advance() + expr = c_ast.UnaryOp("p" + tok.value, expr, expr.coord) + continue + break + return expr + + # BNF: primary_expression : ID | constant | string_literal + # | '(' expression ')' | offsetof + def _parse_primary_expression(self) -> c_ast.Node: + tok_type = self._peek_type() + if tok_type == "ID": + return self._parse_identifier() + if ( + tok_type in _INT_CONST + or tok_type in _FLOAT_CONST + or tok_type in _CHAR_CONST + ): + return self._parse_constant() + if tok_type in _STRING_LITERAL: + return self._parse_unified_string_literal() + if tok_type in _WSTR_LITERAL: + return self._parse_unified_wstring_literal() + if tok_type == "LPAREN": + self._advance() + expr = self._parse_expression() + self._expect("RPAREN") + return expr + if tok_type == "OFFSETOF": + off_tok = self._advance() + self._expect("LPAREN") + typ = self._parse_type_name() + self._expect("COMMA") + designator = self._parse_offsetof_member_designator() + self._expect("RPAREN") + coord = self._tok_coord(off_tok) + return c_ast.FuncCall( + c_ast.ID(off_tok.value, coord), + c_ast.ExprList([typ, designator], coord), + coord, + ) + + self._parse_error("Invalid expression", self.clex.filename) + + # BNF: offsetof_member_designator : identifier_or_typeid + # ('.' identifier_or_typeid | '[' expression ']')* + def _parse_offsetof_member_designator(self) -> c_ast.Node: + node = self._parse_identifier_or_typeid() + while True: + if self._accept("PERIOD"): + field = self._parse_identifier_or_typeid() + node = c_ast.StructRef(node, ".", field, node.coord) + continue + if self._accept("LBRACKET"): + expr = self._parse_expression() + self._expect("RBRACKET") + node = c_ast.ArrayRef(node, expr, node.coord) + continue + break + return node + + # BNF: argument_expression_list : assignment_expression (',' assignment_expression)* + def _parse_argument_expression_list(self) -> c_ast.Node: + expr = self._parse_assignment_expression() + exprs = [expr] + while self._accept("COMMA"): + exprs.append(self._parse_assignment_expression()) + return c_ast.ExprList(exprs, expr.coord) + + # BNF: constant_expression : conditional_expression + def _parse_constant_expression(self) -> c_ast.Node: + return self._parse_conditional_expression() + + # ------------------------------------------------------------------ + # Terminals + # ------------------------------------------------------------------ + # BNF: identifier : ID + def _parse_identifier(self) -> c_ast.Node: + tok = self._expect("ID") + return c_ast.ID(tok.value, self._tok_coord(tok)) + + # BNF: identifier_or_typeid : ID | TYPEID + def _parse_identifier_or_typeid(self) -> c_ast.Node: + tok = self._advance() + if tok.type not in {"ID", "TYPEID"}: + self._parse_error("Expected identifier", self._tok_coord(tok)) + return c_ast.ID(tok.value, self._tok_coord(tok)) + + # BNF: constant : INT_CONST | FLOAT_CONST | CHAR_CONST + def _parse_constant(self) -> c_ast.Node: + tok = self._advance() + if tok.type in _INT_CONST: + u_count = 0 + l_count = 0 + for ch in tok.value[-3:]: + if ch in ("l", "L"): + l_count += 1 + elif ch in ("u", "U"): + u_count += 1 + if u_count > 1: + raise ValueError("Constant cannot have more than one u/U suffix.") + if l_count > 2: + raise ValueError("Constant cannot have more than two l/L suffix.") + prefix = "unsigned " * u_count + "long " * l_count + return c_ast.Constant(prefix + "int", tok.value, self._tok_coord(tok)) + + if tok.type in _FLOAT_CONST: + if tok.value[-1] in ("f", "F"): + t = "float" + elif tok.value[-1] in ("l", "L"): + t = "long double" + else: + t = "double" + return c_ast.Constant(t, tok.value, self._tok_coord(tok)) + + if tok.type in _CHAR_CONST: + return c_ast.Constant("char", tok.value, self._tok_coord(tok)) + + self._parse_error("Invalid constant", self._tok_coord(tok)) + + # BNF: unified_string_literal : STRING_LITERAL+ + def _parse_unified_string_literal(self) -> c_ast.Node: + tok = self._expect("STRING_LITERAL") + node = c_ast.Constant("string", tok.value, self._tok_coord(tok)) + while self._peek_type() == "STRING_LITERAL": + tok2 = self._advance() + node.value = node.value[:-1] + tok2.value[1:] + return node + + # BNF: unified_wstring_literal : WSTRING_LITERAL+ + def _parse_unified_wstring_literal(self) -> c_ast.Node: + tok = self._advance() + if tok.type not in _WSTR_LITERAL: + self._parse_error("Invalid string literal", self._tok_coord(tok)) + node = c_ast.Constant("string", tok.value, self._tok_coord(tok)) + while self._peek_type() in _WSTR_LITERAL: + tok2 = self._advance() + node.value = node.value.rstrip()[:-1] + tok2.value[2:] + return node + + # ------------------------------------------------------------------ + # Initializers + # ------------------------------------------------------------------ + # BNF: initializer : assignment_expression + # | '{' initializer_list ','? '}' + # | '{' '}' + def _parse_initializer(self) -> c_ast.Node: + lbrace_tok = self._accept("LBRACE") + if lbrace_tok: + if self._accept("RBRACE"): + return c_ast.InitList([], self._tok_coord(lbrace_tok)) + init_list = self._parse_initializer_list() + self._accept("COMMA") + self._expect("RBRACE") + return init_list + + return self._parse_assignment_expression() + + # BNF: initializer_list : initializer_item (',' initializer_item)* ','? + def _parse_initializer_list(self) -> c_ast.Node: + items = [self._parse_initializer_item()] + while self._accept("COMMA"): + if self._peek_type() == "RBRACE": + break + items.append(self._parse_initializer_item()) + return c_ast.InitList(items, items[0].coord) + + # BNF: initializer_item : designation? initializer + def _parse_initializer_item(self) -> c_ast.Node: + designation = None + if self._peek_type() in {"LBRACKET", "PERIOD"}: + designation = self._parse_designation() + init = self._parse_initializer() + if designation is not None: + return c_ast.NamedInitializer(designation, init) + return init + + # BNF: designation : designator_list '=' + def _parse_designation(self) -> List[c_ast.Node]: + designators = self._parse_designator_list() + self._expect("EQUALS") + return designators + + # BNF: designator_list : designator+ + def _parse_designator_list(self) -> List[c_ast.Node]: + designators = [] + while self._peek_type() in {"LBRACKET", "PERIOD"}: + designators.append(self._parse_designator()) + return designators + + # BNF: designator : '[' constant_expression ']' + # | '.' identifier_or_typeid + def _parse_designator(self) -> c_ast.Node: + if self._accept("LBRACKET"): + expr = self._parse_constant_expression() + self._expect("RBRACKET") + return expr + if self._accept("PERIOD"): + return self._parse_identifier_or_typeid() + self._parse_error("Invalid designator", self.clex.filename) + + # ------------------------------------------------------------------ + # Preprocessor-like directives + # ------------------------------------------------------------------ + # BNF: pp_directive : '#' ... (unsupported) + def _parse_pp_directive(self) -> NoReturn: + tok = self._expect("PPHASH") + self._parse_error("Directives not supported yet", self._tok_coord(tok)) + + # BNF: pppragma_directive : PPPRAGMA PPPRAGMASTR? + # | _PRAGMA '(' string_literal ')' + def _parse_pppragma_directive(self) -> c_ast.Node: + if self._peek_type() == "PPPRAGMA": + tok = self._advance() + if self._peek_type() == "PPPRAGMASTR": + str_tok = self._advance() + return c_ast.Pragma(str_tok.value, self._tok_coord(str_tok)) + return c_ast.Pragma("", self._tok_coord(tok)) + + if self._peek_type() == "_PRAGMA": + tok = self._advance() + lparen = self._expect("LPAREN") + literal = self._parse_unified_string_literal() + self._expect("RPAREN") + return c_ast.Pragma(literal, self._tok_coord(lparen)) + + self._parse_error("Invalid pragma", self.clex.filename) + + # BNF: pppragma_directive_list : pppragma_directive+ + def _parse_pppragma_directive_list(self) -> List[c_ast.Node]: + pragmas = [] + while self._peek_type() in {"PPPRAGMA", "_PRAGMA"}: + pragmas.append(self._parse_pppragma_directive()) + return pragmas + + # BNF: static_assert : _STATIC_ASSERT '(' constant_expression (',' string_literal)? ')' + def _parse_static_assert(self) -> List[c_ast.Node]: + tok = self._expect("_STATIC_ASSERT") + self._expect("LPAREN") + cond = self._parse_constant_expression() + msg = None + if self._accept("COMMA"): + msg = self._parse_unified_string_literal() + self._expect("RPAREN") + return [c_ast.StaticAssert(cond, msg, self._tok_coord(tok))] + + +_ASSIGNMENT_OPS = { + "EQUALS", + "XOREQUAL", + "TIMESEQUAL", + "DIVEQUAL", + "MODEQUAL", + "PLUSEQUAL", + "MINUSEQUAL", + "LSHIFTEQUAL", + "RSHIFTEQUAL", + "ANDEQUAL", + "OREQUAL", +} + +# Precedence of operators (lower number = weather binding) +# If this changes, c_generator.CGenerator.precedence_map needs to change as +# well +_BINARY_PRECEDENCE = { + "LOR": 0, + "LAND": 1, + "OR": 2, + "XOR": 3, + "AND": 4, + "EQ": 5, + "NE": 5, + "GT": 6, + "GE": 6, + "LT": 6, + "LE": 6, + "RSHIFT": 7, + "LSHIFT": 7, + "PLUS": 8, + "MINUS": 8, + "TIMES": 9, + "DIVIDE": 9, + "MOD": 9, +} + +_STORAGE_CLASS = {"AUTO", "REGISTER", "STATIC", "EXTERN", "TYPEDEF", "_THREAD_LOCAL"} + +_FUNCTION_SPEC = {"INLINE", "_NORETURN"} + +_TYPE_QUALIFIER = {"CONST", "RESTRICT", "VOLATILE", "_ATOMIC"} + +_TYPE_SPEC_SIMPLE = { + "VOID", + "_BOOL", + "CHAR", + "SHORT", + "INT", + "LONG", + "FLOAT", + "DOUBLE", + "_COMPLEX", + "SIGNED", + "UNSIGNED", + "__INT128", +} + +_DECL_START = ( + _STORAGE_CLASS + | _FUNCTION_SPEC + | _TYPE_QUALIFIER + | _TYPE_SPEC_SIMPLE + | {"TYPEID", "STRUCT", "UNION", "ENUM", "_ALIGNAS", "_ATOMIC"} +) + +_EXPR_START = { + "ID", + "LPAREN", + "PLUSPLUS", + "MINUSMINUS", + "PLUS", + "MINUS", + "TIMES", + "AND", + "NOT", + "LNOT", + "SIZEOF", + "_ALIGNOF", + "OFFSETOF", +} + +_INT_CONST = { + "INT_CONST_DEC", + "INT_CONST_OCT", + "INT_CONST_HEX", + "INT_CONST_BIN", + "INT_CONST_CHAR", +} + +_FLOAT_CONST = {"FLOAT_CONST", "HEX_FLOAT_CONST"} + +_CHAR_CONST = { + "CHAR_CONST", + "WCHAR_CONST", + "U8CHAR_CONST", + "U16CHAR_CONST", + "U32CHAR_CONST", +} + +_STRING_LITERAL = {"STRING_LITERAL"} + +_WSTR_LITERAL = { + "WSTRING_LITERAL", + "U8STRING_LITERAL", + "U16STRING_LITERAL", + "U32STRING_LITERAL", +} + +_STARTS_EXPRESSION = ( + _EXPR_START + | _INT_CONST + | _FLOAT_CONST + | _CHAR_CONST + | _STRING_LITERAL + | _WSTR_LITERAL +) + +_STARTS_STATEMENT = { + "LBRACE", + "IF", + "SWITCH", + "WHILE", + "DO", + "FOR", + "GOTO", + "BREAK", + "CONTINUE", + "RETURN", + "CASE", + "DEFAULT", + "PPPRAGMA", + "_PRAGMA", + "_STATIC_ASSERT", + "SEMI", +} + + +class _TokenStream: + """Wraps a lexer to provide convenient, buffered access to the underlying + token stream. The lexer is expected to be initialized with the input + string already. + """ + + def __init__(self, lexer: CLexer) -> None: + self._lexer = lexer + self._buffer: List[Optional[_Token]] = [] + self._index = 0 + + def peek(self, k: int = 1) -> Optional[_Token]: + """Peek at the k-th next token in the stream, without consuming it. + + Examples: + k=1 returns the immediate next token. + k=2 returns the token after that. + """ + if k <= 0: + return None + self._fill(k) + return self._buffer[self._index + k - 1] + + def next(self) -> Optional[_Token]: + """Consume a single token and return it.""" + self._fill(1) + tok = self._buffer[self._index] + self._index += 1 + return tok + + # The 'mark' and 'reset' methods are useful for speculative parsing with + # backtracking; when the parser needs to examine a sequence of tokens + # and potentially decide to try a different path on the same sequence, it + # can call 'mark' to obtain the current token position, and if the first + # path fails restore the position with `reset(pos)`. + def mark(self) -> int: + return self._index + + def reset(self, mark: int) -> None: + self._index = mark + + def _fill(self, n: int) -> None: + while len(self._buffer) < self._index + n: + tok = self._lexer.token() + self._buffer.append(tok) + if tok is None: + break + + +# Declaration specifiers are represented by a dictionary with entries: +# - qual: a list of type qualifiers +# - storage: a list of storage class specifiers +# - type: a list of type specifiers +# - function: a list of function specifiers +# - alignment: a list of alignment specifiers +class _DeclSpec(TypedDict): + qual: List[Any] + storage: List[Any] + type: List[Any] + function: List[Any] + alignment: List[Any] + + +_DeclSpecKind = Literal["qual", "storage", "type", "function", "alignment"] + + +class _DeclInfo(TypedDict): + # Declarator payloads used by declaration/initializer parsing: + # - decl: the declarator node (may be None for abstract/implicit cases) + # - init: optional initializer expression + # - bitsize: optional bit-field width expression (for struct declarators) + decl: Optional[c_ast.Node] + init: Optional[c_ast.Node] + bitsize: Optional[c_ast.Node] diff --git a/lib/pygments-2.19.2.dist-info/INSTALLER b/lib/pygments-2.19.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/pygments-2.19.2.dist-info/METADATA b/lib/pygments-2.19.2.dist-info/METADATA new file mode 100644 index 0000000..2eff6a0 --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/METADATA @@ -0,0 +1,58 @@ +Metadata-Version: 2.4 +Name: Pygments +Version: 2.19.2 +Summary: Pygments is a syntax highlighting package written in Python. +Project-URL: Homepage, https://pygments.org +Project-URL: Documentation, https://pygments.org/docs +Project-URL: Source, https://github.com/pygments/pygments +Project-URL: Bug Tracker, https://github.com/pygments/pygments/issues +Project-URL: Changelog, https://github.com/pygments/pygments/blob/master/CHANGES +Author-email: Georg Brandl +Maintainer: Matthäus G. Chajdas +Maintainer-email: Georg Brandl , Jean Abou Samra +License: BSD-2-Clause +License-File: AUTHORS +License-File: LICENSE +Keywords: syntax highlighting +Classifier: Development Status :: 6 - Mature +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Text Processing :: Filters +Classifier: Topic :: Utilities +Requires-Python: >=3.8 +Provides-Extra: plugins +Provides-Extra: windows-terminal +Requires-Dist: colorama>=0.4.6; extra == 'windows-terminal' +Description-Content-Type: text/x-rst + +Pygments +~~~~~~~~ + +Pygments is a syntax highlighting package written in Python. + +It is a generic syntax highlighter suitable for use in code hosting, forums, +wikis or other applications that need to prettify source code. Highlights +are: + +* a wide range of over 500 languages and other text formats is supported +* special attention is paid to details, increasing quality by a fair amount +* support for new languages and formats are added easily +* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image + formats that PIL supports and ANSI sequences +* it is usable as a command-line tool and as a library + +Copyright 2006-2025 by the Pygments team, see ``AUTHORS``. +Licensed under the BSD, see ``LICENSE`` for details. diff --git a/lib/pygments-2.19.2.dist-info/RECORD b/lib/pygments-2.19.2.dist-info/RECORD new file mode 100644 index 0000000..0597c87 --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/RECORD @@ -0,0 +1,684 @@ +../../bin/pygmentize,sha256=4G-DKUzdIvdIcR07u98yCai0hK7GWRncejmgmoN5PEg,187 +pygments-2.19.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pygments-2.19.2.dist-info/METADATA,sha256=euEA1n1nAGxkeYA92DX89HqbWfrHlEQeqOZqp_WYTYI,2512 +pygments-2.19.2.dist-info/RECORD,, +pygments-2.19.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +pygments-2.19.2.dist-info/entry_points.txt,sha256=uUXw-XhMKBEX4pWcCtpuTTnPhL3h7OEE2jWi51VQsa8,53 +pygments-2.19.2.dist-info/licenses/AUTHORS,sha256=BmDjGKbyFYAq3Icxq4XQxl_yfPzKP10oWX8wZHYZW9k,10824 +pygments-2.19.2.dist-info/licenses/LICENSE,sha256=qdZvHVJt8C4p3Oc0NtNOVuhjL0bCdbvf_HBWnogvnxc,1331 +pygments/__init__.py,sha256=_3UT86TGpHuW8FekdZ8uLidEZH1NhmcLiOy2KKNPCt4,2959 +pygments/__main__.py,sha256=p8AJyoyCOMYGvzWHdnq0_A9qaaVqaj02nIu3xhJp1_4,348 +pygments/__pycache__/__init__.cpython-314.pyc,, +pygments/__pycache__/__main__.cpython-314.pyc,, +pygments/__pycache__/cmdline.cpython-314.pyc,, +pygments/__pycache__/console.cpython-314.pyc,, +pygments/__pycache__/filter.cpython-314.pyc,, +pygments/__pycache__/formatter.cpython-314.pyc,, +pygments/__pycache__/lexer.cpython-314.pyc,, +pygments/__pycache__/modeline.cpython-314.pyc,, +pygments/__pycache__/plugin.cpython-314.pyc,, +pygments/__pycache__/regexopt.cpython-314.pyc,, +pygments/__pycache__/scanner.cpython-314.pyc,, +pygments/__pycache__/sphinxext.cpython-314.pyc,, +pygments/__pycache__/style.cpython-314.pyc,, +pygments/__pycache__/token.cpython-314.pyc,, +pygments/__pycache__/unistring.cpython-314.pyc,, +pygments/__pycache__/util.cpython-314.pyc,, +pygments/cmdline.py,sha256=4pL9Kpn2PUEKPobgrsQgg-vCx2NjsrapKzQ6LxQR7Q0,23536 +pygments/console.py,sha256=AagDWqwea2yBWf10KC9ptBgMpMjxKp8yABAmh-NQOVk,1718 +pygments/filter.py,sha256=YLtpTnZiu07nY3oK9nfR6E9Y1FBHhP5PX8gvkJWcfag,1910 +pygments/filters/__init__.py,sha256=B00KqPCQh5E0XhzaDK74Qa1E4fDSTlD6b0Pvr1v-vEQ,40344 +pygments/filters/__pycache__/__init__.cpython-314.pyc,, +pygments/formatter.py,sha256=H_4J-moKkKfRWUOW9J0u7hhw6n1LiO-2Xu1q2B0sE5w,4366 +pygments/formatters/__init__.py,sha256=7OuvmoYLyoPzoOQV_brHG8GSKYB_wjFSkAQng6x2y9g,5349 +pygments/formatters/__pycache__/__init__.cpython-314.pyc,, +pygments/formatters/__pycache__/_mapping.cpython-314.pyc,, +pygments/formatters/__pycache__/bbcode.cpython-314.pyc,, +pygments/formatters/__pycache__/groff.cpython-314.pyc,, +pygments/formatters/__pycache__/html.cpython-314.pyc,, +pygments/formatters/__pycache__/img.cpython-314.pyc,, +pygments/formatters/__pycache__/irc.cpython-314.pyc,, +pygments/formatters/__pycache__/latex.cpython-314.pyc,, +pygments/formatters/__pycache__/other.cpython-314.pyc,, +pygments/formatters/__pycache__/pangomarkup.cpython-314.pyc,, +pygments/formatters/__pycache__/rtf.cpython-314.pyc,, +pygments/formatters/__pycache__/svg.cpython-314.pyc,, +pygments/formatters/__pycache__/terminal.cpython-314.pyc,, +pygments/formatters/__pycache__/terminal256.cpython-314.pyc,, +pygments/formatters/_mapping.py,sha256=1Cw37FuQlNacnxRKmtlPX4nyLoX9_ttko5ZwscNUZZ4,4176 +pygments/formatters/bbcode.py,sha256=s0Ka35OKuIchoSgEAGf6rj0rl2a9ym9L31JVNSRbZFQ,3296 +pygments/formatters/groff.py,sha256=pLcIHj4jJS_lRAVFnyJODKDu1Xlyl9_AEIdOtbl3DT0,5082 +pygments/formatters/html.py,sha256=FrHJ69FUliEyPY0zTfab0C1gPf7LXsKgeRlhwkniqIs,35953 +pygments/formatters/img.py,sha256=aRpFo8mBmWTL3sBUjRCWkeS3rc6FZrSFC4EksDrl53g,23301 +pygments/formatters/irc.py,sha256=R0Js0TYWySlI2yE9sW6tN4d4X-x3k9ZmudsijGPnLmU,4945 +pygments/formatters/latex.py,sha256=BRYtbLeW_YD1kwhhnFInhJIKylurnri8CF1lP069KWE,19258 +pygments/formatters/other.py,sha256=8pYW27sU_7XicLUqOEt2yWSO0h1IEUM3TIv34KODLwo,4986 +pygments/formatters/pangomarkup.py,sha256=pcFvEC7K1Me0EjGeOZth4oCnEY85bfqc77XzZASEPpY,2206 +pygments/formatters/rtf.py,sha256=kcKMCxTXu-2-hpgEftlGJRm7Ss-yA_Sy8OsHH_qzykA,11921 +pygments/formatters/svg.py,sha256=R6A2ME6JsMQWFiyn8wcKwFUOD6vsu-HLwiIztLu-77E,7138 +pygments/formatters/terminal.py,sha256=J_F_dFXwR9LHWvatIDnwqRYJyjVmSo1Zx8K_XDh6SyM,4626 +pygments/formatters/terminal256.py,sha256=7GQFLE5cfmeu53CAzANO74-kBk2BFkXfn5phmZjYkhM,11717 +pygments/lexer.py,sha256=ib-F_0GxHkwGpb6vWP0DeLMLc7EYgjo3hWFKN5IgOq0,35109 +pygments/lexers/__init__.py,sha256=6YhzxGKlWk38P6JpIJUQ1rVvV0DEZjEmdYsdMQ58hSk,12067 +pygments/lexers/__pycache__/__init__.cpython-314.pyc,, +pygments/lexers/__pycache__/_ada_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_asy_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_cl_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_cocoa_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_csound_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_css_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_googlesql_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_julia_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_lasso_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_lilypond_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_lua_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_luau_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_mapping.cpython-314.pyc,, +pygments/lexers/__pycache__/_mql_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_mysql_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_openedge_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_php_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_postgres_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_qlik_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_scheme_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_scilab_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_sourcemod_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_sql_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_stan_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_stata_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_tsql_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_usd_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_vbscript_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/_vim_builtins.cpython-314.pyc,, +pygments/lexers/__pycache__/actionscript.cpython-314.pyc,, +pygments/lexers/__pycache__/ada.cpython-314.pyc,, +pygments/lexers/__pycache__/agile.cpython-314.pyc,, +pygments/lexers/__pycache__/algebra.cpython-314.pyc,, +pygments/lexers/__pycache__/ambient.cpython-314.pyc,, +pygments/lexers/__pycache__/amdgpu.cpython-314.pyc,, +pygments/lexers/__pycache__/ampl.cpython-314.pyc,, +pygments/lexers/__pycache__/apdlexer.cpython-314.pyc,, +pygments/lexers/__pycache__/apl.cpython-314.pyc,, +pygments/lexers/__pycache__/archetype.cpython-314.pyc,, +pygments/lexers/__pycache__/arrow.cpython-314.pyc,, +pygments/lexers/__pycache__/arturo.cpython-314.pyc,, +pygments/lexers/__pycache__/asc.cpython-314.pyc,, +pygments/lexers/__pycache__/asm.cpython-314.pyc,, +pygments/lexers/__pycache__/asn1.cpython-314.pyc,, +pygments/lexers/__pycache__/automation.cpython-314.pyc,, +pygments/lexers/__pycache__/bare.cpython-314.pyc,, +pygments/lexers/__pycache__/basic.cpython-314.pyc,, +pygments/lexers/__pycache__/bdd.cpython-314.pyc,, +pygments/lexers/__pycache__/berry.cpython-314.pyc,, +pygments/lexers/__pycache__/bibtex.cpython-314.pyc,, +pygments/lexers/__pycache__/blueprint.cpython-314.pyc,, +pygments/lexers/__pycache__/boa.cpython-314.pyc,, +pygments/lexers/__pycache__/bqn.cpython-314.pyc,, +pygments/lexers/__pycache__/business.cpython-314.pyc,, +pygments/lexers/__pycache__/c_cpp.cpython-314.pyc,, +pygments/lexers/__pycache__/c_like.cpython-314.pyc,, +pygments/lexers/__pycache__/capnproto.cpython-314.pyc,, +pygments/lexers/__pycache__/carbon.cpython-314.pyc,, +pygments/lexers/__pycache__/cddl.cpython-314.pyc,, +pygments/lexers/__pycache__/chapel.cpython-314.pyc,, +pygments/lexers/__pycache__/clean.cpython-314.pyc,, +pygments/lexers/__pycache__/codeql.cpython-314.pyc,, +pygments/lexers/__pycache__/comal.cpython-314.pyc,, +pygments/lexers/__pycache__/compiled.cpython-314.pyc,, +pygments/lexers/__pycache__/configs.cpython-314.pyc,, +pygments/lexers/__pycache__/console.cpython-314.pyc,, +pygments/lexers/__pycache__/cplint.cpython-314.pyc,, +pygments/lexers/__pycache__/crystal.cpython-314.pyc,, +pygments/lexers/__pycache__/csound.cpython-314.pyc,, +pygments/lexers/__pycache__/css.cpython-314.pyc,, +pygments/lexers/__pycache__/d.cpython-314.pyc,, +pygments/lexers/__pycache__/dalvik.cpython-314.pyc,, +pygments/lexers/__pycache__/data.cpython-314.pyc,, +pygments/lexers/__pycache__/dax.cpython-314.pyc,, +pygments/lexers/__pycache__/devicetree.cpython-314.pyc,, +pygments/lexers/__pycache__/diff.cpython-314.pyc,, +pygments/lexers/__pycache__/dns.cpython-314.pyc,, +pygments/lexers/__pycache__/dotnet.cpython-314.pyc,, +pygments/lexers/__pycache__/dsls.cpython-314.pyc,, +pygments/lexers/__pycache__/dylan.cpython-314.pyc,, +pygments/lexers/__pycache__/ecl.cpython-314.pyc,, +pygments/lexers/__pycache__/eiffel.cpython-314.pyc,, +pygments/lexers/__pycache__/elm.cpython-314.pyc,, +pygments/lexers/__pycache__/elpi.cpython-314.pyc,, +pygments/lexers/__pycache__/email.cpython-314.pyc,, +pygments/lexers/__pycache__/erlang.cpython-314.pyc,, +pygments/lexers/__pycache__/esoteric.cpython-314.pyc,, +pygments/lexers/__pycache__/ezhil.cpython-314.pyc,, +pygments/lexers/__pycache__/factor.cpython-314.pyc,, +pygments/lexers/__pycache__/fantom.cpython-314.pyc,, +pygments/lexers/__pycache__/felix.cpython-314.pyc,, +pygments/lexers/__pycache__/fift.cpython-314.pyc,, +pygments/lexers/__pycache__/floscript.cpython-314.pyc,, +pygments/lexers/__pycache__/forth.cpython-314.pyc,, +pygments/lexers/__pycache__/fortran.cpython-314.pyc,, +pygments/lexers/__pycache__/foxpro.cpython-314.pyc,, +pygments/lexers/__pycache__/freefem.cpython-314.pyc,, +pygments/lexers/__pycache__/func.cpython-314.pyc,, +pygments/lexers/__pycache__/functional.cpython-314.pyc,, +pygments/lexers/__pycache__/futhark.cpython-314.pyc,, +pygments/lexers/__pycache__/gcodelexer.cpython-314.pyc,, +pygments/lexers/__pycache__/gdscript.cpython-314.pyc,, +pygments/lexers/__pycache__/gleam.cpython-314.pyc,, +pygments/lexers/__pycache__/go.cpython-314.pyc,, +pygments/lexers/__pycache__/grammar_notation.cpython-314.pyc,, +pygments/lexers/__pycache__/graph.cpython-314.pyc,, +pygments/lexers/__pycache__/graphics.cpython-314.pyc,, +pygments/lexers/__pycache__/graphql.cpython-314.pyc,, +pygments/lexers/__pycache__/graphviz.cpython-314.pyc,, +pygments/lexers/__pycache__/gsql.cpython-314.pyc,, +pygments/lexers/__pycache__/hare.cpython-314.pyc,, +pygments/lexers/__pycache__/haskell.cpython-314.pyc,, +pygments/lexers/__pycache__/haxe.cpython-314.pyc,, +pygments/lexers/__pycache__/hdl.cpython-314.pyc,, +pygments/lexers/__pycache__/hexdump.cpython-314.pyc,, +pygments/lexers/__pycache__/html.cpython-314.pyc,, +pygments/lexers/__pycache__/idl.cpython-314.pyc,, +pygments/lexers/__pycache__/igor.cpython-314.pyc,, +pygments/lexers/__pycache__/inferno.cpython-314.pyc,, +pygments/lexers/__pycache__/installers.cpython-314.pyc,, +pygments/lexers/__pycache__/int_fiction.cpython-314.pyc,, +pygments/lexers/__pycache__/iolang.cpython-314.pyc,, +pygments/lexers/__pycache__/j.cpython-314.pyc,, +pygments/lexers/__pycache__/javascript.cpython-314.pyc,, +pygments/lexers/__pycache__/jmespath.cpython-314.pyc,, +pygments/lexers/__pycache__/jslt.cpython-314.pyc,, +pygments/lexers/__pycache__/json5.cpython-314.pyc,, +pygments/lexers/__pycache__/jsonnet.cpython-314.pyc,, +pygments/lexers/__pycache__/jsx.cpython-314.pyc,, +pygments/lexers/__pycache__/julia.cpython-314.pyc,, +pygments/lexers/__pycache__/jvm.cpython-314.pyc,, +pygments/lexers/__pycache__/kuin.cpython-314.pyc,, +pygments/lexers/__pycache__/kusto.cpython-314.pyc,, +pygments/lexers/__pycache__/ldap.cpython-314.pyc,, +pygments/lexers/__pycache__/lean.cpython-314.pyc,, +pygments/lexers/__pycache__/lilypond.cpython-314.pyc,, +pygments/lexers/__pycache__/lisp.cpython-314.pyc,, +pygments/lexers/__pycache__/macaulay2.cpython-314.pyc,, +pygments/lexers/__pycache__/make.cpython-314.pyc,, +pygments/lexers/__pycache__/maple.cpython-314.pyc,, +pygments/lexers/__pycache__/markup.cpython-314.pyc,, +pygments/lexers/__pycache__/math.cpython-314.pyc,, +pygments/lexers/__pycache__/matlab.cpython-314.pyc,, +pygments/lexers/__pycache__/maxima.cpython-314.pyc,, +pygments/lexers/__pycache__/meson.cpython-314.pyc,, +pygments/lexers/__pycache__/mime.cpython-314.pyc,, +pygments/lexers/__pycache__/minecraft.cpython-314.pyc,, +pygments/lexers/__pycache__/mips.cpython-314.pyc,, +pygments/lexers/__pycache__/ml.cpython-314.pyc,, +pygments/lexers/__pycache__/modeling.cpython-314.pyc,, +pygments/lexers/__pycache__/modula2.cpython-314.pyc,, +pygments/lexers/__pycache__/mojo.cpython-314.pyc,, +pygments/lexers/__pycache__/monte.cpython-314.pyc,, +pygments/lexers/__pycache__/mosel.cpython-314.pyc,, +pygments/lexers/__pycache__/ncl.cpython-314.pyc,, +pygments/lexers/__pycache__/nimrod.cpython-314.pyc,, +pygments/lexers/__pycache__/nit.cpython-314.pyc,, +pygments/lexers/__pycache__/nix.cpython-314.pyc,, +pygments/lexers/__pycache__/numbair.cpython-314.pyc,, +pygments/lexers/__pycache__/oberon.cpython-314.pyc,, +pygments/lexers/__pycache__/objective.cpython-314.pyc,, +pygments/lexers/__pycache__/ooc.cpython-314.pyc,, +pygments/lexers/__pycache__/openscad.cpython-314.pyc,, +pygments/lexers/__pycache__/other.cpython-314.pyc,, +pygments/lexers/__pycache__/parasail.cpython-314.pyc,, +pygments/lexers/__pycache__/parsers.cpython-314.pyc,, +pygments/lexers/__pycache__/pascal.cpython-314.pyc,, +pygments/lexers/__pycache__/pawn.cpython-314.pyc,, +pygments/lexers/__pycache__/pddl.cpython-314.pyc,, +pygments/lexers/__pycache__/perl.cpython-314.pyc,, +pygments/lexers/__pycache__/phix.cpython-314.pyc,, +pygments/lexers/__pycache__/php.cpython-314.pyc,, +pygments/lexers/__pycache__/pointless.cpython-314.pyc,, +pygments/lexers/__pycache__/pony.cpython-314.pyc,, +pygments/lexers/__pycache__/praat.cpython-314.pyc,, +pygments/lexers/__pycache__/procfile.cpython-314.pyc,, +pygments/lexers/__pycache__/prolog.cpython-314.pyc,, +pygments/lexers/__pycache__/promql.cpython-314.pyc,, +pygments/lexers/__pycache__/prql.cpython-314.pyc,, +pygments/lexers/__pycache__/ptx.cpython-314.pyc,, +pygments/lexers/__pycache__/python.cpython-314.pyc,, +pygments/lexers/__pycache__/q.cpython-314.pyc,, +pygments/lexers/__pycache__/qlik.cpython-314.pyc,, +pygments/lexers/__pycache__/qvt.cpython-314.pyc,, +pygments/lexers/__pycache__/r.cpython-314.pyc,, +pygments/lexers/__pycache__/rdf.cpython-314.pyc,, +pygments/lexers/__pycache__/rebol.cpython-314.pyc,, +pygments/lexers/__pycache__/rego.cpython-314.pyc,, +pygments/lexers/__pycache__/resource.cpython-314.pyc,, +pygments/lexers/__pycache__/ride.cpython-314.pyc,, +pygments/lexers/__pycache__/rita.cpython-314.pyc,, +pygments/lexers/__pycache__/rnc.cpython-314.pyc,, +pygments/lexers/__pycache__/roboconf.cpython-314.pyc,, +pygments/lexers/__pycache__/robotframework.cpython-314.pyc,, +pygments/lexers/__pycache__/ruby.cpython-314.pyc,, +pygments/lexers/__pycache__/rust.cpython-314.pyc,, +pygments/lexers/__pycache__/sas.cpython-314.pyc,, +pygments/lexers/__pycache__/savi.cpython-314.pyc,, +pygments/lexers/__pycache__/scdoc.cpython-314.pyc,, +pygments/lexers/__pycache__/scripting.cpython-314.pyc,, +pygments/lexers/__pycache__/sgf.cpython-314.pyc,, +pygments/lexers/__pycache__/shell.cpython-314.pyc,, +pygments/lexers/__pycache__/sieve.cpython-314.pyc,, +pygments/lexers/__pycache__/slash.cpython-314.pyc,, +pygments/lexers/__pycache__/smalltalk.cpython-314.pyc,, +pygments/lexers/__pycache__/smithy.cpython-314.pyc,, +pygments/lexers/__pycache__/smv.cpython-314.pyc,, +pygments/lexers/__pycache__/snobol.cpython-314.pyc,, +pygments/lexers/__pycache__/solidity.cpython-314.pyc,, +pygments/lexers/__pycache__/soong.cpython-314.pyc,, +pygments/lexers/__pycache__/sophia.cpython-314.pyc,, +pygments/lexers/__pycache__/special.cpython-314.pyc,, +pygments/lexers/__pycache__/spice.cpython-314.pyc,, +pygments/lexers/__pycache__/sql.cpython-314.pyc,, +pygments/lexers/__pycache__/srcinfo.cpython-314.pyc,, +pygments/lexers/__pycache__/stata.cpython-314.pyc,, +pygments/lexers/__pycache__/supercollider.cpython-314.pyc,, +pygments/lexers/__pycache__/tablegen.cpython-314.pyc,, +pygments/lexers/__pycache__/tact.cpython-314.pyc,, +pygments/lexers/__pycache__/tal.cpython-314.pyc,, +pygments/lexers/__pycache__/tcl.cpython-314.pyc,, +pygments/lexers/__pycache__/teal.cpython-314.pyc,, +pygments/lexers/__pycache__/templates.cpython-314.pyc,, +pygments/lexers/__pycache__/teraterm.cpython-314.pyc,, +pygments/lexers/__pycache__/testing.cpython-314.pyc,, +pygments/lexers/__pycache__/text.cpython-314.pyc,, +pygments/lexers/__pycache__/textedit.cpython-314.pyc,, +pygments/lexers/__pycache__/textfmts.cpython-314.pyc,, +pygments/lexers/__pycache__/theorem.cpython-314.pyc,, +pygments/lexers/__pycache__/thingsdb.cpython-314.pyc,, +pygments/lexers/__pycache__/tlb.cpython-314.pyc,, +pygments/lexers/__pycache__/tls.cpython-314.pyc,, +pygments/lexers/__pycache__/tnt.cpython-314.pyc,, +pygments/lexers/__pycache__/trafficscript.cpython-314.pyc,, +pygments/lexers/__pycache__/typoscript.cpython-314.pyc,, +pygments/lexers/__pycache__/typst.cpython-314.pyc,, +pygments/lexers/__pycache__/ul4.cpython-314.pyc,, +pygments/lexers/__pycache__/unicon.cpython-314.pyc,, +pygments/lexers/__pycache__/urbi.cpython-314.pyc,, +pygments/lexers/__pycache__/usd.cpython-314.pyc,, +pygments/lexers/__pycache__/varnish.cpython-314.pyc,, +pygments/lexers/__pycache__/verification.cpython-314.pyc,, +pygments/lexers/__pycache__/verifpal.cpython-314.pyc,, +pygments/lexers/__pycache__/vip.cpython-314.pyc,, +pygments/lexers/__pycache__/vyper.cpython-314.pyc,, +pygments/lexers/__pycache__/web.cpython-314.pyc,, +pygments/lexers/__pycache__/webassembly.cpython-314.pyc,, +pygments/lexers/__pycache__/webidl.cpython-314.pyc,, +pygments/lexers/__pycache__/webmisc.cpython-314.pyc,, +pygments/lexers/__pycache__/wgsl.cpython-314.pyc,, +pygments/lexers/__pycache__/whiley.cpython-314.pyc,, +pygments/lexers/__pycache__/wowtoc.cpython-314.pyc,, +pygments/lexers/__pycache__/wren.cpython-314.pyc,, +pygments/lexers/__pycache__/x10.cpython-314.pyc,, +pygments/lexers/__pycache__/xorg.cpython-314.pyc,, +pygments/lexers/__pycache__/yang.cpython-314.pyc,, +pygments/lexers/__pycache__/yara.cpython-314.pyc,, +pygments/lexers/__pycache__/zig.cpython-314.pyc,, +pygments/lexers/_ada_builtins.py,sha256=CA_OnShtdc7wWh9oYcRlcrkDAQwYUKl6w7tdSbALQd4,1543 +pygments/lexers/_asy_builtins.py,sha256=cd9M00YH19w5ZL7aqucmC3nwpJGTS04U-01NLy5E2_4,27287 +pygments/lexers/_cl_builtins.py,sha256=kQeUIyZjP4kX0frkICDcKxBYQCLqzIDXa5WV5cevhDo,13994 +pygments/lexers/_cocoa_builtins.py,sha256=Ka1lLJe7JfWtdho4IFIB82X9yBvrbfHCCmEG-peXXhQ,105173 +pygments/lexers/_csound_builtins.py,sha256=qnQYKeI26ZHim316uqy_hDiRiCoHo2RHjD3sYBALyXs,18414 +pygments/lexers/_css_builtins.py,sha256=aD-dhLFXVd1Atn_bZd7gEdQn7Mhe60_VHpvZ340WzDI,12446 +pygments/lexers/_googlesql_builtins.py,sha256=IkrOk-T2v1yzbGzUEEQh5_Cf4uC_cmL_uuhwDpZlTug,16132 +pygments/lexers/_julia_builtins.py,sha256=N2WdSw5zgI2fhDat_i4YeVqurRTC_P8x71ez00SCN6U,11883 +pygments/lexers/_lasso_builtins.py,sha256=8q1gbsrMJeaeUhxIYKhaOxC9j_B-NBpq_XFj2Ze41X0,134510 +pygments/lexers/_lilypond_builtins.py,sha256=XTbGL1z1oKMoqWLEktG33jx5GdGTI9CpeO5NheEi4Y0,108094 +pygments/lexers/_lua_builtins.py,sha256=PhFdZV5-Tzz2j_q4lvG9lr84ELGfL41BhnrSDNNTaG4,8108 +pygments/lexers/_luau_builtins.py,sha256=-IDrU04kUVfjXwSQzMMpXmMYhNsQxZVVZk8cuAA0Lo0,955 +pygments/lexers/_mapping.py,sha256=9fv7xYOUAOr6LzfdFS4MDbPu78o4OQQH-2nsI1bNZf4,70438 +pygments/lexers/_mql_builtins.py,sha256=ybRQjlb7Cul0sDstnzxJl3h0qS6Ieqsr811fqrxyumU,24713 +pygments/lexers/_mysql_builtins.py,sha256=y0kAWZVAs0z2dTFJJV42OZpILgRnd8T3zSlBFv-g_oA,25838 +pygments/lexers/_openedge_builtins.py,sha256=Sz4j9-CPWIaxMa-2fZgY66j7igcu1ob1GR2UtI8zAkg,49398 +pygments/lexers/_php_builtins.py,sha256=Jd4BZpjMDELPi4EVoSxK1-8BFTc63HUwYfm1rLrGj0M,107922 +pygments/lexers/_postgres_builtins.py,sha256=Pqh4z0RBRbnW6rCQtWUdzWCJxNyqpJ7_0HOktxHDxk4,13343 +pygments/lexers/_qlik_builtins.py,sha256=xuJy9c9uZDXv6h8z582P5PrxqkxTZ_nS8gPl9OD9VN8,12595 +pygments/lexers/_scheme_builtins.py,sha256=2hNtJOJmP21lUsikpqMJ2gAmLT3Rwn_KEeqhXwCjgfk,32564 +pygments/lexers/_scilab_builtins.py,sha256=oZYPB1XPdIEz3pII11pFDe6extRRyWGA7pY06X8KZ8w,52411 +pygments/lexers/_sourcemod_builtins.py,sha256=H8AFLsNDdEpymIWOpDwbDJGCP1w-x-1gSlzPDioMF4o,26777 +pygments/lexers/_sql_builtins.py,sha256=oe8F9wWuO2iS6nEsZAdJtCUChBTjgM1Sq_aipu74jXM,6767 +pygments/lexers/_stan_builtins.py,sha256=dwi1hllM_NsaCv-aXJy7lEi57X5Hh5gSD97aCQyT9KM,13445 +pygments/lexers/_stata_builtins.py,sha256=Hqrr6j77zWU3cGGpBPohwexZci43YA4_sVYE4E1sNow,27227 +pygments/lexers/_tsql_builtins.py,sha256=Pi2RhTXcLE3glI9oxNhyVsOMn-fK_1TRxJ-EsYP5LcI,15460 +pygments/lexers/_usd_builtins.py,sha256=c9hbU1cwqBUCFIhNfu_Dob8ywv1rlPhi9w2OTj3kR8s,1658 +pygments/lexers/_vbscript_builtins.py,sha256=MqJ2ABywD21aSRtWYZRG64CCbGstC1kfsiHGJmZzxiw,4225 +pygments/lexers/_vim_builtins.py,sha256=bA4mH8t1mPPQfEiUCKEqRO1O0rL2DUG0Ux1Bt8ZSu0E,57066 +pygments/lexers/actionscript.py,sha256=JBngCe5UhYT_0dLD2j7PnPO0xRRJhmypEuQ-C5in8pY,11727 +pygments/lexers/ada.py,sha256=58k5ra1vGS4iLpW3h1ItY9ftzF3WevaeAAXzAYTiYkQ,5353 +pygments/lexers/agile.py,sha256=DN-7AVIqtG1MshA94rtSGYI_884hVHgzq405wD0_dl8,896 +pygments/lexers/algebra.py,sha256=yGTu9Tt-cQzAISQYIC5MS5a3z4QmL-tGcXnd_pkWGbk,9952 +pygments/lexers/ambient.py,sha256=UnzKpIlfSm3iitHvMd7XTMSY8TjZYYhKOC3AiARS_cE,2605 +pygments/lexers/amdgpu.py,sha256=S8qjn2UMLhBFm3Yn_c06XAGf8cl5x_ZeluelWG_-JAw,1723 +pygments/lexers/ampl.py,sha256=ZBRfDXm760gR1a1gqItnsHuoO3JdUcTBjJ5tFY9UtPA,4176 +pygments/lexers/apdlexer.py,sha256=Zr5-jgjxC8PKzRlEeclakZXPHci7FHBZghQ6wwiuT7A,30800 +pygments/lexers/apl.py,sha256=PTQMp-bxT5P-DbrEvFha10HBTcsDJ5srL3I1s9ljz58,3404 +pygments/lexers/archetype.py,sha256=pQVlP1Fb5OA8nn7QwmFaaaOSvvpoIsQVw43FVCQCve4,11538 +pygments/lexers/arrow.py,sha256=2PKdbWq3xQLF1KoDbWvSxpjwKRrznnDiArTflRGZzBo,3564 +pygments/lexers/arturo.py,sha256=U5MtRNHJtnBn4ZOeWmW6MKlVRG7SX6KhTRamDqzn9tA,11414 +pygments/lexers/asc.py,sha256=-DgZl9jccBDHPlDmjCsrEqx0-Q7ap7XVdNKtxLNWG1w,1693 +pygments/lexers/asm.py,sha256=xm2Y5mcT-sF3oQvair4SWs9EWTyndoaUoSsDy5v6shI,41967 +pygments/lexers/asn1.py,sha256=BlcloIX2bu6Q7BxGcksuhYFHGsXLVKyB4B9mFd4Pj6E,4262 +pygments/lexers/automation.py,sha256=Q61qon8EwpfakMh_2MS2E2zUUT16rG3UNIKPYjITeTs,19831 +pygments/lexers/bare.py,sha256=tWoei86JJX1k-ADhaXd5TgX6ItDTici9yFWpkTPhnfM,3020 +pygments/lexers/basic.py,sha256=qpVe5h8Fa7NJo1EihN-4R_UZpHO6my2Ssgkb-BktkKs,27989 +pygments/lexers/bdd.py,sha256=yysefcOFAEyk9kJ2y4EXmzJTecgLYUHlWixt_3YzPMU,1641 +pygments/lexers/berry.py,sha256=zxGowFb8HMIyN15-m8nmWnW6bPRR4esKtSEVugc9uXM,3209 +pygments/lexers/bibtex.py,sha256=yuNoPxwrJf9DCGUT17hxfDzbq_HtCLkQkRbBtiTVmeQ,4811 +pygments/lexers/blueprint.py,sha256=NzvWHMxCLDWt8hc6gB5jokltxVJgNa7Jwh4c61ng388,6188 +pygments/lexers/boa.py,sha256=dOot1XWNZThPIio2UyAX67K6EpISjSRCFjotD7dcnwE,3921 +pygments/lexers/bqn.py,sha256=nJiwrPKKbRF-qdai5tfqipwBkkko2P3weiZAjHUMimY,3671 +pygments/lexers/business.py,sha256=lRtekOJfsDkb12AGbuz10-G67OJrVJgCBtihTQ8_aoY,28345 +pygments/lexers/c_cpp.py,sha256=D7ZIswaHASlGBgoTlwnSqTQHf8_JyvvSt2L2q1W-F6g,18059 +pygments/lexers/c_like.py,sha256=FTGp17ds6X2rDZOHup2hH6BEn3gKK4nLm9pydNEhm0E,32021 +pygments/lexers/capnproto.py,sha256=XQJAh1WS-0ulqbTn9TdzR6gEgWLcuBqb4sj3jNsrhsY,2174 +pygments/lexers/carbon.py,sha256=av12YuTGZGpOa1Cmxp3lppx3LfSJUWbvOu0ixmUVll0,3211 +pygments/lexers/cddl.py,sha256=MKa70IwABgjBjYu15_Q9v8rsu2sr1a-i2jkiaPTI6sM,5076 +pygments/lexers/chapel.py,sha256=0n_fL3ehLC4pw4YKnmq9jxIXOJcxGPka1Wr1t1zsXPc,5156 +pygments/lexers/clean.py,sha256=dkDPAwF5BTALPeuKFoRKOSD3RfsKcGWbaRo6_G8LHng,6418 +pygments/lexers/codeql.py,sha256=ebvghn2zbrnETV4buVozMDmRCVKSdGiIN8ycLlHpGsE,2576 +pygments/lexers/comal.py,sha256=TC3NzcJ58ew5jw7qwK0kJ-okTA47psZje0yAIS39HR4,3179 +pygments/lexers/compiled.py,sha256=Slfo1sjWqcPawUwf0dIIZLBCL5pkOIoAX2S8Lxs02Mc,1426 +pygments/lexers/configs.py,sha256=wW8pY0Sa5a10pnAeTLGf48HhixQTVageIyHEf1aYMCc,50913 +pygments/lexers/console.py,sha256=-jAG120dupvV3kG3zC70brLJvSLwTFqMubBQuj_GVnU,4180 +pygments/lexers/cplint.py,sha256=DkbyE5EKydLgf6BRr1FhQrK-IeQPL7Zmjk0DVdlRFnQ,1389 +pygments/lexers/crystal.py,sha256=xU-RnpIkpjrquoxtOuOcP8fcesSJl4xhU7kO9m42LZY,15754 +pygments/lexers/csound.py,sha256=ioSw4Q04wdwjUAbnTZ1qLhUq1vxdWFxhh3QtEl5RAJc,16998 +pygments/lexers/css.py,sha256=JN1RBYsee-jrpHWrSmhN3TKc4TkOBn-_BEGpgTCzcqE,25376 +pygments/lexers/d.py,sha256=piOy0EJeiAwPHugiM3gVv0z7HNh3u2gZQoCUSASRbY4,9920 +pygments/lexers/dalvik.py,sha256=deFg2JPBktJ9mEGb9EgxNkmd6vaMjJFQVzUHo8NKIa8,4606 +pygments/lexers/data.py,sha256=o0x0SmB5ms_CPUPljEEEenOON4IQWn86DkwFjkJYCOg,27026 +pygments/lexers/dax.py,sha256=ASi73qmr7OA7cVZXF2GTYGt01Ly1vY8CgD_Pnpm8k-4,8098 +pygments/lexers/devicetree.py,sha256=RecSQCidt8DRE1QFCPUbwwR0hiRlNtsFihdGldeUn3k,4019 +pygments/lexers/diff.py,sha256=F6vxZ64wm5Nag_97de1H_3F700ZwCVnYjKvtT5jilww,5382 +pygments/lexers/dns.py,sha256=Hh5hJ7MXfrq36KgfyIRwK3X8o1LdR98IKERcV4eZ7HY,3891 +pygments/lexers/dotnet.py,sha256=NDE0kOmpe96GLO-zwNLazmj77E9ORGmKpa4ZMCXDXxQ,39441 +pygments/lexers/dsls.py,sha256=GnHKhGL5GxsRFnqC7-65NTPZLOZdmnllNrGP86x_fQE,36746 +pygments/lexers/dylan.py,sha256=7zZ1EbHWXeVHqTD36AqykKqo3fhuIh4sM-whcxUaH_Y,10409 +pygments/lexers/ecl.py,sha256=vhmpa2LBrHxsPkYcf3kPZ1ItVaLRDTebi186wY0xGZA,6371 +pygments/lexers/eiffel.py,sha256=5ydYIEFcgcMoEj4BlK31hZ0aJb8OX0RdAvuCNdlxwqw,2690 +pygments/lexers/elm.py,sha256=uRCddU8jK5vVkH6Y66y8KOsDJprIfrOgeYq3hv1PxAM,3152 +pygments/lexers/elpi.py,sha256=O9j_WKBPyvNFjCRuPciVpW4etVSnILm_T79BhCPZYmo,6877 +pygments/lexers/email.py,sha256=ZZL6yvwCRl1CEQyysuOu0lbabp5tjMutS7f3efFKGR4,4804 +pygments/lexers/erlang.py,sha256=bU11eVHvooLwmVknzN6Xkb2DMk7HbenqdNlYSzhThDM,19147 +pygments/lexers/esoteric.py,sha256=Jfp8UUKyKYsqLaqXRZT3GSM9dzkF65zduwfnH1GoGhU,10500 +pygments/lexers/ezhil.py,sha256=22r-xjvvBVpExTqCI-HycAwunDb1p5gY4tIfDmM0vDw,3272 +pygments/lexers/factor.py,sha256=urZ4En4uKFCLXdEkXLWg9EYUFGHQTTDCwNXtyq-ngok,19530 +pygments/lexers/fantom.py,sha256=JJ13-NwykD-iIESnuzCefCYeQDO95cHMJA8TasF4gHA,10231 +pygments/lexers/felix.py,sha256=F-v0si4zPtRelqzDQWXI1-tarCE-BvawziODxRU7378,9655 +pygments/lexers/fift.py,sha256=rOCwp3v5ocK5YOWvt7Td3Md--97_8e-7Sonx52uS8mA,1644 +pygments/lexers/floscript.py,sha256=aHh82k52jMuDuzl9LatrcSANJiXTCyjGU3SO53bwbb0,2667 +pygments/lexers/forth.py,sha256=ZMtsHdNbnS_0IdSYlfAlfTSPEr0MEsRo-YZriQNueTQ,7193 +pygments/lexers/fortran.py,sha256=1PE5dTxf4Df6LUeXFcmNtyeXWsC8tSiK5dYwPHIJeeQ,10382 +pygments/lexers/foxpro.py,sha256=CBkW62Fuibz3yfyelZCaEO8GGdFJWsuRhqwtsSeBwLM,26295 +pygments/lexers/freefem.py,sha256=LFBQk-m1-nNCgrl-VDH3QwnVWurvb7W29i06LoT207A,26913 +pygments/lexers/func.py,sha256=OR2rkM7gf9fKvad5WcFQln-_U_pb-RUCM9eQatToF4A,3700 +pygments/lexers/functional.py,sha256=fYT2AGZ642cRkIAId0rnXFBsx1c8LLEDRN_VuCEkUyM,693 +pygments/lexers/futhark.py,sha256=Vf1i4t-tR3zqaktVjhTzFNg_ts_9CcyA4ZDfDizbCmk,3743 +pygments/lexers/gcodelexer.py,sha256=4Xs9ax4-JZGupW_qSnHon39wQGpb-tNA3xorMKg841E,874 +pygments/lexers/gdscript.py,sha256=Ws7JKxy0M0IyZ_1iMfRvJPrizEwmeCNLDoeMIFaM-CU,7566 +pygments/lexers/gleam.py,sha256=XIlTcq6cB743pCqbNYo8PocSkjZyDPR6hHgdaJNJ1Vc,2392 +pygments/lexers/go.py,sha256=4LezefgyuqZWHzLZHieUkKTi-ssY6aHJxx7Z-LFaLK0,3783 +pygments/lexers/grammar_notation.py,sha256=LvzhRQHgwZzq9oceukZS_hwnKK58ee7Z5d0cwXOR734,8043 +pygments/lexers/graph.py,sha256=WFqoPA1c_hHYrV0i_F7-eUw3Co4_HmZY3GJ-TyDr670,4108 +pygments/lexers/graphics.py,sha256=tmF9NNALnvPnax8ywYC3pLOla45YXtp9UA0H-5EiTQY,39145 +pygments/lexers/graphql.py,sha256=O_zcrGrBaDaKTlUoJGRruxqk7CJi-NR92Y0Cs-KkCvw,5601 +pygments/lexers/graphviz.py,sha256=mzdXOMpwz9_V-be1eTAMyhkKCBl6UxCIXuq6C2yrtsw,1934 +pygments/lexers/gsql.py,sha256=VPZk9sb26-DumRkWfEaSTeoc0lx5xt5n-6eDDLezMtc,3990 +pygments/lexers/hare.py,sha256=PGCOuILktJsmtTpCZZKkMFtObfJuBpei8HM8HHuq1Tw,2649 +pygments/lexers/haskell.py,sha256=MYr74-PAC8kGJRX-dZmvZsHTc7a2u6yFS2B19LfDD7g,33262 +pygments/lexers/haxe.py,sha256=WHCy_nrXHnfLITfbdp3Ji3lqQU4HAsTUpXsLCp2_4sk,30974 +pygments/lexers/hdl.py,sha256=MOWxhmAuE4Ei0CKDqqaON7T8tl43geancrNYM136Z0U,22738 +pygments/lexers/hexdump.py,sha256=1lj9oJ-KiZXSVYvTMfGmEAQzNEW08WlMcC2I5aYvHK4,3653 +pygments/lexers/html.py,sha256=MxYTI4EeT7QxoGleCAyQq-8n_Sgly6tD95H5zanCNmk,21977 +pygments/lexers/idl.py,sha256=rcihUAGhfuGEaSW6pgFq6NzplT_pv0DagUoefg4zAmk,15449 +pygments/lexers/igor.py,sha256=wVefbUjb3ftaW3LCKGtX1JgLgiY4EmRor5gVOn8vQA8,31633 +pygments/lexers/inferno.py,sha256=ChE_5y5SLH_75Uv7D2dKWQMk2dlN6z1gY1IDjlJZ8rU,3135 +pygments/lexers/installers.py,sha256=ZHliit4Pxz1tYKOIjKkDXI5djTkpzYUMVIPR1xvUrL8,14435 +pygments/lexers/int_fiction.py,sha256=0ZzIa1sZDUQsltd1oHuS-BoNiOF8zKQfcVuDyK1Ttv8,56544 +pygments/lexers/iolang.py,sha256=L6dNDCLH0kxkIUi00fI4Z14QnRu79UcNDrgv02c5Zw8,1905 +pygments/lexers/j.py,sha256=DqNdwQGFLiZW3mCNLRg81gpmsy4Hgcai_9NP3LbWhNU,4853 +pygments/lexers/javascript.py,sha256=TGKQLSrCprCKfhLLGAq_0EOdvqvJKX9pOdKo7tCRurQ,63243 +pygments/lexers/jmespath.py,sha256=R5yA5LJ2nTIaDwnFIpSNGAThd0sAYFccwawA9xBptlg,2082 +pygments/lexers/jslt.py,sha256=OeYQf8O2_9FCaf9W6Q3a7rPdAFLthePCtVSgCrOTcl8,3700 +pygments/lexers/json5.py,sha256=8JZbc8EiTEZdKaIdQg3hXEh0mHWSzPlwd473a0nUuT0,2502 +pygments/lexers/jsonnet.py,sha256=bx2G6J4tJqGrJV1PyZrIWzWHXcoefCX-4lIxxtbn2gw,5636 +pygments/lexers/jsx.py,sha256=wGsoGSB40qAJrVfXwRPtan7OcK0O87RVsHHk0m6gogk,2693 +pygments/lexers/julia.py,sha256=0ZDJ9X83V5GqJzA6T6p0TTN8WHy2JAjvu-FSBXvfXdc,11710 +pygments/lexers/jvm.py,sha256=Yt1iQ3QodXRY-x_HUOGedhyuBBHn5jYH-I8NzOzHTlE,72667 +pygments/lexers/kuin.py,sha256=3dKKJVJlskgrvMKv2tY9NOsFfDjyo-3MLcJ1lFKdXSg,11405 +pygments/lexers/kusto.py,sha256=kaxkoPpEBDsBTCvCOkZZx7oGfv0jk_UNIRIRbfVAsBE,3477 +pygments/lexers/ldap.py,sha256=77vF4t_19x9V522cxRCM5d3HW8Ne3giYsFsMPVYYBw4,6551 +pygments/lexers/lean.py,sha256=7HWRgxFsxS1N9XKqw0vfKwaxl27s5YiVYtZeRUoTHFo,8570 +pygments/lexers/lilypond.py,sha256=yd2Tuv67um6EyCIr-VwBnlPhTHxMaQsBJ4nGgO5fjIk,9752 +pygments/lexers/lisp.py,sha256=EHUy1g4pzEsYPE-zGj2rAXm3YATE1j9dCQOr5-JPSkU,157668 +pygments/lexers/macaulay2.py,sha256=zkV-vxjQYa0Jj9TGfFP1iMgpTZ4ApQuAAIdJVGWb2is,33366 +pygments/lexers/make.py,sha256=YMI5DBCrxWca-pz9cVXcyfuHLcikPx9R_3pW_98Myqo,7831 +pygments/lexers/maple.py,sha256=Rs0dEmOMD3C1YQPd0mntN-vzReq4XfHegH6xV4lvJWo,7960 +pygments/lexers/markup.py,sha256=zWtxsyIx_1OxQzS6wLe8bEqglePv4RqvJjbia8AvV5c,65088 +pygments/lexers/math.py,sha256=P3ZK1ePd8ZnLdlmHezo2irCA8T2-nlHBoSaBoT5mEVI,695 +pygments/lexers/matlab.py,sha256=F9KO4qowIhfP8oVhCRRzE_1sqg4zmQbsB2NZH193PiM,133027 +pygments/lexers/maxima.py,sha256=a0h9Ggs9JEovTrzbJT-BLVbOqI29yPnaMZlkU5f_FeY,2715 +pygments/lexers/meson.py,sha256=BMrsDo6BH2lzTFw7JDwQ9SDNMTrRkXCNRDVf4aFHdsI,4336 +pygments/lexers/mime.py,sha256=yGrf3h37LK4b6ERBpFiL_qzn3JgOfGR5KLagnbWFl6c,7582 +pygments/lexers/minecraft.py,sha256=Nu88snDDPzM0D-742fFdUriczL-EE911pAd4_I4-pAw,13696 +pygments/lexers/mips.py,sha256=STKiZT67b3QERXXn7XKVxlPBu7vwbPC5EyCpuf3Jfbw,4656 +pygments/lexers/ml.py,sha256=t8sCv4BjvuBq6AihKKUwStEONIgdXCC2RMtO0RopNbM,35390 +pygments/lexers/modeling.py,sha256=M7B58bGB-Zwd1EmPxKqtRvg7TgNCyem3MVUHv0_H2SQ,13683 +pygments/lexers/modula2.py,sha256=NtpXBRoUCeHfflgB39LknSkCwhBHBKv2Er_pinjVsNE,53072 +pygments/lexers/mojo.py,sha256=8JRVoftN1E-W2woG0K-4n8PQXTUM9iY6Sl5sWb2uGNg,24233 +pygments/lexers/monte.py,sha256=baWU6zlXloenw9MO1MtEVGE9i3CfiXAYhqU621MIjRk,6289 +pygments/lexers/mosel.py,sha256=gjRdedhA1jTjoYoM1Gpaoog_I9o7TRbYMHk97N1TXwg,9297 +pygments/lexers/ncl.py,sha256=zJ6ahlitit4S0pBXc7Wu96PB7xOn59MwfR2HdY5_C60,63999 +pygments/lexers/nimrod.py,sha256=Q1NSqEkLC5wWt7xJyKC-vzWw_Iw2SfDNP_pyMFBuIfA,6413 +pygments/lexers/nit.py,sha256=p_hVD8GzMRl3CABVKHtYgnXFUQk0i5F2FbWFA6WXm6s,2725 +pygments/lexers/nix.py,sha256=NOrv20gdq-2A7eZ6c2gElPHv1Xx2pvv20-qOymL9GMg,4421 +pygments/lexers/numbair.py,sha256=fxkp2CXeXWKBMewfi1H4JSYkmm4kU58wZ2Sh9BDYAWQ,1758 +pygments/lexers/oberon.py,sha256=jw403qUUs7zpTHAs5CbLjb8qiuwtxLk0spDIYqGZwAw,4210 +pygments/lexers/objective.py,sha256=Fo1WB3JMj8sNeYnvB84H4_qwhOt4WNJtJWjVEOwrJGk,23297 +pygments/lexers/ooc.py,sha256=kD1XaJZaihDF_s-Vyu1Bx68S_9zFt2rhox7NF8LpOZM,3002 +pygments/lexers/openscad.py,sha256=h9I1k8kiuQmhX5vZm6VDSr2fa5Finy0sN8ZDIE-jx1c,3700 +pygments/lexers/other.py,sha256=WLVyqPsvm9oSXIbZwbfyJloS6HGgoFW5nVTaU1uQpTw,1763 +pygments/lexers/parasail.py,sha256=DWMGhtyQgGTXbIgQl_mID6CKqi-Dhbvs_dTkmvrZXfE,2719 +pygments/lexers/parsers.py,sha256=feNgxroPoWRf0NEsON2mtmKDUfslIQppukw6ndEsQ3M,26596 +pygments/lexers/pascal.py,sha256=N2tRAjlXnTxggAzzk2tOOAVzeC2MBzrXy97_HQl5n44,30989 +pygments/lexers/pawn.py,sha256=LWUYQYsebMMt2d5oxX1HYWvBqbakR1h7Av_z8Vw94Wg,8253 +pygments/lexers/pddl.py,sha256=Mk4_BzlROJCd0xR4KKRRSrbj0F7LLQcBRjmsmtWmrCg,2989 +pygments/lexers/perl.py,sha256=9BXn3tyHMA49NvzbM9E2czSCHjeU7bvaPLUcoZrhz-4,39192 +pygments/lexers/phix.py,sha256=hZqychqo5sFMBDESzDPXg1DYHQe_9sn294UfbjihaFk,23249 +pygments/lexers/php.py,sha256=l4hzQrlm0525i5dSw9Vmjcai3TzbPT6DkjzxPg9l6Zc,13061 +pygments/lexers/pointless.py,sha256=WSDjqQyGrNIGmTCdaMxl4zk7OZTlJAMzeUZ02kfgcTI,1974 +pygments/lexers/pony.py,sha256=EXrMkacqMZblI7v4AvBRQe-3Py8__bx5FOgjCLdfXxQ,3279 +pygments/lexers/praat.py,sha256=4UFK-nbC6WkZBhJgcQqEGqq9CocJkW7AmT_OJQbjWzk,12676 +pygments/lexers/procfile.py,sha256=05W2fyofLTP-FbEdSXD1eles-PPqVNfF6RWXjQdW2us,1155 +pygments/lexers/prolog.py,sha256=9Kc5YNUFqkfWu2sYoyzC3RX65abf1bm7oHr86z1s4kQ,12866 +pygments/lexers/promql.py,sha256=n-0vo-o8-ZasqP3Va4ujs562UfZSLfZF-RzT71yL0Tk,4738 +pygments/lexers/prql.py,sha256=PFReuvhbv4K5aeu6lvDfw4m-3hULkB3r43bKAy948os,8747 +pygments/lexers/ptx.py,sha256=KSHAvbiNVUntKilQ6EPYoLFocmJpRsBy_7fW6_Nrs1Y,4501 +pygments/lexers/python.py,sha256=WZe7fBAHKZ_BxPg8qIU26UGhk8qwUYyENJ3IyPW64mc,53805 +pygments/lexers/q.py,sha256=WQFUh3JrpK2j-VGW_Ytn3uJ5frUNmQIFnLtMVGRA9DI,6936 +pygments/lexers/qlik.py,sha256=2wqwdfIjrAz6RNBsP4MyeLX8Z7QpIGzxtf1CvaOlr_g,3693 +pygments/lexers/qvt.py,sha256=XMBnsWRrvCDf989OuDeb-KpszAkeETiACyaghZeL1ns,6103 +pygments/lexers/r.py,sha256=B6WgrD9SY1UTCV1fQBSlZbezPfpYsARn3FQIHcFYOiM,6474 +pygments/lexers/rdf.py,sha256=qUzxLna9v071bHhZAjdsBi8dKaJNk_h9g1ZRUAYCfoo,16056 +pygments/lexers/rebol.py,sha256=4u3N4kzui55HapopXDu3Kt0jczxDZ4buzwR7Mt4tQiM,18259 +pygments/lexers/rego.py,sha256=Rx5Gphbktr9ojg5DbqlyxHeQqqtF7g8W-oF0rmloDNY,1748 +pygments/lexers/resource.py,sha256=ioEzgWksB5HCjoz85XNkQPSd7n5kL0SZiuPkJP1hunQ,2927 +pygments/lexers/ride.py,sha256=kCWdxuR3PclVi4wiA0uUx4CYEFwuTqoMsKjhSW4X3yg,5035 +pygments/lexers/rita.py,sha256=Mj1QNxx1sWAZYC02kw8piVckaiw9B0MqQtiIiDFH0pA,1127 +pygments/lexers/rnc.py,sha256=g7ZD334PMGUqy_Ij64laSN1vJerwHqVkegfMCa3E-y8,1972 +pygments/lexers/roboconf.py,sha256=HbYuK5CqmQdd63SRY2nle01r7-p7mil0SnoauYDmEOY,2074 +pygments/lexers/robotframework.py,sha256=c4U1B9Q9ITBCTohqJTZOvkfyeVbenN4xhzSWIoZh5eU,18448 +pygments/lexers/ruby.py,sha256=uG617E5abBZcECRCqkhIfc-IbZcRb5cGuUZq_xpax90,22753 +pygments/lexers/rust.py,sha256=ZY-9vtsreBP0NfDd0WCouLSp_9MChAL8U8Abe-m9PB8,8260 +pygments/lexers/sas.py,sha256=C1Uz2s9DU6_s2kL-cB_PAGPtpyK5THlmhNmCumC1l48,9456 +pygments/lexers/savi.py,sha256=jrmruK0GnXktgBTWXW3oN3TXtofn3HBbkMlHnR84cko,4878 +pygments/lexers/scdoc.py,sha256=DXRmFDmYuc7h3gPAAVhfcL1OEbNBK5RdPpJqQzF3ZTk,2524 +pygments/lexers/scripting.py,sha256=eaYlkDK-_cAwTcCBHP6QXBCz8n6OzbhzdkRe0uV0xWY,81814 +pygments/lexers/sgf.py,sha256=w6C513ENaO2YCnqrduK7k03NaMDf-pgygvfzq2NaSRk,1985 +pygments/lexers/shell.py,sha256=dCS1zwkf5KwTog4__MnMC7h3Xmwv4_d3fnEV29tSwXI,36381 +pygments/lexers/sieve.py,sha256=eob-L84yf2jmhdNyYZUlbUJozdcd6GXcHW68lmAe8WE,2514 +pygments/lexers/slash.py,sha256=I-cRepmaxhL1SgYvD1hHX3gNBFI8NPszdU7hn1o5JlA,8484 +pygments/lexers/smalltalk.py,sha256=ue2PmqDK2sw0j75WdseiiENJBdZ1OwysH2Op1QN1r24,7204 +pygments/lexers/smithy.py,sha256=VREWoeuz7ANap_Uiopn7rs0Tnsfc-xBisDJKRGQY_y8,2659 +pygments/lexers/smv.py,sha256=He_VBSMbWONMWZmkrB5RYR0cfHVnMyKIXz68IFYl-a8,2805 +pygments/lexers/snobol.py,sha256=qDzb41xQQWMNmjB2MtZs23pFoFgZ2gbRZhK_Ir03r7I,2778 +pygments/lexers/solidity.py,sha256=Tixfnwku4Yezj6nNm8xVaw7EdV1qgAgdwahdTFP0St8,3163 +pygments/lexers/soong.py,sha256=Vm18vV4g6T8UPgjjY2yTRlSXGDpZowmuqQUBFfm4A9A,2339 +pygments/lexers/sophia.py,sha256=2YtYIT8iwAoW0B7TZuuoG_ZILhJV-2A7oBGat-98naE,3376 +pygments/lexers/special.py,sha256=8JuR2Vex8X-RWnC36S0HXTHWp2qmZclc90-TrLUWyaY,3585 +pygments/lexers/spice.py,sha256=m4nK0q4Sq_OFQez7kGWfki0No4ZV24YrONfHVj1Piqs,2790 +pygments/lexers/sql.py,sha256=WSG6vOsR87EEEwSQefP_Z7TauUG_BjqMHUFmPaSOVj4,41476 +pygments/lexers/srcinfo.py,sha256=B8vDs-sJogG3mWa5Hp_7JfHHUMyYRwGvKv6cKbFQXLM,1746 +pygments/lexers/stata.py,sha256=Zr9BC52D5O_3BbdW0N-tzoUmy0NTguL2sC-saXRVM-c,6415 +pygments/lexers/supercollider.py,sha256=_H5wDrn0DiGnlhB_cz6Rt_lo2TvqjSm0o6NPTd9R4Ko,3697 +pygments/lexers/tablegen.py,sha256=1JjedXYY18BNiY9JtNGLOtGfiwduNDZpQLBGTeQ6jAw,3987 +pygments/lexers/tact.py,sha256=X_lsxjFUMaC1TmYysXJq9tmAGifRnil83Bt1zA86Xdo,10809 +pygments/lexers/tal.py,sha256=xS9PlaWQOPj8MVr56fUNq31vUQKRWoLTlyWj9ZHm8AM,2904 +pygments/lexers/tcl.py,sha256=lK97ju4nikkt-oGOzIeyFEM98yq4dZSI8uEmYsq0R6c,5512 +pygments/lexers/teal.py,sha256=t3dqy_Arwv8_yExbX_xiFxv1TqJLPv4vh1MVKjKwS4Y,3522 +pygments/lexers/templates.py,sha256=BVdjYeoacIUuFyHTG39j4PxeNCe5E1oUURjH1rITrI4,75731 +pygments/lexers/teraterm.py,sha256=ciwztagW5Drg2gr17Qykrh6GwMsKy7e4xdQshX95GyQ,9718 +pygments/lexers/testing.py,sha256=YZgDgUEaLEYKSKEqpDsUi3Bn-Db_D42IlyiSsr1oX8U,10810 +pygments/lexers/text.py,sha256=nOCQPssIlKdVWU3PKxZiBPkf_KFM2V48IOssSyqhFY8,1068 +pygments/lexers/textedit.py,sha256=ttT4Ph-hIdgFLG6maRy_GskkziTFK0Wcg28yU0s6lek,7760 +pygments/lexers/textfmts.py,sha256=mi9KLEq4mrzDJbEc8G3VM-mSki_Tylkzodu47yH6z84,15524 +pygments/lexers/theorem.py,sha256=51ppBAEdhJmwU_lC916zMyjEoKLXqf89VAE_Lr0PNCc,17855 +pygments/lexers/thingsdb.py,sha256=x_fHNkLA-hIJyeIs6rg_X8n5OLYvFqaSu1FhI3apI5Y,6017 +pygments/lexers/tlb.py,sha256=ue2gqm45BI512lM13O8skAky9zAb7pLMrxZ8pbt5zRU,1450 +pygments/lexers/tls.py,sha256=_uQUVuMRDOhN-XUyGR5DIlVCk1CUZ1fIOSN4_WQYPKk,1540 +pygments/lexers/tnt.py,sha256=pK4LgoKON7u1xF66JYFncAPSbD8DZaeI_WTZ9HqEFlY,10456 +pygments/lexers/trafficscript.py,sha256=X3B8kgxS54ecuok9ic6Hkp-UMn5DvOmCK0p70Tz27Cw,1506 +pygments/lexers/typoscript.py,sha256=mBuePiVZUoAORPKsHwrx6fBWiy3fAIqG-2O67QmMiFI,8332 +pygments/lexers/typst.py,sha256=zIJBEhUXtWp5OiyAmvFA5m8d1EQG-ocwrJ677dvTUAk,7167 +pygments/lexers/ul4.py,sha256=rCaw0J9j3cdql9lX_HTilg65k9-9S118zOA6TAYfxaM,10499 +pygments/lexers/unicon.py,sha256=RAqoCnAAJBYOAGdR8ng0g6FtB39bGemLRlIqv5mcg9E,18625 +pygments/lexers/urbi.py,sha256=ajNP70NJg32jNnFDZsLvr_-4TToSGqRGkFyAPIJLfCU,6082 +pygments/lexers/usd.py,sha256=2eEGouolodYS402P_gtBrn4lLzpg1z8uHwPCKqjUb_k,3304 +pygments/lexers/varnish.py,sha256=dSh0Ku9SrjmlB29Fi_mWdWavN7M0cMKeepR4a34sOyI,7473 +pygments/lexers/verification.py,sha256=Qu433Q_h3EK3uS4bJoLRFZK0kIVwzX5AFKsa4Z-qnxA,3934 +pygments/lexers/verifpal.py,sha256=buyOOzCo_dGnoC40h0tthylHVVpgDt8qXu4olLvYy_4,2661 +pygments/lexers/vip.py,sha256=2lEV4cLV9p4E37wctBL7zkZ4ZU4p3HVsiLJFzB1bie0,5711 +pygments/lexers/vyper.py,sha256=Zq6sQIUBk6mBdpgOVgu3A6swGoBne0kDlRyjZznm2BY,5615 +pygments/lexers/web.py,sha256=4W9a7vcskrGJnxt4KmoE3SZydWB1qLq7lP2XS85J_m8,913 +pygments/lexers/webassembly.py,sha256=zgcMouzLawcbeFr6w_SOvGoUR68ZtqnnsbOcWEVleLk,5698 +pygments/lexers/webidl.py,sha256=ODtVmw4gVzI8HQWxuEckP6KMwm8WP2G2lSZEjagDXts,10516 +pygments/lexers/webmisc.py,sha256=-_-INDVdk47e2jlj-9bFcuLtntqVorBqIjlnwPfZFdI,40564 +pygments/lexers/wgsl.py,sha256=9igd9dzixGIgNewruv9mPnFms-c9BahkZcCCrZygv84,11880 +pygments/lexers/whiley.py,sha256=lMr750lA4MZsB4xqzVsIRtVMJIC3_dArhFYTHvOPwvA,4017 +pygments/lexers/wowtoc.py,sha256=8xxvf0xGeYtf4PE7KtkHZ_ly9xY_XXHrpCitdKE42Ro,4076 +pygments/lexers/wren.py,sha256=goGXnAMKKa13LLL40ybT3aMGPrk3gCRwZQFYAkKB_w0,3229 +pygments/lexers/x10.py,sha256=Q-AmgdF2E-N7mtOPpZ07CsxrTVnikyqC4uRRv6H75sk,1943 +pygments/lexers/xorg.py,sha256=9ttrBd3_Y2nXANsqtMposSgblYmMYqWXQ-Iz5RH9RsU,925 +pygments/lexers/yang.py,sha256=13CWbSaNr9giOHz4o0SXSklh0bfWt0ah14jJGpTvcn0,4499 +pygments/lexers/yara.py,sha256=jUSv78KTDfguCoAoAZKbYzQERkkyxBBWv5dInVrkDxo,2427 +pygments/lexers/zig.py,sha256=f-80MVOSp1KnczAMokQLVM-_wAEOD16EcGFnaCNlsN0,3976 +pygments/modeline.py,sha256=K5eSkR8GS1r5OkXXTHOcV0aM_6xpk9eWNEIAW-OOJ2g,1005 +pygments/plugin.py,sha256=tPx0rJCTIZ9ioRgLNYG4pifCbAwTRUZddvLw-NfAk2w,1891 +pygments/regexopt.py,sha256=wXaP9Gjp_hKAdnICqoDkRxAOQJSc4v3X6mcxx3z-TNs,3072 +pygments/scanner.py,sha256=nNcETRR1tRuiTaHmHSTTECVYFPcLf6mDZu1e4u91A9E,3092 +pygments/sphinxext.py,sha256=VEe_oHNgLoEGMHc2ROfbee2mF2PPREFyE6_m_JN5FvQ,7898 +pygments/style.py,sha256=Cpw9dCAyW3_JAwFRXOJXmtKb5ZwO2_5KSmlq6q4fZw4,6408 +pygments/styles/__init__.py,sha256=f9KCQXN4uKbe8aI8-L3qTC-_XPfT563FwTg6VTGVfwI,2006 +pygments/styles/__pycache__/__init__.cpython-314.pyc,, +pygments/styles/__pycache__/_mapping.cpython-314.pyc,, +pygments/styles/__pycache__/abap.cpython-314.pyc,, +pygments/styles/__pycache__/algol.cpython-314.pyc,, +pygments/styles/__pycache__/algol_nu.cpython-314.pyc,, +pygments/styles/__pycache__/arduino.cpython-314.pyc,, +pygments/styles/__pycache__/autumn.cpython-314.pyc,, +pygments/styles/__pycache__/borland.cpython-314.pyc,, +pygments/styles/__pycache__/bw.cpython-314.pyc,, +pygments/styles/__pycache__/coffee.cpython-314.pyc,, +pygments/styles/__pycache__/colorful.cpython-314.pyc,, +pygments/styles/__pycache__/default.cpython-314.pyc,, +pygments/styles/__pycache__/dracula.cpython-314.pyc,, +pygments/styles/__pycache__/emacs.cpython-314.pyc,, +pygments/styles/__pycache__/friendly.cpython-314.pyc,, +pygments/styles/__pycache__/friendly_grayscale.cpython-314.pyc,, +pygments/styles/__pycache__/fruity.cpython-314.pyc,, +pygments/styles/__pycache__/gh_dark.cpython-314.pyc,, +pygments/styles/__pycache__/gruvbox.cpython-314.pyc,, +pygments/styles/__pycache__/igor.cpython-314.pyc,, +pygments/styles/__pycache__/inkpot.cpython-314.pyc,, +pygments/styles/__pycache__/lightbulb.cpython-314.pyc,, +pygments/styles/__pycache__/lilypond.cpython-314.pyc,, +pygments/styles/__pycache__/lovelace.cpython-314.pyc,, +pygments/styles/__pycache__/manni.cpython-314.pyc,, +pygments/styles/__pycache__/material.cpython-314.pyc,, +pygments/styles/__pycache__/monokai.cpython-314.pyc,, +pygments/styles/__pycache__/murphy.cpython-314.pyc,, +pygments/styles/__pycache__/native.cpython-314.pyc,, +pygments/styles/__pycache__/nord.cpython-314.pyc,, +pygments/styles/__pycache__/onedark.cpython-314.pyc,, +pygments/styles/__pycache__/paraiso_dark.cpython-314.pyc,, +pygments/styles/__pycache__/paraiso_light.cpython-314.pyc,, +pygments/styles/__pycache__/pastie.cpython-314.pyc,, +pygments/styles/__pycache__/perldoc.cpython-314.pyc,, +pygments/styles/__pycache__/rainbow_dash.cpython-314.pyc,, +pygments/styles/__pycache__/rrt.cpython-314.pyc,, +pygments/styles/__pycache__/sas.cpython-314.pyc,, +pygments/styles/__pycache__/solarized.cpython-314.pyc,, +pygments/styles/__pycache__/staroffice.cpython-314.pyc,, +pygments/styles/__pycache__/stata_dark.cpython-314.pyc,, +pygments/styles/__pycache__/stata_light.cpython-314.pyc,, +pygments/styles/__pycache__/tango.cpython-314.pyc,, +pygments/styles/__pycache__/trac.cpython-314.pyc,, +pygments/styles/__pycache__/vim.cpython-314.pyc,, +pygments/styles/__pycache__/vs.cpython-314.pyc,, +pygments/styles/__pycache__/xcode.cpython-314.pyc,, +pygments/styles/__pycache__/zenburn.cpython-314.pyc,, +pygments/styles/_mapping.py,sha256=6lovFUE29tz6EsV3XYY4hgozJ7q1JL7cfO3UOlgnS8w,3312 +pygments/styles/abap.py,sha256=64Uwr8uPdEdcT-tE-Y2VveTXfH3SkqH9qdMgY49YHQI,749 +pygments/styles/algol.py,sha256=fCuk8ITTehvbJSufiaKlgnFsKbl-xFxxR82xhltc-cQ,2262 +pygments/styles/algol_nu.py,sha256=Gv9WfHJvYegGcUk1zcufQgsdXPNjCUNk8sAHyrSGGh4,2283 +pygments/styles/arduino.py,sha256=NoUB8xk7M1HGPoLfuySOLU0sVwoTuLcZqllXl2EO_iE,4557 +pygments/styles/autumn.py,sha256=fLLfjHXjxCl6crBAxEsBLH372ALMkFacA2bG6KFbJi4,2195 +pygments/styles/borland.py,sha256=_0ySKp4KGCSgtYjPe8uzD6gQhlmAIR4T43i-FoRYNOM,1611 +pygments/styles/bw.py,sha256=vhk8Xoj64fLPdA9IQU6mUVsYMel255jR-FDU7BjIHtI,1406 +pygments/styles/coffee.py,sha256=NqLt-fc7LONma1BGggbceVRY9uDE70WBuZXqK4zwaco,2308 +pygments/styles/colorful.py,sha256=mYcSbehtH7itH_QV9NqJp4Wna1X4lrwl2wkVXS2u-5A,2832 +pygments/styles/default.py,sha256=RTgG2zKWWUxPTDCFxhTnyZI_WZBIVgu5XsUpNvFisCA,2588 +pygments/styles/dracula.py,sha256=vRJmixBoSKV9o8NVQhXGViQqchhIYugfikLmvX0DoBw,2182 +pygments/styles/emacs.py,sha256=TiOG9oc83qToMCRMnJrXtWYqnzAqYycRz_50OoCKtxc,2535 +pygments/styles/friendly.py,sha256=oAi-l9anQTs9STDmUzXGDlOegatEOH4hpD0j6o6dZGM,2604 +pygments/styles/friendly_grayscale.py,sha256=a7Cqkzt6-uTiXvj6GoYBXzRvX5_zviCjjRB04Kf_-Q0,2828 +pygments/styles/fruity.py,sha256=GfSUTG0stlJr5Ow_saCaxbI2IB4-34Dp2TuRTpfUJBs,1324 +pygments/styles/gh_dark.py,sha256=ruNX3d4rf22rx-8HnwvGbNbXRQpXCNcHU1HNq6N4uNg,3590 +pygments/styles/gruvbox.py,sha256=KrFoHEoVnZW6XM9udyXncPomeGyZgIDsNWOH3kCrxFQ,3387 +pygments/styles/igor.py,sha256=fYYPhM0dRCvcDTMVrMVO5oFKnYm-8YVlsuVBoczFLtY,737 +pygments/styles/inkpot.py,sha256=jggSeX9NV15eOL2oJaVmZ6vmV7LWRzXJQRUqcWEqGRs,2404 +pygments/styles/lightbulb.py,sha256=Y8u1qdvlHfBqI2jJex55SkvVatVo_FjEUzE6h-X7m-0,3172 +pygments/styles/lilypond.py,sha256=Y6fp_sEL-zESmxAaMxzjtrKk90cuDC_DalNdC8wj0nw,2066 +pygments/styles/lovelace.py,sha256=cA9uhmbnzY04MccsiYSgMY7fvb4WMRbegWBUrGvXh1M,3178 +pygments/styles/manni.py,sha256=g9FyO7plTwfMm2cU4iiKgdlkMlvQLG6l2Lwkgz5ITS4,2443 +pygments/styles/material.py,sha256=LDmgomAbgtJDZhbv446_zIwgYh50UAqEEtgYNUns1rQ,4201 +pygments/styles/monokai.py,sha256=lrxTJpkBarV9gTLkBQryZ6oNSjekAVheJueKJP5iEYA,5184 +pygments/styles/murphy.py,sha256=-AKZiLkpiWej-otjHMsYCE-I-_IzCOLJY-_GBdKRZRw,2805 +pygments/styles/native.py,sha256=l6tezGSQTB8p_SyOXJ0PWI7KzCeEdtsPmVc4Yn4_CwU,2043 +pygments/styles/nord.py,sha256=GDt3WAaqaWsiCeqpIBPxd8TEUX708fGfwaA7S0w0oy0,5391 +pygments/styles/onedark.py,sha256=k80cZEppCEF-HLoxy_FEA0QmQDZze68nHVMNGyUVa28,1719 +pygments/styles/paraiso_dark.py,sha256=Jkrg4nUKIVNF8U4fPNV_Smq_g9NFbb9eiUrjYpVgQZg,5662 +pygments/styles/paraiso_light.py,sha256=MxN964ZEpze3wF0ss-igaa2I7E684MHe-Zq0rWPH3wo,5668 +pygments/styles/pastie.py,sha256=ZvAs9UpBNYFC-5PFrCRGYnm3FoPKb-eKR-ozbWZP-4g,2525 +pygments/styles/perldoc.py,sha256=HSxB93e4UpQkZspReQ34FeJbZ-59ksGvdaH-hToehi8,2230 +pygments/styles/rainbow_dash.py,sha256=4ugL18Or7aNtaLfPfCLFRiFy0Gu2RA4a9G2LQUE9SrM,2390 +pygments/styles/rrt.py,sha256=fgzfpC0PC_SCcLOMCNEIQTjPUMOncRe7SR10GfSRbXY,1006 +pygments/styles/sas.py,sha256=yzoXmbfQ2ND1WWq93b4vVGYkQSZHPqb4ymes9YYRT3w,1440 +pygments/styles/solarized.py,sha256=qupILFZn02WspnAF5SPYb-W8guo9xnUtjb1HeLw3XgE,4247 +pygments/styles/staroffice.py,sha256=CLbBeMoxay21Xyu3Af2p4xUXyG1_6ydCbvs5RJKYe5w,831 +pygments/styles/stata_dark.py,sha256=vX8SwHV__sG92F4CKribG08MJfSVq98dgs7gEA_n9yc,1257 +pygments/styles/stata_light.py,sha256=uV3GE-ylvffQ0yN3py1YAVqBB5wflIKZbceyK1Lqvrc,1289 +pygments/styles/tango.py,sha256=O2wcM4hHuU1Yt071M9CK7JPtiiSCqyxtT9tbiQICV28,7137 +pygments/styles/trac.py,sha256=9kMv1ZZyMKACWlx2fQVjRP0I2pgcRYCNrd7iGGZg9qk,1981 +pygments/styles/vim.py,sha256=J7_TqvrGkTX_XuTHW0In5wqPLAUPRWyr1122XueZWmM,2019 +pygments/styles/vs.py,sha256=s7YnzbIPuFU3LIke27mc4lAQSn2R3vbbHc1baMGSU_U,1130 +pygments/styles/xcode.py,sha256=PbQdzgGaA4a9LAU1i58alY9kM4IFlQX5jHQwOYmf_Rk,1504 +pygments/styles/zenburn.py,sha256=suZEKzBTCYdhf2cxNwcY7UATJK1tq5eYhGdBcXdf6MU,2203 +pygments/token.py,sha256=WbdWGhYm_Vosb0DDxW9lHNPgITXfWTsQmHt6cy9RbcM,6226 +pygments/unistring.py,sha256=al-_rBemRuGvinsrM6atNsHTmJ6DUbw24q2O2Ru1cBc,63208 +pygments/util.py,sha256=oRtSpiAo5jM9ulntkvVbgXUdiAW57jnuYGB7t9fYuhc,10031 diff --git a/lib/pygments-2.19.2.dist-info/WHEEL b/lib/pygments-2.19.2.dist-info/WHEEL new file mode 100644 index 0000000..12228d4 --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/pygments-2.19.2.dist-info/entry_points.txt b/lib/pygments-2.19.2.dist-info/entry_points.txt new file mode 100644 index 0000000..15498e3 --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +pygmentize = pygments.cmdline:main diff --git a/lib/pygments-2.19.2.dist-info/licenses/AUTHORS b/lib/pygments-2.19.2.dist-info/licenses/AUTHORS new file mode 100644 index 0000000..811c66a --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/licenses/AUTHORS @@ -0,0 +1,291 @@ +Pygments is written and maintained by Georg Brandl . + +Major developers are Tim Hatch and Armin Ronacher +. + +Other contributors, listed alphabetically, are: + +* Sam Aaron -- Ioke lexer +* Jean Abou Samra -- LilyPond lexer +* João Abecasis -- JSLT lexer +* Ali Afshar -- image formatter +* Thomas Aglassinger -- Easytrieve, JCL, Rexx, Transact-SQL and VBScript + lexers +* Maxence Ahlouche -- PostgreSQL Explain lexer +* Muthiah Annamalai -- Ezhil lexer +* Nikolay Antipov -- OpenSCAD lexer +* Kumar Appaiah -- Debian control lexer +* Andreas Amann -- AppleScript lexer +* Timothy Armstrong -- Dart lexer fixes +* Jeffrey Arnold -- R/S, Rd, BUGS, Jags, and Stan lexers +* Eiríkr Åsheim -- Uxntal lexer +* Jeremy Ashkenas -- CoffeeScript lexer +* José Joaquín Atria -- Praat lexer +* Stefan Matthias Aust -- Smalltalk lexer +* Lucas Bajolet -- Nit lexer +* Ben Bangert -- Mako lexers +* Max Battcher -- Darcs patch lexer +* Thomas Baruchel -- APL lexer +* Tim Baumann -- (Literate) Agda lexer +* Paul Baumgart, 280 North, Inc. -- Objective-J lexer +* Michael Bayer -- Myghty lexers +* Thomas Beale -- Archetype lexers +* John Benediktsson -- Factor lexer +* David Benjamin, Google LLC -- TLS lexer +* Trevor Bergeron -- mIRC formatter +* Vincent Bernat -- LessCSS lexer +* Christopher Bertels -- Fancy lexer +* Sébastien Bigaret -- QVT Operational lexer +* Jarrett Billingsley -- MiniD lexer +* Adam Blinkinsop -- Haskell, Redcode lexers +* Stéphane Blondon -- Procfile, SGF and Sieve lexers +* Frits van Bommel -- assembler lexers +* Pierre Bourdon -- bugfixes +* Martijn Braam -- Kernel log lexer, BARE lexer +* JD Browne, Google LLC -- GoogleSQL lexer +* Matthias Bussonnier -- ANSI style handling for terminal-256 formatter +* chebee7i -- Python traceback lexer improvements +* Hiram Chirino -- Scaml and Jade lexers +* Mauricio Caceres -- SAS and Stata lexers. +* Michael Camilleri, John Gabriele, sogaiu -- Janet lexer +* Daren Chandisingh -- Gleam lexer +* Ian Cooper -- VGL lexer +* David Corbett -- Inform, Jasmin, JSGF, Snowball, and TADS 3 lexers +* Leaf Corcoran -- MoonScript lexer +* Fraser Cormack -- TableGen lexer +* Gabriel Corona -- ASN.1 lexer +* Christopher Creutzig -- MuPAD lexer +* Daniël W. Crompton -- Pike lexer +* Pete Curry -- bugfixes +* Bryan Davis -- EBNF lexer +* Bruno Deferrari -- Shen lexer +* Walter Dörwald -- UL4 lexer +* Luke Drummond -- Meson lexer +* Giedrius Dubinskas -- HTML formatter improvements +* Owen Durni -- Haxe lexer +* Alexander Dutton, Oxford University Computing Services -- SPARQL lexer +* James Edwards -- Terraform lexer +* Nick Efford -- Python 3 lexer +* Sven Efftinge -- Xtend lexer +* Artem Egorkine -- terminal256 formatter +* Matthew Fernandez -- CAmkES lexer +* Paweł Fertyk -- GDScript lexer, HTML formatter improvements +* Michael Ficarra -- CPSA lexer +* James H. Fisher -- PostScript lexer +* Amanda Fitch, Google LLC -- GoogleSQL lexer +* William S. Fulton -- SWIG lexer +* Carlos Galdino -- Elixir and Elixir Console lexers +* Michael Galloy -- IDL lexer +* Naveen Garg -- Autohotkey lexer +* Simon Garnotel -- FreeFem++ lexer +* Laurent Gautier -- R/S lexer +* Alex Gaynor -- PyPy log lexer +* Richard Gerkin -- Igor Pro lexer +* Alain Gilbert -- TypeScript lexer +* Alex Gilding -- BlitzBasic lexer +* GitHub, Inc -- DASM16, Augeas, TOML, and Slash lexers +* Bertrand Goetzmann -- Groovy lexer +* Krzysiek Goj -- Scala lexer +* Rostyslav Golda -- FloScript lexer +* Andrey Golovizin -- BibTeX lexers +* Matt Good -- Genshi, Cheetah lexers +* Michał Górny -- vim modeline support +* Alex Gosse -- TrafficScript lexer +* Patrick Gotthardt -- PHP namespaces support +* Hubert Gruniaux -- C and C++ lexer improvements +* Olivier Guibe -- Asymptote lexer +* Phil Hagelberg -- Fennel lexer +* Florian Hahn -- Boogie lexer +* Martin Harriman -- SNOBOL lexer +* Matthew Harrison -- SVG formatter +* Steven Hazel -- Tcl lexer +* Dan Michael Heggø -- Turtle lexer +* Aslak Hellesøy -- Gherkin lexer +* Greg Hendershott -- Racket lexer +* Justin Hendrick -- ParaSail lexer +* Jordi Gutiérrez Hermoso -- Octave lexer +* David Hess, Fish Software, Inc. -- Objective-J lexer +* Ken Hilton -- Typographic Number Theory and Arrow lexers +* Varun Hiremath -- Debian control lexer +* Rob Hoelz -- Perl 6 lexer +* Doug Hogan -- Mscgen lexer +* Ben Hollis -- Mason lexer +* Max Horn -- GAP lexer +* Fred Hornsey -- OMG IDL Lexer +* Alastair Houghton -- Lexer inheritance facility +* Tim Howard -- BlitzMax lexer +* Dustin Howett -- Logos lexer +* Ivan Inozemtsev -- Fantom lexer +* Hiroaki Itoh -- Shell console rewrite, Lexers for PowerShell session, + MSDOS session, BC, WDiff +* Brian R. Jackson -- Tea lexer +* Christian Jann -- ShellSession lexer +* Jonas Camillus Jeppesen -- Line numbers and line highlighting for + RTF-formatter +* Dennis Kaarsemaker -- sources.list lexer +* Dmitri Kabak -- Inferno Limbo lexer +* Igor Kalnitsky -- vhdl lexer +* Colin Kennedy - USD lexer +* Alexander Kit -- MaskJS lexer +* Pekka Klärck -- Robot Framework lexer +* Gerwin Klein -- Isabelle lexer +* Eric Knibbe -- Lasso lexer +* Stepan Koltsov -- Clay lexer +* Oliver Kopp - Friendly grayscale style +* Adam Koprowski -- Opa lexer +* Benjamin Kowarsch -- Modula-2 lexer +* Domen Kožar -- Nix lexer +* Oleh Krekel -- Emacs Lisp lexer +* Alexander Kriegisch -- Kconfig and AspectJ lexers +* Marek Kubica -- Scheme lexer +* Jochen Kupperschmidt -- Markdown processor +* Gerd Kurzbach -- Modelica lexer +* Jon Larimer, Google Inc. -- Smali lexer +* Olov Lassus -- Dart lexer +* Matt Layman -- TAP lexer +* Dan Lazin, Google LLC -- GoogleSQL lexer +* Kristian Lyngstøl -- Varnish lexers +* Sylvestre Ledru -- Scilab lexer +* Chee Sing Lee -- Flatline lexer +* Mark Lee -- Vala lexer +* Thomas Linder Puls -- Visual Prolog lexer +* Pete Lomax -- Phix lexer +* Valentin Lorentz -- C++ lexer improvements +* Ben Mabey -- Gherkin lexer +* Angus MacArthur -- QML lexer +* Louis Mandel -- X10 lexer +* Louis Marchand -- Eiffel lexer +* Simone Margaritelli -- Hybris lexer +* Tim Martin - World of Warcraft TOC lexer +* Kirk McDonald -- D lexer +* Gordon McGregor -- SystemVerilog lexer +* Stephen McKamey -- Duel/JBST lexer +* Brian McKenna -- F# lexer +* Charles McLaughlin -- Puppet lexer +* Kurt McKee -- Tera Term macro lexer, PostgreSQL updates, MySQL overhaul, JSON lexer +* Joe Eli McIlvain -- Savi lexer +* Lukas Meuser -- BBCode formatter, Lua lexer +* Cat Miller -- Pig lexer +* Paul Miller -- LiveScript lexer +* Hong Minhee -- HTTP lexer +* Michael Mior -- Awk lexer +* Bruce Mitchener -- Dylan lexer rewrite +* Reuben Morais -- SourcePawn lexer +* Jon Morton -- Rust lexer +* Paulo Moura -- Logtalk lexer +* Mher Movsisyan -- DTD lexer +* Dejan Muhamedagic -- Crmsh lexer +* Adrien Nayrat -- PostgreSQL Explain lexer +* Ana Nelson -- Ragel, ANTLR, R console lexers +* David Neto, Google LLC -- WebGPU Shading Language lexer +* Kurt Neufeld -- Markdown lexer +* Nam T. Nguyen -- Monokai style +* Jesper Noehr -- HTML formatter "anchorlinenos" +* Mike Nolta -- Julia lexer +* Avery Nortonsmith -- Pointless lexer +* Jonas Obrist -- BBCode lexer +* Edward O'Callaghan -- Cryptol lexer +* David Oliva -- Rebol lexer +* Pat Pannuto -- nesC lexer +* Jon Parise -- Protocol buffers and Thrift lexers +* Benjamin Peterson -- Test suite refactoring +* Ronny Pfannschmidt -- BBCode lexer +* Dominik Picheta -- Nimrod lexer +* Andrew Pinkham -- RTF Formatter Refactoring +* Clément Prévost -- UrbiScript lexer +* Tanner Prynn -- cmdline -x option and loading lexers from files +* Oleh Prypin -- Crystal lexer (based on Ruby lexer) +* Nick Psaris -- K and Q lexers +* Xidorn Quan -- Web IDL lexer +* Elias Rabel -- Fortran fixed form lexer +* raichoo -- Idris lexer +* Daniel Ramirez -- GDScript lexer +* Kashif Rasul -- CUDA lexer +* Nathan Reed -- HLSL lexer +* Justin Reidy -- MXML lexer +* Jonathon Reinhart, Google LLC -- Soong lexer +* Norman Richards -- JSON lexer +* Corey Richardson -- Rust lexer updates +* Fabrizio Riguzzi -- cplint leder +* Lubomir Rintel -- GoodData MAQL and CL lexers +* Andre Roberge -- Tango style +* Georg Rollinger -- HSAIL lexer +* Michiel Roos -- TypoScript lexer +* Konrad Rudolph -- LaTeX formatter enhancements +* Mario Ruggier -- Evoque lexers +* Miikka Salminen -- Lovelace style, Hexdump lexer, lexer enhancements +* Stou Sandalski -- NumPy, FORTRAN, tcsh and XSLT lexers +* Matteo Sasso -- Common Lisp lexer +* Joe Schafer -- Ada lexer +* Max Schillinger -- TiddlyWiki5 lexer +* Andrew Schmidt -- X++ lexer +* Ken Schutte -- Matlab lexers +* René Schwaiger -- Rainbow Dash style +* Sebastian Schweizer -- Whiley lexer +* Tassilo Schweyer -- Io, MOOCode lexers +* Pablo Seminario -- PromQL lexer +* Ted Shaw -- AutoIt lexer +* Joerg Sieker -- ABAP lexer +* Robert Simmons -- Standard ML lexer +* Kirill Simonov -- YAML lexer +* Corbin Simpson -- Monte lexer +* Ville Skyttä -- ASCII armored lexer +* Alexander Smishlajev -- Visual FoxPro lexer +* Steve Spigarelli -- XQuery lexer +* Jerome St-Louis -- eC lexer +* Camil Staps -- Clean and NuSMV lexers; Solarized style +* James Strachan -- Kotlin lexer +* Tom Stuart -- Treetop lexer +* Colin Sullivan -- SuperCollider lexer +* Ben Swift -- Extempore lexer +* tatt61880 -- Kuin lexer +* Edoardo Tenani -- Arduino lexer +* Tiberius Teng -- default style overhaul +* Jeremy Thurgood -- Erlang, Squid config lexers +* Brian Tiffin -- OpenCOBOL lexer +* Bob Tolbert -- Hy lexer +* Doug Torrance -- Macaulay2 lexer +* Matthias Trute -- Forth lexer +* Tuoa Spi T4 -- Bdd lexer +* Erick Tryzelaar -- Felix lexer +* Alexander Udalov -- Kotlin lexer improvements +* Thomas Van Doren -- Chapel lexer +* Dave Van Ee -- Uxntal lexer updates +* Daniele Varrazzo -- PostgreSQL lexers +* Abe Voelker -- OpenEdge ABL lexer +* Pepijn de Vos -- HTML formatter CTags support +* Matthias Vallentin -- Bro lexer +* Benoît Vinot -- AMPL lexer +* Linh Vu Hong -- RSL lexer +* Taavi Väänänen -- Debian control lexer +* Immanuel Washington -- Smithy lexer +* Nathan Weizenbaum -- Haml and Sass lexers +* Nathan Whetsell -- Csound lexers +* Dietmar Winkler -- Modelica lexer +* Nils Winter -- Smalltalk lexer +* Davy Wybiral -- Clojure lexer +* Whitney Young -- ObjectiveC lexer +* Diego Zamboni -- CFengine3 lexer +* Enrique Zamudio -- Ceylon lexer +* Alex Zimin -- Nemerle lexer +* Rob Zimmerman -- Kal lexer +* Evgenii Zheltonozhskii -- Maple lexer +* Vincent Zurczak -- Roboconf lexer +* Hubert Gruniaux -- C and C++ lexer improvements +* Thomas Symalla -- AMDGPU Lexer +* 15b3 -- Image Formatter improvements +* Fabian Neumann -- CDDL lexer +* Thomas Duboucher -- CDDL lexer +* Philipp Imhof -- Pango Markup formatter +* Thomas Voss -- Sed lexer +* Martin Fischer -- WCAG contrast testing +* Marc Auberer -- Spice lexer +* Amr Hesham -- Carbon lexer +* diskdance -- Wikitext lexer +* vanillajonathan -- PRQL lexer +* Nikolay Antipov -- OpenSCAD lexer +* Markus Meyer, Nextron Systems -- YARA lexer +* Hannes Römer -- Mojo lexer +* Jan Frederik Schaefer -- PDDL lexer + +Many thanks for all contributions! diff --git a/lib/pygments-2.19.2.dist-info/licenses/LICENSE b/lib/pygments-2.19.2.dist-info/licenses/LICENSE new file mode 100644 index 0000000..446a1a8 --- /dev/null +++ b/lib/pygments-2.19.2.dist-info/licenses/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2006-2022 by the respective authors (see AUTHORS file). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/pygments/__init__.py b/lib/pygments/__init__.py new file mode 100644 index 0000000..2a391c3 --- /dev/null +++ b/lib/pygments/__init__.py @@ -0,0 +1,82 @@ +""" + Pygments + ~~~~~~~~ + + Pygments is a syntax highlighting package written in Python. + + It is a generic syntax highlighter for general use in all kinds of software + such as forum systems, wikis or other applications that need to prettify + source code. Highlights are: + + * a wide range of common languages and markup formats is supported + * special attention is paid to details, increasing quality by a fair amount + * support for new languages and formats are added easily + * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image + formats that PIL supports, and ANSI sequences + * it is usable as a command-line tool and as a library + * ... and it highlights even Brainfuck! + + The `Pygments master branch`_ is installable with ``easy_install Pygments==dev``. + + .. _Pygments master branch: + https://github.com/pygments/pygments/archive/master.zip#egg=Pygments-dev + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +from io import StringIO, BytesIO + +__version__ = '2.19.2' +__docformat__ = 'restructuredtext' + +__all__ = ['lex', 'format', 'highlight'] + + +def lex(code, lexer): + """ + Lex `code` with the `lexer` (must be a `Lexer` instance) + and return an iterable of tokens. Currently, this only calls + `lexer.get_tokens()`. + """ + try: + return lexer.get_tokens(code) + except TypeError: + # Heuristic to catch a common mistake. + from pygments.lexer import RegexLexer + if isinstance(lexer, type) and issubclass(lexer, RegexLexer): + raise TypeError('lex() argument must be a lexer instance, ' + 'not a class') + raise + + +def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builtin + """ + Format ``tokens`` (an iterable of tokens) with the formatter ``formatter`` + (a `Formatter` instance). + + If ``outfile`` is given and a valid file object (an object with a + ``write`` method), the result will be written to it, otherwise it + is returned as a string. + """ + try: + if not outfile: + realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO() + formatter.format(tokens, realoutfile) + return realoutfile.getvalue() + else: + formatter.format(tokens, outfile) + except TypeError: + # Heuristic to catch a common mistake. + from pygments.formatter import Formatter + if isinstance(formatter, type) and issubclass(formatter, Formatter): + raise TypeError('format() argument must be a formatter instance, ' + 'not a class') + raise + + +def highlight(code, lexer, formatter, outfile=None): + """ + This is the most high-level highlighting function. It combines `lex` and + `format` in one function. + """ + return format(lex(code, lexer), formatter, outfile) diff --git a/lib/pygments/__main__.py b/lib/pygments/__main__.py new file mode 100644 index 0000000..4890a6c --- /dev/null +++ b/lib/pygments/__main__.py @@ -0,0 +1,17 @@ +""" + pygments.__main__ + ~~~~~~~~~~~~~~~~~ + + Main entry point for ``python -m pygments``. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import sys +import pygments.cmdline + +try: + sys.exit(pygments.cmdline.main(sys.argv)) +except KeyboardInterrupt: + sys.exit(1) diff --git a/lib/pygments/__pycache__/__init__.cpython-314.pyc b/lib/pygments/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..d06518f Binary files /dev/null and b/lib/pygments/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/__main__.cpython-314.pyc b/lib/pygments/__pycache__/__main__.cpython-314.pyc new file mode 100644 index 0000000..b87d591 Binary files /dev/null and b/lib/pygments/__pycache__/__main__.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/cmdline.cpython-314.pyc b/lib/pygments/__pycache__/cmdline.cpython-314.pyc new file mode 100644 index 0000000..57fa792 Binary files /dev/null and b/lib/pygments/__pycache__/cmdline.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/console.cpython-314.pyc b/lib/pygments/__pycache__/console.cpython-314.pyc new file mode 100644 index 0000000..0736d5f Binary files /dev/null and b/lib/pygments/__pycache__/console.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/filter.cpython-314.pyc b/lib/pygments/__pycache__/filter.cpython-314.pyc new file mode 100644 index 0000000..da56e1b Binary files /dev/null and b/lib/pygments/__pycache__/filter.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/formatter.cpython-314.pyc b/lib/pygments/__pycache__/formatter.cpython-314.pyc new file mode 100644 index 0000000..f30154c Binary files /dev/null and b/lib/pygments/__pycache__/formatter.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/lexer.cpython-314.pyc b/lib/pygments/__pycache__/lexer.cpython-314.pyc new file mode 100644 index 0000000..baf43d5 Binary files /dev/null and b/lib/pygments/__pycache__/lexer.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/modeline.cpython-314.pyc b/lib/pygments/__pycache__/modeline.cpython-314.pyc new file mode 100644 index 0000000..d0062bd Binary files /dev/null and b/lib/pygments/__pycache__/modeline.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/plugin.cpython-314.pyc b/lib/pygments/__pycache__/plugin.cpython-314.pyc new file mode 100644 index 0000000..ebde555 Binary files /dev/null and b/lib/pygments/__pycache__/plugin.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/regexopt.cpython-314.pyc b/lib/pygments/__pycache__/regexopt.cpython-314.pyc new file mode 100644 index 0000000..f49d8b9 Binary files /dev/null and b/lib/pygments/__pycache__/regexopt.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/scanner.cpython-314.pyc b/lib/pygments/__pycache__/scanner.cpython-314.pyc new file mode 100644 index 0000000..e919f58 Binary files /dev/null and b/lib/pygments/__pycache__/scanner.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/sphinxext.cpython-314.pyc b/lib/pygments/__pycache__/sphinxext.cpython-314.pyc new file mode 100644 index 0000000..a12340d Binary files /dev/null and b/lib/pygments/__pycache__/sphinxext.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/style.cpython-314.pyc b/lib/pygments/__pycache__/style.cpython-314.pyc new file mode 100644 index 0000000..5ec8a15 Binary files /dev/null and b/lib/pygments/__pycache__/style.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/token.cpython-314.pyc b/lib/pygments/__pycache__/token.cpython-314.pyc new file mode 100644 index 0000000..c4d1977 Binary files /dev/null and b/lib/pygments/__pycache__/token.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/unistring.cpython-314.pyc b/lib/pygments/__pycache__/unistring.cpython-314.pyc new file mode 100644 index 0000000..cbabb6c Binary files /dev/null and b/lib/pygments/__pycache__/unistring.cpython-314.pyc differ diff --git a/lib/pygments/__pycache__/util.cpython-314.pyc b/lib/pygments/__pycache__/util.cpython-314.pyc new file mode 100644 index 0000000..b354c25 Binary files /dev/null and b/lib/pygments/__pycache__/util.cpython-314.pyc differ diff --git a/lib/pygments/cmdline.py b/lib/pygments/cmdline.py new file mode 100644 index 0000000..2878fd5 --- /dev/null +++ b/lib/pygments/cmdline.py @@ -0,0 +1,668 @@ +""" + pygments.cmdline + ~~~~~~~~~~~~~~~~ + + Command line interface. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import os +import sys +import shutil +import argparse +from textwrap import dedent + +from pygments import __version__, highlight +from pygments.util import ClassNotFound, OptionError, docstring_headline, \ + guess_decode, guess_decode_from_terminal, terminal_encoding, \ + UnclosingTextIOWrapper +from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \ + load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename +from pygments.lexers.special import TextLexer +from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter +from pygments.formatters import get_all_formatters, get_formatter_by_name, \ + load_formatter_from_file, get_formatter_for_filename, find_formatter_class +from pygments.formatters.terminal import TerminalFormatter +from pygments.formatters.terminal256 import Terminal256Formatter, TerminalTrueColorFormatter +from pygments.filters import get_all_filters, find_filter_class +from pygments.styles import get_all_styles, get_style_by_name + + +def _parse_options(o_strs): + opts = {} + if not o_strs: + return opts + for o_str in o_strs: + if not o_str.strip(): + continue + o_args = o_str.split(',') + for o_arg in o_args: + o_arg = o_arg.strip() + try: + o_key, o_val = o_arg.split('=', 1) + o_key = o_key.strip() + o_val = o_val.strip() + except ValueError: + opts[o_arg] = True + else: + opts[o_key] = o_val + return opts + + +def _parse_filters(f_strs): + filters = [] + if not f_strs: + return filters + for f_str in f_strs: + if ':' in f_str: + fname, fopts = f_str.split(':', 1) + filters.append((fname, _parse_options([fopts]))) + else: + filters.append((f_str, {})) + return filters + + +def _print_help(what, name): + try: + if what == 'lexer': + cls = get_lexer_by_name(name) + print(f"Help on the {cls.name} lexer:") + print(dedent(cls.__doc__)) + elif what == 'formatter': + cls = find_formatter_class(name) + print(f"Help on the {cls.name} formatter:") + print(dedent(cls.__doc__)) + elif what == 'filter': + cls = find_filter_class(name) + print(f"Help on the {name} filter:") + print(dedent(cls.__doc__)) + return 0 + except (AttributeError, ValueError): + print(f"{what} not found!", file=sys.stderr) + return 1 + + +def _print_list(what): + if what == 'lexer': + print() + print("Lexers:") + print("~~~~~~~") + + info = [] + for fullname, names, exts, _ in get_all_lexers(): + tup = (', '.join(names)+':', fullname, + exts and '(filenames ' + ', '.join(exts) + ')' or '') + info.append(tup) + info.sort() + for i in info: + print(('* {}\n {} {}').format(*i)) + + elif what == 'formatter': + print() + print("Formatters:") + print("~~~~~~~~~~~") + + info = [] + for cls in get_all_formatters(): + doc = docstring_headline(cls) + tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and + '(filenames ' + ', '.join(cls.filenames) + ')' or '') + info.append(tup) + info.sort() + for i in info: + print(('* {}\n {} {}').format(*i)) + + elif what == 'filter': + print() + print("Filters:") + print("~~~~~~~~") + + for name in get_all_filters(): + cls = find_filter_class(name) + print("* " + name + ':') + print(f" {docstring_headline(cls)}") + + elif what == 'style': + print() + print("Styles:") + print("~~~~~~~") + + for name in get_all_styles(): + cls = get_style_by_name(name) + print("* " + name + ':') + print(f" {docstring_headline(cls)}") + + +def _print_list_as_json(requested_items): + import json + result = {} + if 'lexer' in requested_items: + info = {} + for fullname, names, filenames, mimetypes in get_all_lexers(): + info[fullname] = { + 'aliases': names, + 'filenames': filenames, + 'mimetypes': mimetypes + } + result['lexers'] = info + + if 'formatter' in requested_items: + info = {} + for cls in get_all_formatters(): + doc = docstring_headline(cls) + info[cls.name] = { + 'aliases': cls.aliases, + 'filenames': cls.filenames, + 'doc': doc + } + result['formatters'] = info + + if 'filter' in requested_items: + info = {} + for name in get_all_filters(): + cls = find_filter_class(name) + info[name] = { + 'doc': docstring_headline(cls) + } + result['filters'] = info + + if 'style' in requested_items: + info = {} + for name in get_all_styles(): + cls = get_style_by_name(name) + info[name] = { + 'doc': docstring_headline(cls) + } + result['styles'] = info + + json.dump(result, sys.stdout) + +def main_inner(parser, argns): + if argns.help: + parser.print_help() + return 0 + + if argns.V: + print(f'Pygments version {__version__}, (c) 2006-2024 by Georg Brandl, Matthäus ' + 'Chajdas and contributors.') + return 0 + + def is_only_option(opt): + return not any(v for (k, v) in vars(argns).items() if k != opt) + + # handle ``pygmentize -L`` + if argns.L is not None: + arg_set = set() + for k, v in vars(argns).items(): + if v: + arg_set.add(k) + + arg_set.discard('L') + arg_set.discard('json') + + if arg_set: + parser.print_help(sys.stderr) + return 2 + + # print version + if not argns.json: + main(['', '-V']) + allowed_types = {'lexer', 'formatter', 'filter', 'style'} + largs = [arg.rstrip('s') for arg in argns.L] + if any(arg not in allowed_types for arg in largs): + parser.print_help(sys.stderr) + return 0 + if not largs: + largs = allowed_types + if not argns.json: + for arg in largs: + _print_list(arg) + else: + _print_list_as_json(largs) + return 0 + + # handle ``pygmentize -H`` + if argns.H: + if not is_only_option('H'): + parser.print_help(sys.stderr) + return 2 + what, name = argns.H + if what not in ('lexer', 'formatter', 'filter'): + parser.print_help(sys.stderr) + return 2 + return _print_help(what, name) + + # parse -O options + parsed_opts = _parse_options(argns.O or []) + + # parse -P options + for p_opt in argns.P or []: + try: + name, value = p_opt.split('=', 1) + except ValueError: + parsed_opts[p_opt] = True + else: + parsed_opts[name] = value + + # encodings + inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding')) + outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding')) + + # handle ``pygmentize -N`` + if argns.N: + lexer = find_lexer_class_for_filename(argns.N) + if lexer is None: + lexer = TextLexer + + print(lexer.aliases[0]) + return 0 + + # handle ``pygmentize -C`` + if argns.C: + inp = sys.stdin.buffer.read() + try: + lexer = guess_lexer(inp, inencoding=inencoding) + except ClassNotFound: + lexer = TextLexer + + print(lexer.aliases[0]) + return 0 + + # handle ``pygmentize -S`` + S_opt = argns.S + a_opt = argns.a + if S_opt is not None: + f_opt = argns.f + if not f_opt: + parser.print_help(sys.stderr) + return 2 + if argns.l or argns.INPUTFILE: + parser.print_help(sys.stderr) + return 2 + + try: + parsed_opts['style'] = S_opt + fmter = get_formatter_by_name(f_opt, **parsed_opts) + except ClassNotFound as err: + print(err, file=sys.stderr) + return 1 + + print(fmter.get_style_defs(a_opt or '')) + return 0 + + # if no -S is given, -a is not allowed + if argns.a is not None: + parser.print_help(sys.stderr) + return 2 + + # parse -F options + F_opts = _parse_filters(argns.F or []) + + # -x: allow custom (eXternal) lexers and formatters + allow_custom_lexer_formatter = bool(argns.x) + + # select lexer + lexer = None + + # given by name? + lexername = argns.l + if lexername: + # custom lexer, located relative to user's cwd + if allow_custom_lexer_formatter and '.py' in lexername: + try: + filename = None + name = None + if ':' in lexername: + filename, name = lexername.rsplit(':', 1) + + if '.py' in name: + # This can happen on Windows: If the lexername is + # C:\lexer.py -- return to normal load path in that case + name = None + + if filename and name: + lexer = load_lexer_from_file(filename, name, + **parsed_opts) + else: + lexer = load_lexer_from_file(lexername, **parsed_opts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + else: + try: + lexer = get_lexer_by_name(lexername, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 + + # read input code + code = None + + if argns.INPUTFILE: + if argns.s: + print('Error: -s option not usable when input file specified', + file=sys.stderr) + return 2 + + infn = argns.INPUTFILE + try: + with open(infn, 'rb') as infp: + code = infp.read() + except Exception as err: + print('Error: cannot read infile:', err, file=sys.stderr) + return 1 + if not inencoding: + code, inencoding = guess_decode(code) + + # do we have to guess the lexer? + if not lexer: + try: + lexer = get_lexer_for_filename(infn, code, **parsed_opts) + except ClassNotFound as err: + if argns.g: + try: + lexer = guess_lexer(code, **parsed_opts) + except ClassNotFound: + lexer = TextLexer(**parsed_opts) + else: + print('Error:', err, file=sys.stderr) + return 1 + except OptionError as err: + print('Error:', err, file=sys.stderr) + return 1 + + elif not argns.s: # treat stdin as full file (-s support is later) + # read code from terminal, always in binary mode since we want to + # decode ourselves and be tolerant with it + code = sys.stdin.buffer.read() # use .buffer to get a binary stream + if not inencoding: + code, inencoding = guess_decode_from_terminal(code, sys.stdin) + # else the lexer will do the decoding + if not lexer: + try: + lexer = guess_lexer(code, **parsed_opts) + except ClassNotFound: + lexer = TextLexer(**parsed_opts) + + else: # -s option needs a lexer with -l + if not lexer: + print('Error: when using -s a lexer has to be selected with -l', + file=sys.stderr) + return 2 + + # process filters + for fname, fopts in F_opts: + try: + lexer.add_filter(fname, **fopts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + + # select formatter + outfn = argns.o + fmter = argns.f + if fmter: + # custom formatter, located relative to user's cwd + if allow_custom_lexer_formatter and '.py' in fmter: + try: + filename = None + name = None + if ':' in fmter: + # Same logic as above for custom lexer + filename, name = fmter.rsplit(':', 1) + + if '.py' in name: + name = None + + if filename and name: + fmter = load_formatter_from_file(filename, name, + **parsed_opts) + else: + fmter = load_formatter_from_file(fmter, **parsed_opts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + else: + try: + fmter = get_formatter_by_name(fmter, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 + + if outfn: + if not fmter: + try: + fmter = get_formatter_for_filename(outfn, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 + try: + outfile = open(outfn, 'wb') + except Exception as err: + print('Error: cannot open outfile:', err, file=sys.stderr) + return 1 + else: + if not fmter: + if os.environ.get('COLORTERM','') in ('truecolor', '24bit'): + fmter = TerminalTrueColorFormatter(**parsed_opts) + elif '256' in os.environ.get('TERM', ''): + fmter = Terminal256Formatter(**parsed_opts) + else: + fmter = TerminalFormatter(**parsed_opts) + outfile = sys.stdout.buffer + + # determine output encoding if not explicitly selected + if not outencoding: + if outfn: + # output file? use lexer encoding for now (can still be None) + fmter.encoding = inencoding + else: + # else use terminal encoding + fmter.encoding = terminal_encoding(sys.stdout) + + # provide coloring under Windows, if possible + if not outfn and sys.platform in ('win32', 'cygwin') and \ + fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover + # unfortunately colorama doesn't support binary streams on Py3 + outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding) + fmter.encoding = None + try: + import colorama.initialise + except ImportError: + pass + else: + outfile = colorama.initialise.wrap_stream( + outfile, convert=None, strip=None, autoreset=False, wrap=True) + + # When using the LaTeX formatter and the option `escapeinside` is + # specified, we need a special lexer which collects escaped text + # before running the chosen language lexer. + escapeinside = parsed_opts.get('escapeinside', '') + if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter): + left = escapeinside[0] + right = escapeinside[1] + lexer = LatexEmbeddedLexer(left, right, lexer) + + # ... and do it! + if not argns.s: + # process whole input as per normal... + try: + highlight(code, lexer, fmter, outfile) + finally: + if outfn: + outfile.close() + return 0 + else: + # line by line processing of stdin (eg: for 'tail -f')... + try: + while 1: + line = sys.stdin.buffer.readline() + if not line: + break + if not inencoding: + line = guess_decode_from_terminal(line, sys.stdin)[0] + highlight(line, lexer, fmter, outfile) + if hasattr(outfile, 'flush'): + outfile.flush() + return 0 + except KeyboardInterrupt: # pragma: no cover + return 0 + finally: + if outfn: + outfile.close() + + +class HelpFormatter(argparse.HelpFormatter): + def __init__(self, prog, indent_increment=2, max_help_position=16, width=None): + if width is None: + try: + width = shutil.get_terminal_size().columns - 2 + except Exception: + pass + argparse.HelpFormatter.__init__(self, prog, indent_increment, + max_help_position, width) + + +def main(args=sys.argv): + """ + Main command line entry point. + """ + desc = "Highlight an input file and write the result to an output file." + parser = argparse.ArgumentParser(description=desc, add_help=False, + formatter_class=HelpFormatter) + + operation = parser.add_argument_group('Main operation') + lexersel = operation.add_mutually_exclusive_group() + lexersel.add_argument( + '-l', metavar='LEXER', + help='Specify the lexer to use. (Query names with -L.) If not ' + 'given and -g is not present, the lexer is guessed from the filename.') + lexersel.add_argument( + '-g', action='store_true', + help='Guess the lexer from the file contents, or pass through ' + 'as plain text if nothing can be guessed.') + operation.add_argument( + '-F', metavar='FILTER[:options]', action='append', + help='Add a filter to the token stream. (Query names with -L.) ' + 'Filter options are given after a colon if necessary.') + operation.add_argument( + '-f', metavar='FORMATTER', + help='Specify the formatter to use. (Query names with -L.) ' + 'If not given, the formatter is guessed from the output filename, ' + 'and defaults to the terminal formatter if the output is to the ' + 'terminal or an unknown file extension.') + operation.add_argument( + '-O', metavar='OPTION=value[,OPTION=value,...]', action='append', + help='Give options to the lexer and formatter as a comma-separated ' + 'list of key-value pairs. ' + 'Example: `-O bg=light,python=cool`.') + operation.add_argument( + '-P', metavar='OPTION=value', action='append', + help='Give a single option to the lexer and formatter - with this ' + 'you can pass options whose value contains commas and equal signs. ' + 'Example: `-P "heading=Pygments, the Python highlighter"`.') + operation.add_argument( + '-o', metavar='OUTPUTFILE', + help='Where to write the output. Defaults to standard output.') + + operation.add_argument( + 'INPUTFILE', nargs='?', + help='Where to read the input. Defaults to standard input.') + + flags = parser.add_argument_group('Operation flags') + flags.add_argument( + '-v', action='store_true', + help='Print a detailed traceback on unhandled exceptions, which ' + 'is useful for debugging and bug reports.') + flags.add_argument( + '-s', action='store_true', + help='Process lines one at a time until EOF, rather than waiting to ' + 'process the entire file. This only works for stdin, only for lexers ' + 'with no line-spanning constructs, and is intended for streaming ' + 'input such as you get from `tail -f`. ' + 'Example usage: `tail -f sql.log | pygmentize -s -l sql`.') + flags.add_argument( + '-x', action='store_true', + help='Allow custom lexers and formatters to be loaded from a .py file ' + 'relative to the current working directory. For example, ' + '`-l ./customlexer.py -x`. By default, this option expects a file ' + 'with a class named CustomLexer or CustomFormatter; you can also ' + 'specify your own class name with a colon (`-l ./lexer.py:MyLexer`). ' + 'Users should be very careful not to use this option with untrusted ' + 'files, because it will import and run them.') + flags.add_argument('--json', help='Output as JSON. This can ' + 'be only used in conjunction with -L.', + default=False, + action='store_true') + + special_modes_group = parser.add_argument_group( + 'Special modes - do not do any highlighting') + special_modes = special_modes_group.add_mutually_exclusive_group() + special_modes.add_argument( + '-S', metavar='STYLE -f formatter', + help='Print style definitions for STYLE for a formatter ' + 'given with -f. The argument given by -a is formatter ' + 'dependent.') + special_modes.add_argument( + '-L', nargs='*', metavar='WHAT', + help='List lexers, formatters, styles or filters -- ' + 'give additional arguments for the thing(s) you want to list ' + '(e.g. "styles"), or omit them to list everything.') + special_modes.add_argument( + '-N', metavar='FILENAME', + help='Guess and print out a lexer name based solely on the given ' + 'filename. Does not take input or highlight anything. If no specific ' + 'lexer can be determined, "text" is printed.') + special_modes.add_argument( + '-C', action='store_true', + help='Like -N, but print out a lexer name based solely on ' + 'a given content from standard input.') + special_modes.add_argument( + '-H', action='store', nargs=2, metavar=('NAME', 'TYPE'), + help='Print detailed help for the object of type , ' + 'where is one of "lexer", "formatter" or "filter".') + special_modes.add_argument( + '-V', action='store_true', + help='Print the package version.') + special_modes.add_argument( + '-h', '--help', action='store_true', + help='Print this help.') + special_modes_group.add_argument( + '-a', metavar='ARG', + help='Formatter-specific additional argument for the -S (print ' + 'style sheet) mode.') + + argns = parser.parse_args(args[1:]) + + try: + return main_inner(parser, argns) + except BrokenPipeError: + # someone closed our stdout, e.g. by quitting a pager. + return 0 + except Exception: + if argns.v: + print(file=sys.stderr) + print('*' * 65, file=sys.stderr) + print('An unhandled exception occurred while highlighting.', + file=sys.stderr) + print('Please report the whole traceback to the issue tracker at', + file=sys.stderr) + print('.', + file=sys.stderr) + print('*' * 65, file=sys.stderr) + print(file=sys.stderr) + raise + import traceback + info = traceback.format_exception(*sys.exc_info()) + msg = info[-1].strip() + if len(info) >= 3: + # extract relevant file and position info + msg += '\n (f{})'.format(info[-2].split('\n')[0].strip()[1:]) + print(file=sys.stderr) + print('*** Error while highlighting:', file=sys.stderr) + print(msg, file=sys.stderr) + print('*** If this is a bug you want to report, please rerun with -v.', + file=sys.stderr) + return 1 diff --git a/lib/pygments/console.py b/lib/pygments/console.py new file mode 100644 index 0000000..ee1ac27 --- /dev/null +++ b/lib/pygments/console.py @@ -0,0 +1,70 @@ +""" + pygments.console + ~~~~~~~~~~~~~~~~ + + Format colored console output. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +esc = "\x1b[" + +codes = {} +codes[""] = "" +codes["reset"] = esc + "39;49;00m" + +codes["bold"] = esc + "01m" +codes["faint"] = esc + "02m" +codes["standout"] = esc + "03m" +codes["underline"] = esc + "04m" +codes["blink"] = esc + "05m" +codes["overline"] = esc + "06m" + +dark_colors = ["black", "red", "green", "yellow", "blue", + "magenta", "cyan", "gray"] +light_colors = ["brightblack", "brightred", "brightgreen", "brightyellow", "brightblue", + "brightmagenta", "brightcyan", "white"] + +x = 30 +for dark, light in zip(dark_colors, light_colors): + codes[dark] = esc + "%im" % x + codes[light] = esc + "%im" % (60 + x) + x += 1 + +del dark, light, x + +codes["white"] = codes["bold"] + + +def reset_color(): + return codes["reset"] + + +def colorize(color_key, text): + return codes[color_key] + text + codes["reset"] + + +def ansiformat(attr, text): + """ + Format ``text`` with a color and/or some attributes:: + + color normal color + *color* bold color + _color_ underlined color + +color+ blinking color + """ + result = [] + if attr[:1] == attr[-1:] == '+': + result.append(codes['blink']) + attr = attr[1:-1] + if attr[:1] == attr[-1:] == '*': + result.append(codes['bold']) + attr = attr[1:-1] + if attr[:1] == attr[-1:] == '_': + result.append(codes['underline']) + attr = attr[1:-1] + result.append(codes[attr]) + result.append(text) + result.append(codes['reset']) + return ''.join(result) diff --git a/lib/pygments/filter.py b/lib/pygments/filter.py new file mode 100644 index 0000000..5efff43 --- /dev/null +++ b/lib/pygments/filter.py @@ -0,0 +1,70 @@ +""" + pygments.filter + ~~~~~~~~~~~~~~~ + + Module that implements the default filter. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +def apply_filters(stream, filters, lexer=None): + """ + Use this method to apply an iterable of filters to + a stream. If lexer is given it's forwarded to the + filter, otherwise the filter receives `None`. + """ + def _apply(filter_, stream): + yield from filter_.filter(lexer, stream) + for filter_ in filters: + stream = _apply(filter_, stream) + return stream + + +def simplefilter(f): + """ + Decorator that converts a function into a filter:: + + @simplefilter + def lowercase(self, lexer, stream, options): + for ttype, value in stream: + yield ttype, value.lower() + """ + return type(f.__name__, (FunctionFilter,), { + '__module__': getattr(f, '__module__'), + '__doc__': f.__doc__, + 'function': f, + }) + + +class Filter: + """ + Default filter. Subclass this class or use the `simplefilter` + decorator to create own filters. + """ + + def __init__(self, **options): + self.options = options + + def filter(self, lexer, stream): + raise NotImplementedError() + + +class FunctionFilter(Filter): + """ + Abstract class used by `simplefilter` to create simple + function filters on the fly. The `simplefilter` decorator + automatically creates subclasses of this class for + functions passed to it. + """ + function = None + + def __init__(self, **options): + if not hasattr(self, 'function'): + raise TypeError(f'{self.__class__.__name__!r} used without bound function') + Filter.__init__(self, **options) + + def filter(self, lexer, stream): + # pylint: disable=not-callable + yield from self.function(lexer, stream, self.options) diff --git a/lib/pygments/filters/__init__.py b/lib/pygments/filters/__init__.py new file mode 100644 index 0000000..2fed761 --- /dev/null +++ b/lib/pygments/filters/__init__.py @@ -0,0 +1,940 @@ +""" + pygments.filters + ~~~~~~~~~~~~~~~~ + + Module containing filter lookup functions and default + filters. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \ + string_to_tokentype +from pygments.filter import Filter +from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \ + get_choice_opt, ClassNotFound, OptionError +from pygments.plugin import find_plugin_filters + + +def find_filter_class(filtername): + """Lookup a filter by name. Return None if not found.""" + if filtername in FILTERS: + return FILTERS[filtername] + for name, cls in find_plugin_filters(): + if name == filtername: + return cls + return None + + +def get_filter_by_name(filtername, **options): + """Return an instantiated filter. + + Options are passed to the filter initializer if wanted. + Raise a ClassNotFound if not found. + """ + cls = find_filter_class(filtername) + if cls: + return cls(**options) + else: + raise ClassNotFound(f'filter {filtername!r} not found') + + +def get_all_filters(): + """Return a generator of all filter names.""" + yield from FILTERS + for name, _ in find_plugin_filters(): + yield name + + +def _replace_special(ttype, value, regex, specialttype, + replacefunc=lambda x: x): + last = 0 + for match in regex.finditer(value): + start, end = match.start(), match.end() + if start != last: + yield ttype, value[last:start] + yield specialttype, replacefunc(value[start:end]) + last = end + if last != len(value): + yield ttype, value[last:] + + +class CodeTagFilter(Filter): + """Highlight special code tags in comments and docstrings. + + Options accepted: + + `codetags` : list of strings + A list of strings that are flagged as code tags. The default is to + highlight ``XXX``, ``TODO``, ``FIXME``, ``BUG`` and ``NOTE``. + + .. versionchanged:: 2.13 + Now recognizes ``FIXME`` by default. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + tags = get_list_opt(options, 'codetags', + ['XXX', 'TODO', 'FIXME', 'BUG', 'NOTE']) + self.tag_re = re.compile(r'\b({})\b'.format('|'.join([ + re.escape(tag) for tag in tags if tag + ]))) + + def filter(self, lexer, stream): + regex = self.tag_re + for ttype, value in stream: + if ttype in String.Doc or \ + ttype in Comment and \ + ttype not in Comment.Preproc: + yield from _replace_special(ttype, value, regex, Comment.Special) + else: + yield ttype, value + + +class SymbolFilter(Filter): + """Convert mathematical symbols such as \\ in Isabelle + or \\longrightarrow in LaTeX into Unicode characters. + + This is mostly useful for HTML or console output when you want to + approximate the source rendering you'd see in an IDE. + + Options accepted: + + `lang` : string + The symbol language. Must be one of ``'isabelle'`` or + ``'latex'``. The default is ``'isabelle'``. + """ + + latex_symbols = { + '\\alpha' : '\U000003b1', + '\\beta' : '\U000003b2', + '\\gamma' : '\U000003b3', + '\\delta' : '\U000003b4', + '\\varepsilon' : '\U000003b5', + '\\zeta' : '\U000003b6', + '\\eta' : '\U000003b7', + '\\vartheta' : '\U000003b8', + '\\iota' : '\U000003b9', + '\\kappa' : '\U000003ba', + '\\lambda' : '\U000003bb', + '\\mu' : '\U000003bc', + '\\nu' : '\U000003bd', + '\\xi' : '\U000003be', + '\\pi' : '\U000003c0', + '\\varrho' : '\U000003c1', + '\\sigma' : '\U000003c3', + '\\tau' : '\U000003c4', + '\\upsilon' : '\U000003c5', + '\\varphi' : '\U000003c6', + '\\chi' : '\U000003c7', + '\\psi' : '\U000003c8', + '\\omega' : '\U000003c9', + '\\Gamma' : '\U00000393', + '\\Delta' : '\U00000394', + '\\Theta' : '\U00000398', + '\\Lambda' : '\U0000039b', + '\\Xi' : '\U0000039e', + '\\Pi' : '\U000003a0', + '\\Sigma' : '\U000003a3', + '\\Upsilon' : '\U000003a5', + '\\Phi' : '\U000003a6', + '\\Psi' : '\U000003a8', + '\\Omega' : '\U000003a9', + '\\leftarrow' : '\U00002190', + '\\longleftarrow' : '\U000027f5', + '\\rightarrow' : '\U00002192', + '\\longrightarrow' : '\U000027f6', + '\\Leftarrow' : '\U000021d0', + '\\Longleftarrow' : '\U000027f8', + '\\Rightarrow' : '\U000021d2', + '\\Longrightarrow' : '\U000027f9', + '\\leftrightarrow' : '\U00002194', + '\\longleftrightarrow' : '\U000027f7', + '\\Leftrightarrow' : '\U000021d4', + '\\Longleftrightarrow' : '\U000027fa', + '\\mapsto' : '\U000021a6', + '\\longmapsto' : '\U000027fc', + '\\relbar' : '\U00002500', + '\\Relbar' : '\U00002550', + '\\hookleftarrow' : '\U000021a9', + '\\hookrightarrow' : '\U000021aa', + '\\leftharpoondown' : '\U000021bd', + '\\rightharpoondown' : '\U000021c1', + '\\leftharpoonup' : '\U000021bc', + '\\rightharpoonup' : '\U000021c0', + '\\rightleftharpoons' : '\U000021cc', + '\\leadsto' : '\U0000219d', + '\\downharpoonleft' : '\U000021c3', + '\\downharpoonright' : '\U000021c2', + '\\upharpoonleft' : '\U000021bf', + '\\upharpoonright' : '\U000021be', + '\\restriction' : '\U000021be', + '\\uparrow' : '\U00002191', + '\\Uparrow' : '\U000021d1', + '\\downarrow' : '\U00002193', + '\\Downarrow' : '\U000021d3', + '\\updownarrow' : '\U00002195', + '\\Updownarrow' : '\U000021d5', + '\\langle' : '\U000027e8', + '\\rangle' : '\U000027e9', + '\\lceil' : '\U00002308', + '\\rceil' : '\U00002309', + '\\lfloor' : '\U0000230a', + '\\rfloor' : '\U0000230b', + '\\flqq' : '\U000000ab', + '\\frqq' : '\U000000bb', + '\\bot' : '\U000022a5', + '\\top' : '\U000022a4', + '\\wedge' : '\U00002227', + '\\bigwedge' : '\U000022c0', + '\\vee' : '\U00002228', + '\\bigvee' : '\U000022c1', + '\\forall' : '\U00002200', + '\\exists' : '\U00002203', + '\\nexists' : '\U00002204', + '\\neg' : '\U000000ac', + '\\Box' : '\U000025a1', + '\\Diamond' : '\U000025c7', + '\\vdash' : '\U000022a2', + '\\models' : '\U000022a8', + '\\dashv' : '\U000022a3', + '\\surd' : '\U0000221a', + '\\le' : '\U00002264', + '\\ge' : '\U00002265', + '\\ll' : '\U0000226a', + '\\gg' : '\U0000226b', + '\\lesssim' : '\U00002272', + '\\gtrsim' : '\U00002273', + '\\lessapprox' : '\U00002a85', + '\\gtrapprox' : '\U00002a86', + '\\in' : '\U00002208', + '\\notin' : '\U00002209', + '\\subset' : '\U00002282', + '\\supset' : '\U00002283', + '\\subseteq' : '\U00002286', + '\\supseteq' : '\U00002287', + '\\sqsubset' : '\U0000228f', + '\\sqsupset' : '\U00002290', + '\\sqsubseteq' : '\U00002291', + '\\sqsupseteq' : '\U00002292', + '\\cap' : '\U00002229', + '\\bigcap' : '\U000022c2', + '\\cup' : '\U0000222a', + '\\bigcup' : '\U000022c3', + '\\sqcup' : '\U00002294', + '\\bigsqcup' : '\U00002a06', + '\\sqcap' : '\U00002293', + '\\Bigsqcap' : '\U00002a05', + '\\setminus' : '\U00002216', + '\\propto' : '\U0000221d', + '\\uplus' : '\U0000228e', + '\\bigplus' : '\U00002a04', + '\\sim' : '\U0000223c', + '\\doteq' : '\U00002250', + '\\simeq' : '\U00002243', + '\\approx' : '\U00002248', + '\\asymp' : '\U0000224d', + '\\cong' : '\U00002245', + '\\equiv' : '\U00002261', + '\\Join' : '\U000022c8', + '\\bowtie' : '\U00002a1d', + '\\prec' : '\U0000227a', + '\\succ' : '\U0000227b', + '\\preceq' : '\U0000227c', + '\\succeq' : '\U0000227d', + '\\parallel' : '\U00002225', + '\\mid' : '\U000000a6', + '\\pm' : '\U000000b1', + '\\mp' : '\U00002213', + '\\times' : '\U000000d7', + '\\div' : '\U000000f7', + '\\cdot' : '\U000022c5', + '\\star' : '\U000022c6', + '\\circ' : '\U00002218', + '\\dagger' : '\U00002020', + '\\ddagger' : '\U00002021', + '\\lhd' : '\U000022b2', + '\\rhd' : '\U000022b3', + '\\unlhd' : '\U000022b4', + '\\unrhd' : '\U000022b5', + '\\triangleleft' : '\U000025c3', + '\\triangleright' : '\U000025b9', + '\\triangle' : '\U000025b3', + '\\triangleq' : '\U0000225c', + '\\oplus' : '\U00002295', + '\\bigoplus' : '\U00002a01', + '\\otimes' : '\U00002297', + '\\bigotimes' : '\U00002a02', + '\\odot' : '\U00002299', + '\\bigodot' : '\U00002a00', + '\\ominus' : '\U00002296', + '\\oslash' : '\U00002298', + '\\dots' : '\U00002026', + '\\cdots' : '\U000022ef', + '\\sum' : '\U00002211', + '\\prod' : '\U0000220f', + '\\coprod' : '\U00002210', + '\\infty' : '\U0000221e', + '\\int' : '\U0000222b', + '\\oint' : '\U0000222e', + '\\clubsuit' : '\U00002663', + '\\diamondsuit' : '\U00002662', + '\\heartsuit' : '\U00002661', + '\\spadesuit' : '\U00002660', + '\\aleph' : '\U00002135', + '\\emptyset' : '\U00002205', + '\\nabla' : '\U00002207', + '\\partial' : '\U00002202', + '\\flat' : '\U0000266d', + '\\natural' : '\U0000266e', + '\\sharp' : '\U0000266f', + '\\angle' : '\U00002220', + '\\copyright' : '\U000000a9', + '\\textregistered' : '\U000000ae', + '\\textonequarter' : '\U000000bc', + '\\textonehalf' : '\U000000bd', + '\\textthreequarters' : '\U000000be', + '\\textordfeminine' : '\U000000aa', + '\\textordmasculine' : '\U000000ba', + '\\euro' : '\U000020ac', + '\\pounds' : '\U000000a3', + '\\yen' : '\U000000a5', + '\\textcent' : '\U000000a2', + '\\textcurrency' : '\U000000a4', + '\\textdegree' : '\U000000b0', + } + + isabelle_symbols = { + '\\' : '\U0001d7ec', + '\\' : '\U0001d7ed', + '\\' : '\U0001d7ee', + '\\' : '\U0001d7ef', + '\\' : '\U0001d7f0', + '\\' : '\U0001d7f1', + '\\' : '\U0001d7f2', + '\\' : '\U0001d7f3', + '\\' : '\U0001d7f4', + '\\' : '\U0001d7f5', + '\\' : '\U0001d49c', + '\\' : '\U0000212c', + '\\' : '\U0001d49e', + '\\' : '\U0001d49f', + '\\' : '\U00002130', + '\\' : '\U00002131', + '\\' : '\U0001d4a2', + '\\' : '\U0000210b', + '\\' : '\U00002110', + '\\' : '\U0001d4a5', + '\\' : '\U0001d4a6', + '\\' : '\U00002112', + '\\' : '\U00002133', + '\\' : '\U0001d4a9', + '\\' : '\U0001d4aa', + '\\

' : '\U0001d5c9', + '\\' : '\U0001d5ca', + '\\' : '\U0001d5cb', + '\\' : '\U0001d5cc', + '\\' : '\U0001d5cd', + '\\' : '\U0001d5ce', + '\\' : '\U0001d5cf', + '\\' : '\U0001d5d0', + '\\' : '\U0001d5d1', + '\\' : '\U0001d5d2', + '\\' : '\U0001d5d3', + '\\' : '\U0001d504', + '\\' : '\U0001d505', + '\\' : '\U0000212d', + '\\

' : '\U0001d507', + '\\' : '\U0001d508', + '\\' : '\U0001d509', + '\\' : '\U0001d50a', + '\\' : '\U0000210c', + '\\' : '\U00002111', + '\\' : '\U0001d50d', + '\\' : '\U0001d50e', + '\\' : '\U0001d50f', + '\\' : '\U0001d510', + '\\' : '\U0001d511', + '\\' : '\U0001d512', + '\\' : '\U0001d513', + '\\' : '\U0001d514', + '\\' : '\U0000211c', + '\\' : '\U0001d516', + '\\' : '\U0001d517', + '\\' : '\U0001d518', + '\\' : '\U0001d519', + '\\' : '\U0001d51a', + '\\' : '\U0001d51b', + '\\' : '\U0001d51c', + '\\' : '\U00002128', + '\\' : '\U0001d51e', + '\\' : '\U0001d51f', + '\\' : '\U0001d520', + '\\
' : '\U0001d521', + '\\' : '\U0001d522', + '\\' : '\U0001d523', + '\\' : '\U0001d524', + '\\' : '\U0001d525', + '\\' : '\U0001d526', + '\\' : '\U0001d527', + '\\' : '\U0001d528', + '\\' : '\U0001d529', + '\\' : '\U0001d52a', + '\\' : '\U0001d52b', + '\\' : '\U0001d52c', + '\\' : '\U0001d52d', + '\\' : '\U0001d52e', + '\\' : '\U0001d52f', + '\\' : '\U0001d530', + '\\' : '\U0001d531', + '\\' : '\U0001d532', + '\\' : '\U0001d533', + '\\' : '\U0001d534', + '\\' : '\U0001d535', + '\\' : '\U0001d536', + '\\' : '\U0001d537', + '\\' : '\U000003b1', + '\\' : '\U000003b2', + '\\' : '\U000003b3', + '\\' : '\U000003b4', + '\\' : '\U000003b5', + '\\' : '\U000003b6', + '\\' : '\U000003b7', + '\\' : '\U000003b8', + '\\' : '\U000003b9', + '\\' : '\U000003ba', + '\\' : '\U000003bb', + '\\' : '\U000003bc', + '\\' : '\U000003bd', + '\\' : '\U000003be', + '\\' : '\U000003c0', + '\\' : '\U000003c1', + '\\' : '\U000003c3', + '\\' : '\U000003c4', + '\\' : '\U000003c5', + '\\' : '\U000003c6', + '\\' : '\U000003c7', + '\\' : '\U000003c8', + '\\' : '\U000003c9', + '\\' : '\U00000393', + '\\' : '\U00000394', + '\\' : '\U00000398', + '\\' : '\U0000039b', + '\\' : '\U0000039e', + '\\' : '\U000003a0', + '\\' : '\U000003a3', + '\\' : '\U000003a5', + '\\' : '\U000003a6', + '\\' : '\U000003a8', + '\\' : '\U000003a9', + '\\' : '\U0001d539', + '\\' : '\U00002102', + '\\' : '\U00002115', + '\\' : '\U0000211a', + '\\' : '\U0000211d', + '\\' : '\U00002124', + '\\' : '\U00002190', + '\\' : '\U000027f5', + '\\' : '\U00002192', + '\\' : '\U000027f6', + '\\' : '\U000021d0', + '\\' : '\U000027f8', + '\\' : '\U000021d2', + '\\' : '\U000027f9', + '\\' : '\U00002194', + '\\' : '\U000027f7', + '\\' : '\U000021d4', + '\\' : '\U000027fa', + '\\' : '\U000021a6', + '\\' : '\U000027fc', + '\\' : '\U00002500', + '\\' : '\U00002550', + '\\' : '\U000021a9', + '\\' : '\U000021aa', + '\\' : '\U000021bd', + '\\' : '\U000021c1', + '\\' : '\U000021bc', + '\\' : '\U000021c0', + '\\' : '\U000021cc', + '\\' : '\U0000219d', + '\\' : '\U000021c3', + '\\' : '\U000021c2', + '\\' : '\U000021bf', + '\\' : '\U000021be', + '\\' : '\U000021be', + '\\' : '\U00002237', + '\\' : '\U00002191', + '\\' : '\U000021d1', + '\\' : '\U00002193', + '\\' : '\U000021d3', + '\\' : '\U00002195', + '\\' : '\U000021d5', + '\\' : '\U000027e8', + '\\' : '\U000027e9', + '\\' : '\U00002308', + '\\' : '\U00002309', + '\\' : '\U0000230a', + '\\' : '\U0000230b', + '\\' : '\U00002987', + '\\' : '\U00002988', + '\\' : '\U000027e6', + '\\' : '\U000027e7', + '\\' : '\U00002983', + '\\' : '\U00002984', + '\\' : '\U000000ab', + '\\' : '\U000000bb', + '\\' : '\U000022a5', + '\\' : '\U000022a4', + '\\' : '\U00002227', + '\\' : '\U000022c0', + '\\' : '\U00002228', + '\\' : '\U000022c1', + '\\' : '\U00002200', + '\\' : '\U00002203', + '\\' : '\U00002204', + '\\' : '\U000000ac', + '\\' : '\U000025a1', + '\\' : '\U000025c7', + '\\' : '\U000022a2', + '\\' : '\U000022a8', + '\\' : '\U000022a9', + '\\' : '\U000022ab', + '\\' : '\U000022a3', + '\\' : '\U0000221a', + '\\' : '\U00002264', + '\\' : '\U00002265', + '\\' : '\U0000226a', + '\\' : '\U0000226b', + '\\' : '\U00002272', + '\\' : '\U00002273', + '\\' : '\U00002a85', + '\\' : '\U00002a86', + '\\' : '\U00002208', + '\\' : '\U00002209', + '\\' : '\U00002282', + '\\' : '\U00002283', + '\\' : '\U00002286', + '\\' : '\U00002287', + '\\' : '\U0000228f', + '\\' : '\U00002290', + '\\' : '\U00002291', + '\\' : '\U00002292', + '\\' : '\U00002229', + '\\' : '\U000022c2', + '\\' : '\U0000222a', + '\\' : '\U000022c3', + '\\' : '\U00002294', + '\\' : '\U00002a06', + '\\' : '\U00002293', + '\\' : '\U00002a05', + '\\' : '\U00002216', + '\\' : '\U0000221d', + '\\' : '\U0000228e', + '\\' : '\U00002a04', + '\\' : '\U00002260', + '\\' : '\U0000223c', + '\\' : '\U00002250', + '\\' : '\U00002243', + '\\' : '\U00002248', + '\\' : '\U0000224d', + '\\' : '\U00002245', + '\\' : '\U00002323', + '\\' : '\U00002261', + '\\' : '\U00002322', + '\\' : '\U000022c8', + '\\' : '\U00002a1d', + '\\' : '\U0000227a', + '\\' : '\U0000227b', + '\\' : '\U0000227c', + '\\' : '\U0000227d', + '\\' : '\U00002225', + '\\' : '\U000000a6', + '\\' : '\U000000b1', + '\\' : '\U00002213', + '\\' : '\U000000d7', + '\\
' : '\U000000f7', + '\\' : '\U000022c5', + '\\' : '\U000022c6', + '\\' : '\U00002219', + '\\' : '\U00002218', + '\\' : '\U00002020', + '\\' : '\U00002021', + '\\' : '\U000022b2', + '\\' : '\U000022b3', + '\\' : '\U000022b4', + '\\' : '\U000022b5', + '\\' : '\U000025c3', + '\\' : '\U000025b9', + '\\' : '\U000025b3', + '\\' : '\U0000225c', + '\\' : '\U00002295', + '\\' : '\U00002a01', + '\\' : '\U00002297', + '\\' : '\U00002a02', + '\\' : '\U00002299', + '\\' : '\U00002a00', + '\\' : '\U00002296', + '\\' : '\U00002298', + '\\' : '\U00002026', + '\\' : '\U000022ef', + '\\' : '\U00002211', + '\\' : '\U0000220f', + '\\' : '\U00002210', + '\\' : '\U0000221e', + '\\' : '\U0000222b', + '\\' : '\U0000222e', + '\\' : '\U00002663', + '\\' : '\U00002662', + '\\' : '\U00002661', + '\\' : '\U00002660', + '\\' : '\U00002135', + '\\' : '\U00002205', + '\\' : '\U00002207', + '\\' : '\U00002202', + '\\' : '\U0000266d', + '\\' : '\U0000266e', + '\\' : '\U0000266f', + '\\' : '\U00002220', + '\\' : '\U000000a9', + '\\' : '\U000000ae', + '\\' : '\U000000ad', + '\\' : '\U000000af', + '\\' : '\U000000bc', + '\\' : '\U000000bd', + '\\' : '\U000000be', + '\\' : '\U000000aa', + '\\' : '\U000000ba', + '\\
' : '\U000000a7', + '\\' : '\U000000b6', + '\\' : '\U000000a1', + '\\' : '\U000000bf', + '\\' : '\U000020ac', + '\\' : '\U000000a3', + '\\' : '\U000000a5', + '\\' : '\U000000a2', + '\\' : '\U000000a4', + '\\' : '\U000000b0', + '\\' : '\U00002a3f', + '\\' : '\U00002127', + '\\' : '\U000025ca', + '\\' : '\U00002118', + '\\' : '\U00002240', + '\\' : '\U000022c4', + '\\' : '\U000000b4', + '\\' : '\U00000131', + '\\' : '\U000000a8', + '\\' : '\U000000b8', + '\\' : '\U000002dd', + '\\' : '\U000003f5', + '\\' : '\U000023ce', + '\\' : '\U00002039', + '\\' : '\U0000203a', + '\\' : '\U00002302', + '\\<^sub>' : '\U000021e9', + '\\<^sup>' : '\U000021e7', + '\\<^bold>' : '\U00002759', + '\\<^bsub>' : '\U000021d8', + '\\<^esub>' : '\U000021d9', + '\\<^bsup>' : '\U000021d7', + '\\<^esup>' : '\U000021d6', + } + + lang_map = {'isabelle' : isabelle_symbols, 'latex' : latex_symbols} + + def __init__(self, **options): + Filter.__init__(self, **options) + lang = get_choice_opt(options, 'lang', + ['isabelle', 'latex'], 'isabelle') + self.symbols = self.lang_map[lang] + + def filter(self, lexer, stream): + for ttype, value in stream: + if value in self.symbols: + yield ttype, self.symbols[value] + else: + yield ttype, value + + +class KeywordCaseFilter(Filter): + """Convert keywords to lowercase or uppercase or capitalize them, which + means first letter uppercase, rest lowercase. + + This can be useful e.g. if you highlight Pascal code and want to adapt the + code to your styleguide. + + Options accepted: + + `case` : string + The casing to convert keywords to. Must be one of ``'lower'``, + ``'upper'`` or ``'capitalize'``. The default is ``'lower'``. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + case = get_choice_opt(options, 'case', + ['lower', 'upper', 'capitalize'], 'lower') + self.convert = getattr(str, case) + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype in Keyword: + yield ttype, self.convert(value) + else: + yield ttype, value + + +class NameHighlightFilter(Filter): + """Highlight a normal Name (and Name.*) token with a different token type. + + Example:: + + filter = NameHighlightFilter( + names=['foo', 'bar', 'baz'], + tokentype=Name.Function, + ) + + This would highlight the names "foo", "bar" and "baz" + as functions. `Name.Function` is the default token type. + + Options accepted: + + `names` : list of strings + A list of names that should be given the different token type. + There is no default. + `tokentype` : TokenType or string + A token type or a string containing a token type name that is + used for highlighting the strings in `names`. The default is + `Name.Function`. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + self.names = set(get_list_opt(options, 'names', [])) + tokentype = options.get('tokentype') + if tokentype: + self.tokentype = string_to_tokentype(tokentype) + else: + self.tokentype = Name.Function + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype in Name and value in self.names: + yield self.tokentype, value + else: + yield ttype, value + + +class ErrorToken(Exception): + pass + + +class RaiseOnErrorTokenFilter(Filter): + """Raise an exception when the lexer generates an error token. + + Options accepted: + + `excclass` : Exception class + The exception class to raise. + The default is `pygments.filters.ErrorToken`. + + .. versionadded:: 0.8 + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + self.exception = options.get('excclass', ErrorToken) + try: + # issubclass() will raise TypeError if first argument is not a class + if not issubclass(self.exception, Exception): + raise TypeError + except TypeError: + raise OptionError('excclass option is not an exception class') + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype is Error: + raise self.exception(value) + yield ttype, value + + +class VisibleWhitespaceFilter(Filter): + """Convert tabs, newlines and/or spaces to visible characters. + + Options accepted: + + `spaces` : string or bool + If this is a one-character string, spaces will be replaces by this string. + If it is another true value, spaces will be replaced by ``·`` (unicode + MIDDLE DOT). If it is a false value, spaces will not be replaced. The + default is ``False``. + `tabs` : string or bool + The same as for `spaces`, but the default replacement character is ``»`` + (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value + is ``False``. Note: this will not work if the `tabsize` option for the + lexer is nonzero, as tabs will already have been expanded then. + `tabsize` : int + If tabs are to be replaced by this filter (see the `tabs` option), this + is the total number of characters that a tab should be expanded to. + The default is ``8``. + `newlines` : string or bool + The same as for `spaces`, but the default replacement character is ``¶`` + (unicode PILCROW SIGN). The default value is ``False``. + `wstokentype` : bool + If true, give whitespace the special `Whitespace` token type. This allows + styling the visible whitespace differently (e.g. greyed out), but it can + disrupt background colors. The default is ``True``. + + .. versionadded:: 0.8 + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + for name, default in [('spaces', '·'), + ('tabs', '»'), + ('newlines', '¶')]: + opt = options.get(name, False) + if isinstance(opt, str) and len(opt) == 1: + setattr(self, name, opt) + else: + setattr(self, name, (opt and default or '')) + tabsize = get_int_opt(options, 'tabsize', 8) + if self.tabs: + self.tabs += ' ' * (tabsize - 1) + if self.newlines: + self.newlines += '\n' + self.wstt = get_bool_opt(options, 'wstokentype', True) + + def filter(self, lexer, stream): + if self.wstt: + spaces = self.spaces or ' ' + tabs = self.tabs or '\t' + newlines = self.newlines or '\n' + regex = re.compile(r'\s') + + def replacefunc(wschar): + if wschar == ' ': + return spaces + elif wschar == '\t': + return tabs + elif wschar == '\n': + return newlines + return wschar + + for ttype, value in stream: + yield from _replace_special(ttype, value, regex, Whitespace, + replacefunc) + else: + spaces, tabs, newlines = self.spaces, self.tabs, self.newlines + # simpler processing + for ttype, value in stream: + if spaces: + value = value.replace(' ', spaces) + if tabs: + value = value.replace('\t', tabs) + if newlines: + value = value.replace('\n', newlines) + yield ttype, value + + +class GobbleFilter(Filter): + """Gobbles source code lines (eats initial characters). + + This filter drops the first ``n`` characters off every line of code. This + may be useful when the source code fed to the lexer is indented by a fixed + amount of space that isn't desired in the output. + + Options accepted: + + `n` : int + The number of characters to gobble. + + .. versionadded:: 1.2 + """ + def __init__(self, **options): + Filter.__init__(self, **options) + self.n = get_int_opt(options, 'n', 0) + + def gobble(self, value, left): + if left < len(value): + return value[left:], 0 + else: + return '', left - len(value) + + def filter(self, lexer, stream): + n = self.n + left = n # How many characters left to gobble. + for ttype, value in stream: + # Remove ``left`` tokens from first line, ``n`` from all others. + parts = value.split('\n') + (parts[0], left) = self.gobble(parts[0], left) + for i in range(1, len(parts)): + (parts[i], left) = self.gobble(parts[i], n) + value = '\n'.join(parts) + + if value != '': + yield ttype, value + + +class TokenMergeFilter(Filter): + """Merges consecutive tokens with the same token type in the output + stream of a lexer. + + .. versionadded:: 1.2 + """ + def __init__(self, **options): + Filter.__init__(self, **options) + + def filter(self, lexer, stream): + current_type = None + current_value = None + for ttype, value in stream: + if ttype is current_type: + current_value += value + else: + if current_type is not None: + yield current_type, current_value + current_type = ttype + current_value = value + if current_type is not None: + yield current_type, current_value + + +FILTERS = { + 'codetagify': CodeTagFilter, + 'keywordcase': KeywordCaseFilter, + 'highlight': NameHighlightFilter, + 'raiseonerror': RaiseOnErrorTokenFilter, + 'whitespace': VisibleWhitespaceFilter, + 'gobble': GobbleFilter, + 'tokenmerge': TokenMergeFilter, + 'symbols': SymbolFilter, +} diff --git a/lib/pygments/filters/__pycache__/__init__.cpython-314.pyc b/lib/pygments/filters/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..3e33b52 Binary files /dev/null and b/lib/pygments/filters/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/pygments/formatter.py b/lib/pygments/formatter.py new file mode 100644 index 0000000..a20d303 --- /dev/null +++ b/lib/pygments/formatter.py @@ -0,0 +1,129 @@ +""" + pygments.formatter + ~~~~~~~~~~~~~~~~~~ + + Base formatter class. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import codecs + +from pygments.util import get_bool_opt +from pygments.styles import get_style_by_name + +__all__ = ['Formatter'] + + +def _lookup_style(style): + if isinstance(style, str): + return get_style_by_name(style) + return style + + +class Formatter: + """ + Converts a token stream to text. + + Formatters should have attributes to help selecting them. These + are similar to the corresponding :class:`~pygments.lexer.Lexer` + attributes. + + .. autoattribute:: name + :no-value: + + .. autoattribute:: aliases + :no-value: + + .. autoattribute:: filenames + :no-value: + + You can pass options as keyword arguments to the constructor. + All formatters accept these basic options: + + ``style`` + The style to use, can be a string or a Style subclass + (default: "default"). Not used by e.g. the + TerminalFormatter. + ``full`` + Tells the formatter to output a "full" document, i.e. + a complete self-contained document. This doesn't have + any effect for some formatters (default: false). + ``title`` + If ``full`` is true, the title that should be used to + caption the document (default: ''). + ``encoding`` + If given, must be an encoding name. This will be used to + convert the Unicode token strings to byte strings in the + output. If it is "" or None, Unicode strings will be written + to the output file, which most file-like objects do not + support (default: None). + ``outencoding`` + Overrides ``encoding`` if given. + + """ + + #: Full name for the formatter, in human-readable form. + name = None + + #: A list of short, unique identifiers that can be used to lookup + #: the formatter from a list, e.g. using :func:`.get_formatter_by_name()`. + aliases = [] + + #: A list of fnmatch patterns that match filenames for which this + #: formatter can produce output. The patterns in this list should be unique + #: among all formatters. + filenames = [] + + #: If True, this formatter outputs Unicode strings when no encoding + #: option is given. + unicodeoutput = True + + def __init__(self, **options): + """ + As with lexers, this constructor takes arbitrary optional arguments, + and if you override it, you should first process your own options, then + call the base class implementation. + """ + self.style = _lookup_style(options.get('style', 'default')) + self.full = get_bool_opt(options, 'full', False) + self.title = options.get('title', '') + self.encoding = options.get('encoding', None) or None + if self.encoding in ('guess', 'chardet'): + # can happen for e.g. pygmentize -O encoding=guess + self.encoding = 'utf-8' + self.encoding = options.get('outencoding') or self.encoding + self.options = options + + def get_style_defs(self, arg=''): + """ + This method must return statements or declarations suitable to define + the current style for subsequent highlighted text (e.g. CSS classes + in the `HTMLFormatter`). + + The optional argument `arg` can be used to modify the generation and + is formatter dependent (it is standardized because it can be given on + the command line). + + This method is called by the ``-S`` :doc:`command-line option `, + the `arg` is then given by the ``-a`` option. + """ + return '' + + def format(self, tokensource, outfile): + """ + This method must format the tokens from the `tokensource` iterable and + write the formatted version to the file object `outfile`. + + Formatter options can control how exactly the tokens are converted. + """ + if self.encoding: + # wrap the outfile in a StreamWriter + outfile = codecs.lookup(self.encoding)[3](outfile) + return self.format_unencoded(tokensource, outfile) + + # Allow writing Formatter[str] or Formatter[bytes]. That's equivalent to + # Formatter. This helps when using third-party type stubs from typeshed. + def __class_getitem__(cls, name): + return cls diff --git a/lib/pygments/formatters/__init__.py b/lib/pygments/formatters/__init__.py new file mode 100644 index 0000000..b24931c --- /dev/null +++ b/lib/pygments/formatters/__init__.py @@ -0,0 +1,157 @@ +""" + pygments.formatters + ~~~~~~~~~~~~~~~~~~~ + + Pygments formatters. + + :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +import sys +import types +import fnmatch +from os.path import basename + +from pygments.formatters._mapping import FORMATTERS +from pygments.plugin import find_plugin_formatters +from pygments.util import ClassNotFound + +__all__ = ['get_formatter_by_name', 'get_formatter_for_filename', + 'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS) + +_formatter_cache = {} # classes by name +_pattern_cache = {} + + +def _fn_matches(fn, glob): + """Return whether the supplied file name fn matches pattern filename.""" + if glob not in _pattern_cache: + pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob)) + return pattern.match(fn) + return _pattern_cache[glob].match(fn) + + +def _load_formatters(module_name): + """Load a formatter (and all others in the module too).""" + mod = __import__(module_name, None, None, ['__all__']) + for formatter_name in mod.__all__: + cls = getattr(mod, formatter_name) + _formatter_cache[cls.name] = cls + + +def get_all_formatters(): + """Return a generator for all formatter classes.""" + # NB: this returns formatter classes, not info like get_all_lexers(). + for info in FORMATTERS.values(): + if info[1] not in _formatter_cache: + _load_formatters(info[0]) + yield _formatter_cache[info[1]] + for _, formatter in find_plugin_formatters(): + yield formatter + + +def find_formatter_class(alias): + """Lookup a formatter by alias. + + Returns None if not found. + """ + for module_name, name, aliases, _, _ in FORMATTERS.values(): + if alias in aliases: + if name not in _formatter_cache: + _load_formatters(module_name) + return _formatter_cache[name] + for _, cls in find_plugin_formatters(): + if alias in cls.aliases: + return cls + + +def get_formatter_by_name(_alias, **options): + """ + Return an instance of a :class:`.Formatter` subclass that has `alias` in its + aliases list. The formatter is given the `options` at its instantiation. + + Will raise :exc:`pygments.util.ClassNotFound` if no formatter with that + alias is found. + """ + cls = find_formatter_class(_alias) + if cls is None: + raise ClassNotFound(f"no formatter found for name {_alias!r}") + return cls(**options) + + +def load_formatter_from_file(filename, formattername="CustomFormatter", **options): + """ + Return a `Formatter` subclass instance loaded from the provided file, relative + to the current directory. + + The file is expected to contain a Formatter class named ``formattername`` + (by default, CustomFormatter). Users should be very careful with the input, because + this method is equivalent to running ``eval()`` on the input file. The formatter is + given the `options` at its instantiation. + + :exc:`pygments.util.ClassNotFound` is raised if there are any errors loading + the formatter. + + .. versionadded:: 2.2 + """ + try: + # This empty dict will contain the namespace for the exec'd file + custom_namespace = {} + with open(filename, 'rb') as f: + exec(f.read(), custom_namespace) + # Retrieve the class `formattername` from that namespace + if formattername not in custom_namespace: + raise ClassNotFound(f'no valid {formattername} class found in {filename}') + formatter_class = custom_namespace[formattername] + # And finally instantiate it with the options + return formatter_class(**options) + except OSError as err: + raise ClassNotFound(f'cannot read {filename}: {err}') + except ClassNotFound: + raise + except Exception as err: + raise ClassNotFound(f'error when loading custom formatter: {err}') + + +def get_formatter_for_filename(fn, **options): + """ + Return a :class:`.Formatter` subclass instance that has a filename pattern + matching `fn`. The formatter is given the `options` at its instantiation. + + Will raise :exc:`pygments.util.ClassNotFound` if no formatter for that filename + is found. + """ + fn = basename(fn) + for modname, name, _, filenames, _ in FORMATTERS.values(): + for filename in filenames: + if _fn_matches(fn, filename): + if name not in _formatter_cache: + _load_formatters(modname) + return _formatter_cache[name](**options) + for _name, cls in find_plugin_formatters(): + for filename in cls.filenames: + if _fn_matches(fn, filename): + return cls(**options) + raise ClassNotFound(f"no formatter found for file name {fn!r}") + + +class _automodule(types.ModuleType): + """Automatically import formatters.""" + + def __getattr__(self, name): + info = FORMATTERS.get(name) + if info: + _load_formatters(info[0]) + cls = _formatter_cache[info[1]] + setattr(self, name, cls) + return cls + raise AttributeError(name) + + +oldmod = sys.modules[__name__] +newmod = _automodule(__name__) +newmod.__dict__.update(oldmod.__dict__) +sys.modules[__name__] = newmod +del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types diff --git a/lib/pygments/formatters/__pycache__/__init__.cpython-314.pyc b/lib/pygments/formatters/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..a0f18b9 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/__init__.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/_mapping.cpython-314.pyc b/lib/pygments/formatters/__pycache__/_mapping.cpython-314.pyc new file mode 100644 index 0000000..7ebbf53 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/_mapping.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/bbcode.cpython-314.pyc b/lib/pygments/formatters/__pycache__/bbcode.cpython-314.pyc new file mode 100644 index 0000000..2e57566 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/bbcode.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/groff.cpython-314.pyc b/lib/pygments/formatters/__pycache__/groff.cpython-314.pyc new file mode 100644 index 0000000..dab46f1 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/groff.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/html.cpython-314.pyc b/lib/pygments/formatters/__pycache__/html.cpython-314.pyc new file mode 100644 index 0000000..a130519 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/html.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/img.cpython-314.pyc b/lib/pygments/formatters/__pycache__/img.cpython-314.pyc new file mode 100644 index 0000000..fe80408 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/img.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/irc.cpython-314.pyc b/lib/pygments/formatters/__pycache__/irc.cpython-314.pyc new file mode 100644 index 0000000..5bcefae Binary files /dev/null and b/lib/pygments/formatters/__pycache__/irc.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/latex.cpython-314.pyc b/lib/pygments/formatters/__pycache__/latex.cpython-314.pyc new file mode 100644 index 0000000..e18bc16 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/latex.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/other.cpython-314.pyc b/lib/pygments/formatters/__pycache__/other.cpython-314.pyc new file mode 100644 index 0000000..25351a1 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/other.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/pangomarkup.cpython-314.pyc b/lib/pygments/formatters/__pycache__/pangomarkup.cpython-314.pyc new file mode 100644 index 0000000..bc4c889 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/pangomarkup.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/rtf.cpython-314.pyc b/lib/pygments/formatters/__pycache__/rtf.cpython-314.pyc new file mode 100644 index 0000000..0fa50d8 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/rtf.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/svg.cpython-314.pyc b/lib/pygments/formatters/__pycache__/svg.cpython-314.pyc new file mode 100644 index 0000000..85071bc Binary files /dev/null and b/lib/pygments/formatters/__pycache__/svg.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/terminal.cpython-314.pyc b/lib/pygments/formatters/__pycache__/terminal.cpython-314.pyc new file mode 100644 index 0000000..f5eb1c5 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/terminal.cpython-314.pyc differ diff --git a/lib/pygments/formatters/__pycache__/terminal256.cpython-314.pyc b/lib/pygments/formatters/__pycache__/terminal256.cpython-314.pyc new file mode 100644 index 0000000..c7a1f02 Binary files /dev/null and b/lib/pygments/formatters/__pycache__/terminal256.cpython-314.pyc differ diff --git a/lib/pygments/formatters/_mapping.py b/lib/pygments/formatters/_mapping.py new file mode 100644 index 0000000..72ca840 --- /dev/null +++ b/lib/pygments/formatters/_mapping.py @@ -0,0 +1,23 @@ +# Automatically generated by scripts/gen_mapfiles.py. +# DO NOT EDIT BY HAND; run `tox -e mapfiles` instead. + +FORMATTERS = { + 'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'), + 'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'GroffFormatter': ('pygments.formatters.groff', 'groff', ('groff', 'troff', 'roff'), (), 'Format tokens with groff escapes to change their color and font style.'), + 'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ```` tags. By default, the content is enclosed in a ``
`` tag, itself wrapped in a ``
`` tag (but see the `nowrap` option). The ``
``'s CSS class can be set by the `cssclass` option."),
+    'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'),
+    'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
+    'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
+    'PangoMarkupFormatter': ('pygments.formatters.pangomarkup', 'Pango Markup', ('pango', 'pangomarkup'), (), 'Format tokens as Pango Markup code. It can then be rendered to an SVG.'),
+    'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
+    'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'),
+    'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ```` element with explicit ``x`` and ``y`` coordinates containing ```` elements with the individual token styles.'),
+    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.'),
+}
diff --git a/lib/pygments/formatters/bbcode.py b/lib/pygments/formatters/bbcode.py
new file mode 100644
index 0000000..339edf9
--- /dev/null
+++ b/lib/pygments/formatters/bbcode.py
@@ -0,0 +1,108 @@
+"""
+    pygments.formatters.bbcode
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    BBcode formatter.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+from pygments.formatter import Formatter
+from pygments.util import get_bool_opt
+
+__all__ = ['BBCodeFormatter']
+
+
+class BBCodeFormatter(Formatter):
+    """
+    Format tokens with BBcodes. These formatting codes are used by many
+    bulletin boards, so you can highlight your sourcecode with pygments before
+    posting it there.
+
+    This formatter has no support for background colors and borders, as there
+    are no common BBcode tags for that.
+
+    Some board systems (e.g. phpBB) don't support colors in their [code] tag,
+    so you can't use the highlighting together with that tag.
+    Text in a [code] tag usually is shown with a monospace font (which this
+    formatter can do with the ``monofont`` option) and no spaces (which you
+    need for indentation) are removed.
+
+    Additional options accepted:
+
+    `style`
+        The style to use, can be a string or a Style subclass (default:
+        ``'default'``).
+
+    `codetag`
+        If set to true, put the output into ``[code]`` tags (default:
+        ``false``)
+
+    `monofont`
+        If set to true, add a tag to show the code with a monospace font
+        (default: ``false``).
+    """
+    name = 'BBCode'
+    aliases = ['bbcode', 'bb']
+    filenames = []
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+        self._code = get_bool_opt(options, 'codetag', False)
+        self._mono = get_bool_opt(options, 'monofont', False)
+
+        self.styles = {}
+        self._make_styles()
+
+    def _make_styles(self):
+        for ttype, ndef in self.style:
+            start = end = ''
+            if ndef['color']:
+                start += '[color=#{}]'.format(ndef['color'])
+                end = '[/color]' + end
+            if ndef['bold']:
+                start += '[b]'
+                end = '[/b]' + end
+            if ndef['italic']:
+                start += '[i]'
+                end = '[/i]' + end
+            if ndef['underline']:
+                start += '[u]'
+                end = '[/u]' + end
+            # there are no common BBcodes for background-color and border
+
+            self.styles[ttype] = start, end
+
+    def format_unencoded(self, tokensource, outfile):
+        if self._code:
+            outfile.write('[code]')
+        if self._mono:
+            outfile.write('[font=monospace]')
+
+        lastval = ''
+        lasttype = None
+
+        for ttype, value in tokensource:
+            while ttype not in self.styles:
+                ttype = ttype.parent
+            if ttype == lasttype:
+                lastval += value
+            else:
+                if lastval:
+                    start, end = self.styles[lasttype]
+                    outfile.write(''.join((start, lastval, end)))
+                lastval = value
+                lasttype = ttype
+
+        if lastval:
+            start, end = self.styles[lasttype]
+            outfile.write(''.join((start, lastval, end)))
+
+        if self._mono:
+            outfile.write('[/font]')
+        if self._code:
+            outfile.write('[/code]')
+        if self._code or self._mono:
+            outfile.write('\n')
diff --git a/lib/pygments/formatters/groff.py b/lib/pygments/formatters/groff.py
new file mode 100644
index 0000000..028fec4
--- /dev/null
+++ b/lib/pygments/formatters/groff.py
@@ -0,0 +1,170 @@
+"""
+    pygments.formatters.groff
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Formatter for groff output.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import math
+from pygments.formatter import Formatter
+from pygments.util import get_bool_opt, get_int_opt
+
+__all__ = ['GroffFormatter']
+
+
+class GroffFormatter(Formatter):
+    """
+    Format tokens with groff escapes to change their color and font style.
+
+    .. versionadded:: 2.11
+
+    Additional options accepted:
+
+    `style`
+        The style to use, can be a string or a Style subclass (default:
+        ``'default'``).
+
+    `monospaced`
+        If set to true, monospace font will be used (default: ``true``).
+
+    `linenos`
+        If set to true, print the line numbers (default: ``false``).
+
+    `wrap`
+        Wrap lines to the specified number of characters. Disabled if set to 0
+        (default: ``0``).
+    """
+
+    name = 'groff'
+    aliases = ['groff','troff','roff']
+    filenames = []
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+
+        self.monospaced = get_bool_opt(options, 'monospaced', True)
+        self.linenos = get_bool_opt(options, 'linenos', False)
+        self._lineno = 0
+        self.wrap = get_int_opt(options, 'wrap', 0)
+        self._linelen = 0
+
+        self.styles = {}
+        self._make_styles()
+
+
+    def _make_styles(self):
+        regular = '\\f[CR]' if self.monospaced else '\\f[R]'
+        bold = '\\f[CB]' if self.monospaced else '\\f[B]'
+        italic = '\\f[CI]' if self.monospaced else '\\f[I]'
+
+        for ttype, ndef in self.style:
+            start = end = ''
+            if ndef['color']:
+                start += '\\m[{}]'.format(ndef['color'])
+                end = '\\m[]' + end
+            if ndef['bold']:
+                start += bold
+                end = regular + end
+            if ndef['italic']:
+                start += italic
+                end = regular + end
+            if ndef['bgcolor']:
+                start += '\\M[{}]'.format(ndef['bgcolor'])
+                end = '\\M[]' + end
+
+            self.styles[ttype] = start, end
+
+
+    def _define_colors(self, outfile):
+        colors = set()
+        for _, ndef in self.style:
+            if ndef['color'] is not None:
+                colors.add(ndef['color'])
+
+        for color in sorted(colors):
+            outfile.write('.defcolor ' + color + ' rgb #' + color + '\n')
+
+
+    def _write_lineno(self, outfile):
+        self._lineno += 1
+        outfile.write("%s% 4d " % (self._lineno != 1 and '\n' or '', self._lineno))
+
+
+    def _wrap_line(self, line):
+        length = len(line.rstrip('\n'))
+        space = '     ' if self.linenos else ''
+        newline = ''
+
+        if length > self.wrap:
+            for i in range(0, math.floor(length / self.wrap)):
+                chunk = line[i*self.wrap:i*self.wrap+self.wrap]
+                newline += (chunk + '\n' + space)
+            remainder = length % self.wrap
+            if remainder > 0:
+                newline += line[-remainder-1:]
+                self._linelen = remainder
+        elif self._linelen + length > self.wrap:
+            newline = ('\n' + space) + line
+            self._linelen = length
+        else:
+            newline = line
+            self._linelen += length
+
+        return newline
+
+
+    def _escape_chars(self, text):
+        text = text.replace('\\', '\\[u005C]'). \
+                    replace('.', '\\[char46]'). \
+                    replace('\'', '\\[u0027]'). \
+                    replace('`', '\\[u0060]'). \
+                    replace('~', '\\[u007E]')
+        copy = text
+
+        for char in copy:
+            if len(char) != len(char.encode()):
+                uni = char.encode('unicode_escape') \
+                    .decode()[1:] \
+                    .replace('x', 'u00') \
+                    .upper()
+                text = text.replace(char, '\\[u' + uni[1:] + ']')
+
+        return text
+
+
+    def format_unencoded(self, tokensource, outfile):
+        self._define_colors(outfile)
+
+        outfile.write('.nf\n\\f[CR]\n')
+
+        if self.linenos:
+            self._write_lineno(outfile)
+
+        for ttype, value in tokensource:
+            while ttype not in self.styles:
+                ttype = ttype.parent
+            start, end = self.styles[ttype]
+
+            for line in value.splitlines(True):
+                if self.wrap > 0:
+                    line = self._wrap_line(line)
+
+                if start and end:
+                    text = self._escape_chars(line.rstrip('\n'))
+                    if text != '':
+                        outfile.write(''.join((start, text, end)))
+                else:
+                    outfile.write(self._escape_chars(line.rstrip('\n')))
+
+                if line.endswith('\n'):
+                    if self.linenos:
+                        self._write_lineno(outfile)
+                        self._linelen = 0
+                    else:
+                        outfile.write('\n')
+                        self._linelen = 0
+
+        outfile.write('\n.fi')
diff --git a/lib/pygments/formatters/html.py b/lib/pygments/formatters/html.py
new file mode 100644
index 0000000..4ef1836
--- /dev/null
+++ b/lib/pygments/formatters/html.py
@@ -0,0 +1,995 @@
+"""
+    pygments.formatters.html
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Formatter for HTML output.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import functools
+import os
+import sys
+import os.path
+from io import StringIO
+
+from pygments.formatter import Formatter
+from pygments.token import Token, Text, STANDARD_TYPES
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt
+
+try:
+    import ctags
+except ImportError:
+    ctags = None
+
+__all__ = ['HtmlFormatter']
+
+
+_escape_html_table = {
+    ord('&'): '&',
+    ord('<'): '<',
+    ord('>'): '>',
+    ord('"'): '"',
+    ord("'"): ''',
+}
+
+
+def escape_html(text, table=_escape_html_table):
+    """Escape &, <, > as well as single and double quotes for HTML."""
+    return text.translate(table)
+
+
+def webify(color):
+    if color.startswith('calc') or color.startswith('var'):
+        return color
+    else:
+        # Check if the color can be shortened from 6 to 3 characters
+        color = color.upper()
+        if (len(color) == 6 and
+            (    color[0] == color[1]
+             and color[2] == color[3]
+             and color[4] == color[5])):
+            return f'#{color[0]}{color[2]}{color[4]}'
+        else:
+            return f'#{color}'
+
+
+def _get_ttype_class(ttype):
+    fname = STANDARD_TYPES.get(ttype)
+    if fname:
+        return fname
+    aname = ''
+    while fname is None:
+        aname = '-' + ttype[-1] + aname
+        ttype = ttype.parent
+        fname = STANDARD_TYPES.get(ttype)
+    return fname + aname
+
+
+CSSFILE_TEMPLATE = '''\
+/*
+generated by Pygments 
+Copyright 2006-2025 by the Pygments team.
+Licensed under the BSD license, see LICENSE for details.
+*/
+%(styledefs)s
+'''
+
+DOC_HEADER = '''\
+
+
+
+
+  %(title)s
+  
+  
+
+
+
%(title)s

+
+'''
+
+DOC_HEADER_EXTERNALCSS = '''\
+
+
+
+
+  %(title)s
+  
+  
+
+
+
%(title)s

+
+'''
+
+DOC_FOOTER = '''\
+
+
+'''
+
+
+class HtmlFormatter(Formatter):
+    r"""
+    Format tokens as HTML 4 ```` tags. By default, the content is enclosed
+    in a ``
`` tag, itself wrapped in a ``
`` tag (but see the `nowrap` option).
+    The ``
``'s CSS class can be set by the `cssclass` option.
+
+    If the `linenos` option is set to ``"table"``, the ``
`` is
+    additionally wrapped inside a ```` which has one row and two
+    cells: one containing the line numbers and one containing the code.
+    Example:
+
+    .. sourcecode:: html
+
+        

+        

+          
+          
+        

+            
1
+            2

+          		
+            
def foo(bar):
+              pass
+            

+          

+
+    (whitespace added to improve clarity).
+
+    A list of lines can be specified using the `hl_lines` option to make these
+    lines highlighted (as of Pygments 0.11).
+
+    With the `full` option, a complete HTML 4 document is output, including
+    the style definitions inside a ``$)', _handle_cssblock),
+
+            include('keywords'),
+            include('inline'),
+        ],
+        'keywords': [
+            (words((
+                '\\define', '\\end', 'caption', 'created', 'modified', 'tags',
+                'title', 'type'), prefix=r'^', suffix=r'\b'),
+             Keyword),
+        ],
+        'inline': [
+            # escape
+            (r'\\.', Text),
+            # created or modified date
+            (r'\d{17}', Number.Integer),
+            # italics
+            (r'(\s)(//[^/]+//)((?=\W|\n))',
+             bygroups(Text, Generic.Emph, Text)),
+            # superscript
+            (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)),
+            # subscript
+            (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)),
+            # underscore
+            (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)),
+            # bold
+            (r"(\s)(''[^']+'')((?=\W|\n))",
+             bygroups(Text, Generic.Strong, Text)),
+            # strikethrough
+            (r'(\s)(~~[^~]+~~)((?=\W|\n))',
+             bygroups(Text, Generic.Deleted, Text)),
+            # TiddlyWiki variables
+            (r'<<[^>]+>>', Name.Tag),
+            (r'\$\$[^$]+\$\$', Name.Tag),
+            (r'\$\([^)]+\)\$', Name.Tag),
+            # TiddlyWiki style or class
+            (r'^@@.*$', Name.Tag),
+            # HTML tags
+            (r']+>', Name.Tag),
+            # inline code
+            (r'`[^`]+`', String.Backtick),
+            # HTML escaped symbols
+            (r'&\S*?;', String.Regex),
+            # Wiki links
+            (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)),
+            # External links
+            (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})',
+            bygroups(Text, Name.Tag, Text, Name.Attribute, Text)),
+            # Transclusion
+            (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)),
+            # URLs
+            (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)),
+
+            # general text, must come last!
+            (r'[\w]+', Text),
+            (r'.', Text)
+        ],
+    }
+
+    def __init__(self, **options):
+        self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
+        RegexLexer.__init__(self, **options)
+
+
+class WikitextLexer(RegexLexer):
+    """
+    For MediaWiki Wikitext.
+
+    Parsing Wikitext is tricky, and results vary between different MediaWiki
+    installations, so we only highlight common syntaxes (built-in or from
+    popular extensions), and also assume templates produce no unbalanced
+    syntaxes.
+    """
+    name = 'Wikitext'
+    url = 'https://www.mediawiki.org/wiki/Wikitext'
+    aliases = ['wikitext', 'mediawiki']
+    filenames = []
+    mimetypes = ['text/x-wiki']
+    version_added = '2.15'
+    flags = re.MULTILINE
+
+    def nowiki_tag_rules(tag_name):
+        return [
+            (rf'(?i)()', bygroups(Punctuation,
+             Name.Tag, Whitespace, Punctuation), '#pop'),
+            include('entity'),
+            include('text'),
+        ]
+
+    def plaintext_tag_rules(tag_name):
+        return [
+            (rf'(?si)(.*?)()', bygroups(Text,
+             Punctuation, Name.Tag, Whitespace, Punctuation), '#pop'),
+        ]
+
+    def delegate_tag_rules(tag_name, lexer, **lexer_kwargs):
+        return [
+            (rf'(?i)()', bygroups(Punctuation,
+             Name.Tag, Whitespace, Punctuation), '#pop'),
+            (rf'(?si).+?(?=)', using(lexer, **lexer_kwargs)),
+        ]
+
+    def text_rules(token):
+        return [
+            (r'\w+', token),
+            (r'[^\S\n]+', token),
+            (r'(?s).', token),
+        ]
+
+    def handle_syntaxhighlight(self, match, ctx):
+        from pygments.lexers import get_lexer_by_name
+
+        attr_content = match.group()
+        start = 0
+        index = 0
+        while True:
+            index = attr_content.find('>', start)
+            # Exclude comment end (-->)
+            if attr_content[index-2:index] != '--':
+                break
+            start = index + 1
+
+        if index == -1:
+            # No tag end
+            yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr'])
+            return
+        attr = attr_content[:index]
+        yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr'])
+        yield match.start(3) + index, Punctuation, '>'
+
+        lexer = None
+        content = attr_content[index+1:]
+        lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr)
+
+        if len(lang_match) >= 1:
+            # Pick the last match in case of multiple matches
+            lang = lang_match[-1][1]
+            try:
+                lexer = get_lexer_by_name(lang)
+            except ClassNotFound:
+                pass
+
+        if lexer is None:
+            yield match.start() + index + 1, Text, content
+        else:
+            yield from lexer.get_tokens_unprocessed(content)
+
+    def handle_score(self, match, ctx):
+        attr_content = match.group()
+        start = 0
+        index = 0
+        while True:
+            index = attr_content.find('>', start)
+            # Exclude comment end (-->)
+            if attr_content[index-2:index] != '--':
+                break
+            start = index + 1
+
+        if index == -1:
+            # No tag end
+            yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr'])
+            return
+        attr = attr_content[:index]
+        content = attr_content[index+1:]
+        yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr'])
+        yield match.start(3) + index, Punctuation, '>'
+
+        lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr)
+        # Pick the last match in case of multiple matches
+        lang = lang_match[-1][1] if len(lang_match) >= 1 else 'lilypond'
+
+        if lang == 'lilypond':  # Case sensitive
+            yield from LilyPondLexer().get_tokens_unprocessed(content)
+        else:  # ABC
+            # FIXME: Use ABC lexer in the future
+            yield match.start() + index + 1, Text, content
+
+    # a-z removed to prevent linter from complaining, REMEMBER to use (?i)
+    title_char = r' %!"$&\'()*,\-./0-9:;=?@A-Z\\\^_`~+\u0080-\uFFFF'
+    nbsp_char = r'(?:\t| |&\#0*160;|&\#[Xx]0*[Aa]0;|[ \xA0\u1680\u2000-\u200A\u202F\u205F\u3000])'
+    link_address = r'(?:[0-9.]+|\[[0-9a-f:.]+\]|[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD])'
+    link_char_class = r'[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD]'
+    double_slashes_i = {
+        '__FORCETOC__', '__NOCONTENTCONVERT__', '__NOCC__', '__NOEDITSECTION__', '__NOGALLERY__',
+        '__NOTITLECONVERT__', '__NOTC__', '__NOTOC__', '__TOC__',
+    }
+    double_slashes = {
+        '__EXPECTUNUSEDCATEGORY__',  '__HIDDENCAT__', '__INDEX__',  '__NEWSECTIONLINK__',
+        '__NOINDEX__',  '__NONEWSECTIONLINK__',  '__STATICREDIRECT__', '__NOGLOBAL__',
+        '__DISAMBIG__', '__EXPECTED_UNCONNECTED_PAGE__',
+    }
+    protocols = {
+        'bitcoin:', 'ftp://', 'ftps://', 'geo:', 'git://', 'gopher://', 'http://', 'https://',
+        'irc://', 'ircs://', 'magnet:', 'mailto:', 'mms://', 'news:', 'nntp://', 'redis://',
+        'sftp://', 'sip:', 'sips:', 'sms:', 'ssh://', 'svn://', 'tel:', 'telnet://', 'urn:',
+        'worldwind://', 'xmpp:', '//',
+    }
+    non_relative_protocols = protocols - {'//'}
+    html_tags = {
+        'abbr', 'b', 'bdi', 'bdo', 'big', 'blockquote', 'br', 'caption', 'center', 'cite', 'code',
+        'data', 'dd', 'del', 'dfn', 'div', 'dl', 'dt', 'em', 'font', 'h1', 'h2', 'h3', 'h4', 'h5',
+        'h6', 'hr', 'i', 'ins', 'kbd', 'li', 'link', 'mark', 'meta', 'ol', 'p', 'q', 'rb', 'rp',
+        'rt', 'rtc', 'ruby', 's', 'samp', 'small', 'span', 'strike', 'strong', 'sub', 'sup',
+        'table', 'td', 'th', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'wbr',
+    }
+    parser_tags = {
+        'graph', 'charinsert', 'rss', 'chem', 'categorytree', 'nowiki', 'inputbox', 'math',
+        'hiero', 'score', 'pre', 'ref', 'translate', 'imagemap', 'templatestyles', 'languages',
+        'noinclude', 'mapframe', 'section', 'poem', 'syntaxhighlight', 'includeonly', 'tvar',
+        'onlyinclude', 'templatedata', 'langconvert', 'timeline', 'dynamicpagelist', 'gallery',
+        'maplink', 'ce', 'references',
+    }
+    variant_langs = {
+        # ZhConverter.php
+        'zh', 'zh-hans', 'zh-hant', 'zh-cn', 'zh-hk', 'zh-mo', 'zh-my', 'zh-sg', 'zh-tw',
+        # WuuConverter.php
+        'wuu', 'wuu-hans', 'wuu-hant',
+        # UzConverter.php
+        'uz', 'uz-latn', 'uz-cyrl',
+        # TlyConverter.php
+        'tly', 'tly-cyrl',
+        # TgConverter.php
+        'tg', 'tg-latn',
+        # SrConverter.php
+        'sr', 'sr-ec', 'sr-el',
+        # ShiConverter.php
+        'shi', 'shi-tfng', 'shi-latn',
+        # ShConverter.php
+        'sh-latn', 'sh-cyrl',
+        # KuConverter.php
+        'ku', 'ku-arab', 'ku-latn',
+        # IuConverter.php
+        'iu', 'ike-cans', 'ike-latn',
+        # GanConverter.php
+        'gan', 'gan-hans', 'gan-hant',
+        # EnConverter.php
+        'en', 'en-x-piglatin',
+        # CrhConverter.php
+        'crh', 'crh-cyrl', 'crh-latn',
+        # BanConverter.php
+        'ban', 'ban-bali', 'ban-x-dharma', 'ban-x-palmleaf', 'ban-x-pku',
+    }
+    magic_vars_i = {
+        'ARTICLEPATH', 'INT', 'PAGEID', 'SCRIPTPATH', 'SERVER', 'SERVERNAME', 'STYLEPATH',
+    }
+    magic_vars = {
+        '!', '=', 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'CONTENTLANGUAGE',
+        'CONTENTLANG', 'CURRENTDAY', 'CURRENTDAY2', 'CURRENTDAYNAME', 'CURRENTDOW', 'CURRENTHOUR',
+        'CURRENTMONTH', 'CURRENTMONTH2', 'CURRENTMONTH1', 'CURRENTMONTHABBREV', 'CURRENTMONTHNAME',
+        'CURRENTMONTHNAMEGEN', 'CURRENTTIME', 'CURRENTTIMESTAMP', 'CURRENTVERSION', 'CURRENTWEEK',
+        'CURRENTYEAR', 'DIRECTIONMARK', 'DIRMARK', 'FULLPAGENAME', 'FULLPAGENAMEE', 'LOCALDAY',
+        'LOCALDAY2', 'LOCALDAYNAME', 'LOCALDOW', 'LOCALHOUR', 'LOCALMONTH', 'LOCALMONTH2',
+        'LOCALMONTH1', 'LOCALMONTHABBREV', 'LOCALMONTHNAME', 'LOCALMONTHNAMEGEN', 'LOCALTIME',
+        'LOCALTIMESTAMP', 'LOCALWEEK', 'LOCALYEAR', 'NAMESPACE', 'NAMESPACEE', 'NAMESPACENUMBER',
+        'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', 'NUMBEROFARTICLES', 'NUMBEROFEDITS',
+        'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', 'PAGELANGUAGE', 'PAGENAME', 'PAGENAMEE',
+        'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', 'REVISIONMONTH1',
+        'REVISIONSIZE', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME',
+        'ROOTPAGENAMEE', 'SITENAME', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE',
+        'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE',
+        'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE',
+    }
+    parser_functions_i = {
+        'ANCHORENCODE', 'BIDI', 'CANONICALURL', 'CANONICALURLE', 'FILEPATH', 'FORMATNUM',
+        'FULLURL', 'FULLURLE', 'GENDER', 'GRAMMAR', 'INT', r'\#LANGUAGE', 'LC', 'LCFIRST', 'LOCALURL',
+        'LOCALURLE', 'NS', 'NSE', 'PADLEFT', 'PADRIGHT', 'PAGEID', 'PLURAL', 'UC', 'UCFIRST',
+        'URLENCODE',
+    }
+    parser_functions = {
+        'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'DEFAULTSORT', 'DEFAULTSORTKEY',
+        'DEFAULTCATEGORYSORT', 'FULLPAGENAME', 'FULLPAGENAMEE', 'NAMESPACE', 'NAMESPACEE',
+        'NAMESPACENUMBER', 'NUMBERINGROUP', 'NUMINGROUP', 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS',
+        'NUMBEROFARTICLES', 'NUMBEROFEDITS', 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS',
+        'PAGENAME', 'PAGENAMEE', 'PAGESINCATEGORY', 'PAGESINCAT', 'PAGESIZE', 'PROTECTIONEXPIRY',
+        'PROTECTIONLEVEL', 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH',
+        'REVISIONMONTH1', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME',
+        'ROOTPAGENAMEE', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE',
+        'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE',
+        'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE',
+        'INT', 'DISPLAYTITLE', 'PAGESINNAMESPACE', 'PAGESINNS',
+    }
+
+    tokens = {
+        'root': [
+            # Redirects
+            (r"""(?xi)
+                (\A\s*?)(\#REDIRECT:?) # may contain a colon
+                (\s+)(\[\[) (?=[^\]\n]* \]\]$)
+            """,
+             bygroups(Whitespace, Keyword, Whitespace, Punctuation), 'redirect-inner'),
+            # Subheadings
+            (r'^(={2,6})(.+?)(\1)(\s*$\n)',
+             bygroups(Generic.Subheading, Generic.Subheading, Generic.Subheading, Whitespace)),
+            # Headings
+            (r'^(=.+?=)(\s*$\n)',
+             bygroups(Generic.Heading, Whitespace)),
+            # Double-slashed magic words
+            (words(double_slashes_i, prefix=r'(?i)'), Name.Function.Magic),
+            (words(double_slashes), Name.Function.Magic),
+            # Raw URLs
+            (r'(?i)\b(?:{}){}{}*'.format('|'.join(protocols),
+             link_address, link_char_class), Name.Label),
+            # Magic links
+            (rf'\b(?:RFC|PMID){nbsp_char}+[0-9]+\b',
+             Name.Function.Magic),
+            (r"""(?x)
+                \bISBN {nbsp_char}
+                (?: 97[89] {nbsp_dash}? )?
+                (?: [0-9] {nbsp_dash}? ){{9}} # escape format()
+                [0-9Xx]\b
+            """.format(nbsp_char=nbsp_char, nbsp_dash=f'(?:-|{nbsp_char})'), Name.Function.Magic),
+            include('list'),
+            include('inline'),
+            include('text'),
+        ],
+        'redirect-inner': [
+            (r'(\]\])(\s*?\n)', bygroups(Punctuation, Whitespace), '#pop'),
+            (r'(\#)([^#]*?)', bygroups(Punctuation, Name.Label)),
+            (rf'(?i)[{title_char}]+', Name.Tag),
+        ],
+        'list': [
+            # Description lists
+            (r'^;', Keyword, 'dt'),
+            # Ordered lists, unordered lists and indents
+            (r'^[#:*]+', Keyword),
+            # Horizontal rules
+            (r'^-{4,}', Keyword),
+        ],
+        'inline': [
+            # Signatures
+            (r'~{3,5}', Keyword),
+            # Entities
+            include('entity'),
+            # Bold & italic
+            (r"('')(''')(?!')", bygroups(Generic.Emph,
+             Generic.EmphStrong), 'inline-italic-bold'),
+            (r"'''(?!')", Generic.Strong, 'inline-bold'),
+            (r"''(?!')", Generic.Emph, 'inline-italic'),
+            # Comments & parameters & templates
+            include('replaceable'),
+            # Media links
+            (
+                r"""(?xi)
+                (\[\[)
+                    (File|Image) (:)
+                    ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} |  )*)
+                    (?: (\#) ([{}]*?) )?
+                """.format(title_char, f'{title_char}#'),
+                bygroups(Punctuation, Name.Namespace,  Punctuation,
+                         using(this, state=['wikilink-name']), Punctuation, Name.Label),
+                'medialink-inner'
+            ),
+            # Wikilinks
+            (
+                r"""(?xi)
+                (\[\[)(?!{}) # Should not contain URLs
+                    (?: ([{}]*) (:))?
+                    ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} |  )*?)
+                    (?: (\#) ([{}]*?) )?
+                (\]\])
+                """.format('|'.join(protocols), title_char.replace('/', ''),
+                       title_char, f'{title_char}#'),
+                bygroups(Punctuation, Name.Namespace,  Punctuation,
+                         using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation)
+            ),
+            (
+                r"""(?xi)
+                (\[\[)(?!{})
+                    (?: ([{}]*) (:))?
+                    ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} |  )*?)
+                    (?: (\#) ([{}]*?) )?
+                    (\|)
+                """.format('|'.join(protocols), title_char.replace('/', ''),
+                       title_char, f'{title_char}#'),
+                bygroups(Punctuation, Name.Namespace,  Punctuation,
+                         using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation),
+                'wikilink-inner'
+            ),
+            # External links
+            (
+                r"""(?xi)
+                (\[)
+                    ((?:{}) {} {}*)
+                    (\s*)
+                """.format('|'.join(protocols), link_address, link_char_class),
+                bygroups(Punctuation, Name.Label, Whitespace),
+                'extlink-inner'
+            ),
+            # Tables
+            (r'^(:*)(\s*?)(\{\|)([^\n]*)$', bygroups(Keyword,
+             Whitespace, Punctuation, using(this, state=['root', 'attr'])), 'table'),
+            # HTML tags
+            (r'(?i)(<)({})\b'.format('|'.join(html_tags)),
+             bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
+            (r'(?i)()'.format('|'.join(html_tags)),
+             bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
+            # 
+            (r'(?i)(<)(nowiki)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-nowiki', 'tag-inner')),
+            # 
+            (r'(?i)(<)(pre)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-pre', 'tag-inner')),
+            # 
+            (r'(?i)(<)(categorytree)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-categorytree', 'tag-inner')),
+            # 
+            (r'(?i)(<)(hiero)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-hiero', 'tag-inner')),
+            # 
+            (r'(?i)(<)(math)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-math', 'tag-inner')),
+            # 
+            (r'(?i)(<)(chem)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-chem', 'tag-inner')),
+            # 
+            (r'(?i)(<)(ce)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-ce', 'tag-inner')),
+            # 
+            (r'(?i)(<)(charinsert)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-charinsert', 'tag-inner')),
+            # 
+            (r'(?i)(<)(templatedata)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-templatedata', 'tag-inner')),
+            # 
+            (r'(?i)(<)(gallery)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-gallery', 'tag-inner')),
+            # 
+            (r'(?i)(<)(gallery)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-graph', 'tag-inner')),
+            # 
+            (r'(?i)(<)(dynamicpagelist)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-dynamicpagelist', 'tag-inner')),
+            # 
+            (r'(?i)(<)(inputbox)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-inputbox', 'tag-inner')),
+            # 
+            (r'(?i)(<)(rss)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-rss', 'tag-inner')),
+            # 
+            (r'(?i)(<)(imagemap)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-imagemap', 'tag-inner')),
+            # 
+            (r'(?i)()',
+             bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
+            (r'(?si)(<)(syntaxhighlight)\b([^>]*?(?.*?)(?=)',
+             bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)),
+            # : Fallback case for self-closing tags
+            (r'(?i)(<)(syntaxhighlight)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?)*?)(/\s*?(?)*?)(/\s*?(?|\Z)', Comment.Multiline),
+            # Parameters
+            (
+                r"""(?x)
+                (\{{3})
+                    ([^|]*?)
+                    (?=\}{3}|\|)
+                """,
+                bygroups(Punctuation, Name.Variable),
+                'parameter-inner',
+            ),
+            # Magic variables
+            (r'(?i)(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars_i)),
+             bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
+            (r'(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars)),
+                bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
+            # Parser functions & templates
+            (r'\{\{', Punctuation, 'template-begin-space'),
+            #  legacy syntax
+            (r'(?i)(<)(tvar)\b(\|)([^>]*?)(>)', bygroups(Punctuation,
+             Name.Tag, Punctuation, String, Punctuation)),
+            (r'', Punctuation, '#pop'),
+            # 
+            (r'(?i)(<)(tvar)\b', bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
+            (r'(?i)()',
+             bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
+        ],
+        'parameter-inner': [
+            (r'\}{3}', Punctuation, '#pop'),
+            (r'\|', Punctuation),
+            include('inline'),
+            include('text'),
+        ],
+        'template-begin-space': [
+            # Templates allow line breaks at the beginning, and due to how MediaWiki handles
+            # comments, an extra state is required to handle things like {{\n\n name}}
+            (r'|\Z)', Comment.Multiline),
+            (r'\s+', Whitespace),
+            # Parser functions
+            (
+                r'(?i)(\#[{}]*?|{})(:)'.format(title_char,
+                                           '|'.join(parser_functions_i)),
+                bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
+            ),
+            (
+                r'({})(:)'.format('|'.join(parser_functions)),
+                bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
+            ),
+            # Templates
+            (
+                rf'(?i)([{title_char}]*?)(:)',
+                bygroups(Name.Namespace, Punctuation), ('#pop', 'template-name')
+            ),
+            default(('#pop', 'template-name'),),
+        ],
+        'template-name': [
+            (r'(\s*?)(\|)', bygroups(Text, Punctuation), ('#pop', 'template-inner')),
+            (r'\}\}', Punctuation, '#pop'),
+            (r'\n', Text, '#pop'),
+            include('replaceable'),
+            *text_rules(Name.Tag),
+        ],
+        'template-inner': [
+            (r'\}\}', Punctuation, '#pop'),
+            (r'\|', Punctuation),
+            (
+                r"""(?x)
+                    (?<=\|)
+                    ( (?: (?! \{\{ | \}\} )[^=\|<])*? ) # Exclude templates and tags
+                    (=)
+                """,
+                bygroups(Name.Label, Operator)
+            ),
+            include('inline'),
+            include('text'),
+        ],
+        'table': [
+            # Use [ \t\n\r\0\x0B] instead of \s to follow PHP trim() behavior
+            # Endings
+            (r'^([ \t\n\r\0\x0B]*?)(\|\})',
+             bygroups(Whitespace, Punctuation), '#pop'),
+            # Table rows
+            (r'^([ \t\n\r\0\x0B]*?)(\|-+)(.*)$', bygroups(Whitespace, Punctuation,
+             using(this, state=['root', 'attr']))),
+            # Captions
+            (
+                r"""(?x)
+                ^([ \t\n\r\0\x0B]*?)(\|\+)
+                # Exclude links, template and tags
+                (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|) )?
+                (.*?)$
+                """,
+                bygroups(Whitespace, Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation, Generic.Heading),
+            ),
+            # Table data
+            (
+                r"""(?x)
+                ( ^(?:[ \t\n\r\0\x0B]*?)\| | \|\| )
+                (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
+                """,
+                bygroups(Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation),
+            ),
+            # Table headers
+            (
+                r"""(?x)
+                ( ^(?:[ \t\n\r\0\x0B]*?)!  )
+                (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
+                """,
+                bygroups(Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation),
+                'table-header',
+            ),
+            include('list'),
+            include('inline'),
+            include('text'),
+        ],
+        'table-header': [
+            # Requires another state for || handling inside headers
+            (r'\n', Text, '#pop'),
+            (
+                r"""(?x)
+                (!!|\|\|)
+                (?:
+                    ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )
+                    (\|)(?!\|)
+                )?
+                """,
+                bygroups(Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation)
+            ),
+            *text_rules(Generic.Subheading),
+        ],
+        'entity': [
+            (r'&\S*?;', Name.Entity),
+        ],
+        'dt': [
+            (r'\n', Text, '#pop'),
+            include('inline'),
+            (r':', Keyword, '#pop'),
+            include('text'),
+        ],
+        'extlink-inner': [
+            (r'\]', Punctuation, '#pop'),
+            include('inline'),
+            include('text'),
+        ],
+        'nowiki-ish': [
+            include('entity'),
+            include('text'),
+        ],
+        'attr': [
+            include('replaceable'),
+            (r'\s+', Whitespace),
+            (r'(=)(\s*)(")', bygroups(Operator, Whitespace, String.Double), 'attr-val-2'),
+            (r"(=)(\s*)(')", bygroups(Operator, Whitespace, String.Single), 'attr-val-1'),
+            (r'(=)(\s*)', bygroups(Operator, Whitespace), 'attr-val-0'),
+            (r'[\w:-]+', Name.Attribute),
+
+        ],
+        'attr-val-0': [
+            (r'\s', Whitespace, '#pop'),
+            include('replaceable'),
+            *text_rules(String),
+        ],
+        'attr-val-1': [
+            (r"'", String.Single, '#pop'),
+            include('replaceable'),
+            *text_rules(String.Single),
+        ],
+        'attr-val-2': [
+            (r'"', String.Double, '#pop'),
+            include('replaceable'),
+            *text_rules(String.Double),
+        ],
+        'tag-inner-ordinary': [
+            (r'/?\s*>', Punctuation, '#pop'),
+            include('tag-attr'),
+        ],
+        'tag-inner': [
+            # Return to root state for self-closing tags
+            (r'/\s*>', Punctuation, '#pop:2'),
+            (r'\s*>', Punctuation, '#pop'),
+            include('tag-attr'),
+        ],
+        # There states below are just like their non-tag variants, the key difference is
+        # they forcibly quit when encountering tag closing markup
+        'tag-attr': [
+            include('replaceable'),
+            (r'\s+', Whitespace),
+            (r'(=)(\s*)(")', bygroups(Operator,
+             Whitespace, String.Double), 'tag-attr-val-2'),
+            (r"(=)(\s*)(')", bygroups(Operator,
+             Whitespace, String.Single), 'tag-attr-val-1'),
+            (r'(=)(\s*)', bygroups(Operator, Whitespace), 'tag-attr-val-0'),
+            (r'[\w:-]+', Name.Attribute),
+
+        ],
+        'tag-attr-val-0': [
+            (r'\s', Whitespace, '#pop'),
+            (r'/?>', Punctuation, '#pop:2'),
+            include('replaceable'),
+            *text_rules(String),
+        ],
+        'tag-attr-val-1': [
+            (r"'", String.Single, '#pop'),
+            (r'/?>', Punctuation, '#pop:2'),
+            include('replaceable'),
+            *text_rules(String.Single),
+        ],
+        'tag-attr-val-2': [
+            (r'"', String.Double, '#pop'),
+            (r'/?>', Punctuation, '#pop:2'),
+            include('replaceable'),
+            *text_rules(String.Double),
+        ],
+        'tag-nowiki': nowiki_tag_rules('nowiki'),
+        'tag-pre': nowiki_tag_rules('pre'),
+        'tag-categorytree': plaintext_tag_rules('categorytree'),
+        'tag-dynamicpagelist': plaintext_tag_rules('dynamicpagelist'),
+        'tag-hiero': plaintext_tag_rules('hiero'),
+        'tag-inputbox': plaintext_tag_rules('inputbox'),
+        'tag-imagemap': plaintext_tag_rules('imagemap'),
+        'tag-charinsert': plaintext_tag_rules('charinsert'),
+        'tag-timeline': plaintext_tag_rules('timeline'),
+        'tag-gallery': plaintext_tag_rules('gallery'),
+        'tag-graph': plaintext_tag_rules('graph'),
+        'tag-rss': plaintext_tag_rules('rss'),
+        'tag-math': delegate_tag_rules('math', TexLexer, state='math'),
+        'tag-chem': delegate_tag_rules('chem', TexLexer, state='math'),
+        'tag-ce': delegate_tag_rules('ce', TexLexer, state='math'),
+        'tag-templatedata': delegate_tag_rules('templatedata', JsonLexer),
+        'text-italic': text_rules(Generic.Emph),
+        'text-bold': text_rules(Generic.Strong),
+        'text-bold-italic': text_rules(Generic.EmphStrong),
+        'text': text_rules(Text),
+    }
diff --git a/lib/pygments/lexers/math.py b/lib/pygments/lexers/math.py
new file mode 100644
index 0000000..b225ffc
--- /dev/null
+++ b/lib/pygments/lexers/math.py
@@ -0,0 +1,21 @@
+"""
+    pygments.lexers.math
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Just export lexers that were contained in this module.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.python import NumPyLexer
+from pygments.lexers.matlab import MatlabLexer, MatlabSessionLexer, \
+    OctaveLexer, ScilabLexer
+from pygments.lexers.julia import JuliaLexer, JuliaConsoleLexer
+from pygments.lexers.r import RConsoleLexer, SLexer, RdLexer
+from pygments.lexers.modeling import BugsLexer, JagsLexer, StanLexer
+from pygments.lexers.idl import IDLLexer
+from pygments.lexers.algebra import MuPADLexer
+
+__all__ = []
diff --git a/lib/pygments/lexers/matlab.py b/lib/pygments/lexers/matlab.py
new file mode 100644
index 0000000..8eeffc9
--- /dev/null
+++ b/lib/pygments/lexers/matlab.py
@@ -0,0 +1,3307 @@
+"""
+    pygments.lexers.matlab
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Matlab and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, default, words, \
+    do_insertions, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Generic, Whitespace
+
+from pygments.lexers import _scilab_builtins
+
+__all__ = ['MatlabLexer', 'MatlabSessionLexer', 'OctaveLexer', 'ScilabLexer']
+
+
+class MatlabLexer(RegexLexer):
+    """
+    For Matlab source code.
+    """
+    name = 'Matlab'
+    aliases = ['matlab']
+    filenames = ['*.m']
+    mimetypes = ['text/matlab']
+    url = 'https://www.mathworks.com/products/matlab.html'
+    version_added = '0.10'
+
+    _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\^|\.\\|\./|/|\\'
+
+    tokens = {
+        'expressions': [
+            # operators:
+            (_operators, Operator),
+
+            # numbers (must come before punctuation to handle `.5`; cannot use
+            # `\b` due to e.g. `5. + .5`).  The negative lookahead on operators
+            # avoids including the dot in `1./x` (the dot is part of `./`).
+            (rf'(? and then
+            # (equal | open-parenthesis |  | ).
+            (rf'(?:^|(?<=;))(\s*)(\w+)(\s+)(?!=|\(|{_operators}\s|\s)',
+             bygroups(Whitespace, Name, Whitespace), 'commandargs'),
+
+            include('expressions')
+        ],
+        'blockcomment': [
+            (r'^\s*%\}', Comment.Multiline, '#pop'),
+            (r'^.*\n', Comment.Multiline),
+            (r'.', Comment.Multiline),
+        ],
+        'deffunc': [
+            (r'(\s*)(?:(\S+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)',
+             bygroups(Whitespace, Text, Whitespace, Punctuation,
+                      Whitespace, Name.Function, Punctuation, Text,
+                      Punctuation, Whitespace), '#pop'),
+            # function with no args
+            (r'(\s*)([a-zA-Z_]\w*)',
+             bygroups(Whitespace, Name.Function), '#pop'),
+        ],
+        'propattrs': [
+            (r'(\w+)(\s*)(=)(\s*)(\d+)',
+             bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace,
+                      Number)),
+            (r'(\w+)(\s*)(=)(\s*)([a-zA-Z]\w*)',
+             bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace,
+                      Keyword)),
+            (r',', Punctuation),
+            (r'\)', Punctuation, '#pop'),
+            (r'\s+', Whitespace),
+            (r'.', Text),
+        ],
+        'defprops': [
+            (r'%\{\s*\n', Comment.Multiline, 'blockcomment'),
+            (r'%.*$', Comment),
+            (r'(?.
+    """
+    name = 'Matlab session'
+    aliases = ['matlabsession']
+    url = 'https://www.mathworks.com/products/matlab.html'
+    version_added = '0.10'
+    _example = "matlabsession/matlabsession_sample.txt"
+
+    def get_tokens_unprocessed(self, text):
+        mlexer = MatlabLexer(**self.options)
+
+        curcode = ''
+        insertions = []
+        continuation = False
+
+        for match in line_re.finditer(text):
+            line = match.group()
+
+            if line.startswith('>> '):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:3])]))
+                curcode += line[3:]
+
+            elif line.startswith('>>'):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:2])]))
+                curcode += line[2:]
+
+            elif line.startswith('???'):
+
+                idx = len(curcode)
+
+                # without is showing error on same line as before...?
+                # line = "\n" + line
+                token = (0, Generic.Traceback, line)
+                insertions.append((idx, [token]))
+            elif continuation and insertions:
+                # line_start is the length of the most recent prompt symbol
+                line_start = len(insertions[-1][-1][-1])
+                # Set leading spaces with the length of the prompt to be a generic prompt
+                # This keeps code aligned when prompts are removed, say with some Javascript
+                if line.startswith(' '*line_start):
+                    insertions.append(
+                        (len(curcode), [(0, Generic.Prompt, line[:line_start])]))
+                    curcode += line[line_start:]
+                else:
+                    curcode += line
+            else:
+                if curcode:
+                    yield from do_insertions(
+                        insertions, mlexer.get_tokens_unprocessed(curcode))
+                    curcode = ''
+                    insertions = []
+
+                yield match.start(), Generic.Output, line
+
+            # Does not allow continuation if a comment is included after the ellipses.
+            # Continues any line that ends with ..., even comments (lines that start with %)
+            if line.strip().endswith('...'):
+                continuation = True
+            else:
+                continuation = False
+
+        if curcode:  # or item:
+            yield from do_insertions(
+                insertions, mlexer.get_tokens_unprocessed(curcode))
+
+
+class OctaveLexer(RegexLexer):
+    """
+    For GNU Octave source code.
+    """
+    name = 'Octave'
+    url = 'https://www.gnu.org/software/octave/index'
+    aliases = ['octave']
+    filenames = ['*.m']
+    mimetypes = ['text/octave']
+    version_added = '1.5'
+
+    # These lists are generated automatically.
+    # Run the following in bash shell:
+    #
+    # First dump all of the Octave manual into a plain text file:
+    #
+    #   $ info octave --subnodes -o octave-manual
+    #
+    # Now grep through it:
+
+    # for i in \
+    #     "Built-in Function" "Command" "Function File" \
+    #     "Loadable Function" "Mapping Function";
+    # do
+    #     perl -e '@name = qw('"$i"');
+    #              print lc($name[0]),"_kw = [\n"';
+    #
+    #     perl -n -e 'print "\"$1\",\n" if /-- '"$i"': .* (\w*) \(/;' \
+    #         octave-manual | sort | uniq ;
+    #     echo "]" ;
+    #     echo;
+    # done
+
+    # taken from Octave Mercurial changeset 8cc154f45e37 (30-jan-2011)
+
+    builtin_kw = (
+        "addlistener", "addpath", "addproperty", "all",
+        "and", "any", "argnames", "argv", "assignin",
+        "atexit", "autoload",
+        "available_graphics_toolkits", "beep_on_error",
+        "bitand", "bitmax", "bitor", "bitshift", "bitxor",
+        "cat", "cell", "cellstr", "char", "class", "clc",
+        "columns", "command_line_path",
+        "completion_append_char", "completion_matches",
+        "complex", "confirm_recursive_rmdir", "cputime",
+        "crash_dumps_octave_core", "ctranspose", "cumprod",
+        "cumsum", "debug_on_error", "debug_on_interrupt",
+        "debug_on_warning", "default_save_options",
+        "dellistener", "diag", "diff", "disp",
+        "doc_cache_file", "do_string_escapes", "double",
+        "drawnow", "e", "echo_executing_commands", "eps",
+        "eq", "errno", "errno_list", "error", "eval",
+        "evalin", "exec", "exist", "exit", "eye", "false",
+        "fclear", "fclose", "fcntl", "fdisp", "feof",
+        "ferror", "feval", "fflush", "fgetl", "fgets",
+        "fieldnames", "file_in_loadpath", "file_in_path",
+        "filemarker", "filesep", "find_dir_in_path",
+        "fixed_point_format", "fnmatch", "fopen", "fork",
+        "formula", "fprintf", "fputs", "fread", "freport",
+        "frewind", "fscanf", "fseek", "fskipl", "ftell",
+        "functions", "fwrite", "ge", "genpath", "get",
+        "getegid", "getenv", "geteuid", "getgid",
+        "getpgrp", "getpid", "getppid", "getuid", "glob",
+        "gt", "gui_mode", "history_control",
+        "history_file", "history_size",
+        "history_timestamp_format_string", "home",
+        "horzcat", "hypot", "ifelse",
+        "ignore_function_time_stamp", "inferiorto",
+        "info_file", "info_program", "inline", "input",
+        "intmax", "intmin", "ipermute",
+        "is_absolute_filename", "isargout", "isbool",
+        "iscell", "iscellstr", "ischar", "iscomplex",
+        "isempty", "isfield", "isfloat", "isglobal",
+        "ishandle", "isieee", "isindex", "isinteger",
+        "islogical", "ismatrix", "ismethod", "isnull",
+        "isnumeric", "isobject", "isreal",
+        "is_rooted_relative_filename", "issorted",
+        "isstruct", "isvarname", "kbhit", "keyboard",
+        "kill", "lasterr", "lasterror", "lastwarn",
+        "ldivide", "le", "length", "link", "linspace",
+        "logical", "lstat", "lt", "make_absolute_filename",
+        "makeinfo_program", "max_recursion_depth", "merge",
+        "methods", "mfilename", "minus", "mislocked",
+        "mkdir", "mkfifo", "mkstemp", "mldivide", "mlock",
+        "mouse_wheel_zoom", "mpower", "mrdivide", "mtimes",
+        "munlock", "nargin", "nargout",
+        "native_float_format", "ndims", "ne", "nfields",
+        "nnz", "norm", "not", "numel", "nzmax",
+        "octave_config_info", "octave_core_file_limit",
+        "octave_core_file_name",
+        "octave_core_file_options", "ones", "or",
+        "output_max_field_width", "output_precision",
+        "page_output_immediately", "page_screen_output",
+        "path", "pathsep", "pause", "pclose", "permute",
+        "pi", "pipe", "plus", "popen", "power",
+        "print_empty_dimensions", "printf",
+        "print_struct_array_contents", "prod",
+        "program_invocation_name", "program_name",
+        "putenv", "puts", "pwd", "quit", "rats", "rdivide",
+        "readdir", "readlink", "read_readline_init_file",
+        "realmax", "realmin", "rehash", "rename",
+        "repelems", "re_read_readline_init_file", "reset",
+        "reshape", "resize", "restoredefaultpath",
+        "rethrow", "rmdir", "rmfield", "rmpath", "rows",
+        "save_header_format_string", "save_precision",
+        "saving_history", "scanf", "set", "setenv",
+        "shell_cmd", "sighup_dumps_octave_core",
+        "sigterm_dumps_octave_core", "silent_functions",
+        "single", "size", "size_equal", "sizemax",
+        "sizeof", "sleep", "source", "sparse_auto_mutate",
+        "split_long_rows", "sprintf", "squeeze", "sscanf",
+        "stat", "stderr", "stdin", "stdout", "strcmp",
+        "strcmpi", "string_fill_char", "strncmp",
+        "strncmpi", "struct", "struct_levels_to_print",
+        "strvcat", "subsasgn", "subsref", "sum", "sumsq",
+        "superiorto", "suppress_verbose_help_message",
+        "symlink", "system", "tic", "tilde_expand",
+        "times", "tmpfile", "tmpnam", "toc", "toupper",
+        "transpose", "true", "typeinfo", "umask", "uminus",
+        "uname", "undo_string_escapes", "unlink", "uplus",
+        "upper", "usage", "usleep", "vec", "vectorize",
+        "vertcat", "waitpid", "warning", "warranty",
+        "whos_line_format", "yes_or_no", "zeros",
+        "inf", "Inf", "nan", "NaN")
+
+    command_kw = ("close", "load", "who", "whos")
+
+    function_kw = (
+        "accumarray", "accumdim", "acosd", "acotd",
+        "acscd", "addtodate", "allchild", "ancestor",
+        "anova", "arch_fit", "arch_rnd", "arch_test",
+        "area", "arma_rnd", "arrayfun", "ascii", "asctime",
+        "asecd", "asind", "assert", "atand",
+        "autoreg_matrix", "autumn", "axes", "axis", "bar",
+        "barh", "bartlett", "bartlett_test", "beep",
+        "betacdf", "betainv", "betapdf", "betarnd",
+        "bicgstab", "bicubic", "binary", "binocdf",
+        "binoinv", "binopdf", "binornd", "bitcmp",
+        "bitget", "bitset", "blackman", "blanks",
+        "blkdiag", "bone", "box", "brighten", "calendar",
+        "cast", "cauchy_cdf", "cauchy_inv", "cauchy_pdf",
+        "cauchy_rnd", "caxis", "celldisp", "center", "cgs",
+        "chisquare_test_homogeneity",
+        "chisquare_test_independence", "circshift", "cla",
+        "clabel", "clf", "clock", "cloglog", "closereq",
+        "colon", "colorbar", "colormap", "colperm",
+        "comet", "common_size", "commutation_matrix",
+        "compan", "compare_versions", "compass",
+        "computer", "cond", "condest", "contour",
+        "contourc", "contourf", "contrast", "conv",
+        "convhull", "cool", "copper", "copyfile", "cor",
+        "corrcoef", "cor_test", "cosd", "cotd", "cov",
+        "cplxpair", "cross", "cscd", "cstrcat", "csvread",
+        "csvwrite", "ctime", "cumtrapz", "curl", "cut",
+        "cylinder", "date", "datenum", "datestr",
+        "datetick", "datevec", "dblquad", "deal",
+        "deblank", "deconv", "delaunay", "delaunayn",
+        "delete", "demo", "detrend", "diffpara", "diffuse",
+        "dir", "discrete_cdf", "discrete_inv",
+        "discrete_pdf", "discrete_rnd", "display",
+        "divergence", "dlmwrite", "dos", "dsearch",
+        "dsearchn", "duplication_matrix", "durbinlevinson",
+        "ellipsoid", "empirical_cdf", "empirical_inv",
+        "empirical_pdf", "empirical_rnd", "eomday",
+        "errorbar", "etime", "etreeplot", "example",
+        "expcdf", "expinv", "expm", "exppdf", "exprnd",
+        "ezcontour", "ezcontourf", "ezmesh", "ezmeshc",
+        "ezplot", "ezpolar", "ezsurf", "ezsurfc", "factor",
+        "factorial", "fail", "fcdf", "feather", "fftconv",
+        "fftfilt", "fftshift", "figure", "fileattrib",
+        "fileparts", "fill", "findall", "findobj",
+        "findstr", "finv", "flag", "flipdim", "fliplr",
+        "flipud", "fpdf", "fplot", "fractdiff", "freqz",
+        "freqz_plot", "frnd", "fsolve",
+        "f_test_regression", "ftp", "fullfile", "fzero",
+        "gamcdf", "gaminv", "gampdf", "gamrnd", "gca",
+        "gcbf", "gcbo", "gcf", "genvarname", "geocdf",
+        "geoinv", "geopdf", "geornd", "getfield", "ginput",
+        "glpk", "gls", "gplot", "gradient",
+        "graphics_toolkit", "gray", "grid", "griddata",
+        "griddatan", "gtext", "gunzip", "gzip", "hadamard",
+        "hamming", "hankel", "hanning", "hggroup",
+        "hidden", "hilb", "hist", "histc", "hold", "hot",
+        "hotelling_test", "housh", "hsv", "hurst",
+        "hygecdf", "hygeinv", "hygepdf", "hygernd",
+        "idivide", "ifftshift", "image", "imagesc",
+        "imfinfo", "imread", "imshow", "imwrite", "index",
+        "info", "inpolygon", "inputname", "interpft",
+        "interpn", "intersect", "invhilb", "iqr", "isa",
+        "isdefinite", "isdir", "is_duplicate_entry",
+        "isequal", "isequalwithequalnans", "isfigure",
+        "ishermitian", "ishghandle", "is_leap_year",
+        "isletter", "ismac", "ismember", "ispc", "isprime",
+        "isprop", "isscalar", "issquare", "isstrprop",
+        "issymmetric", "isunix", "is_valid_file_id",
+        "isvector", "jet", "kendall",
+        "kolmogorov_smirnov_cdf",
+        "kolmogorov_smirnov_test", "kruskal_wallis_test",
+        "krylov", "kurtosis", "laplace_cdf", "laplace_inv",
+        "laplace_pdf", "laplace_rnd", "legend", "legendre",
+        "license", "line", "linkprop", "list_primes",
+        "loadaudio", "loadobj", "logistic_cdf",
+        "logistic_inv", "logistic_pdf", "logistic_rnd",
+        "logit", "loglog", "loglogerr", "logm", "logncdf",
+        "logninv", "lognpdf", "lognrnd", "logspace",
+        "lookfor", "ls_command", "lsqnonneg", "magic",
+        "mahalanobis", "manova", "matlabroot",
+        "mcnemar_test", "mean", "meansq", "median", "menu",
+        "mesh", "meshc", "meshgrid", "meshz", "mexext",
+        "mget", "mkpp", "mode", "moment", "movefile",
+        "mpoles", "mput", "namelengthmax", "nargchk",
+        "nargoutchk", "nbincdf", "nbininv", "nbinpdf",
+        "nbinrnd", "nchoosek", "ndgrid", "newplot", "news",
+        "nonzeros", "normcdf", "normest", "norminv",
+        "normpdf", "normrnd", "now", "nthroot", "null",
+        "ocean", "ols", "onenormest", "optimget",
+        "optimset", "orderfields", "orient", "orth",
+        "pack", "pareto", "parseparams", "pascal", "patch",
+        "pathdef", "pcg", "pchip", "pcolor", "pcr",
+        "peaks", "periodogram", "perl", "perms", "pie",
+        "pink", "planerot", "playaudio", "plot",
+        "plotmatrix", "plotyy", "poisscdf", "poissinv",
+        "poisspdf", "poissrnd", "polar", "poly",
+        "polyaffine", "polyarea", "polyderiv", "polyfit",
+        "polygcd", "polyint", "polyout", "polyreduce",
+        "polyval", "polyvalm", "postpad", "powerset",
+        "ppder", "ppint", "ppjumps", "ppplot", "ppval",
+        "pqpnonneg", "prepad", "primes", "print",
+        "print_usage", "prism", "probit", "qp", "qqplot",
+        "quadcc", "quadgk", "quadl", "quadv", "quiver",
+        "qzhess", "rainbow", "randi", "range", "rank",
+        "ranks", "rat", "reallog", "realpow", "realsqrt",
+        "record", "rectangle_lw", "rectangle_sw",
+        "rectint", "refresh", "refreshdata",
+        "regexptranslate", "repmat", "residue", "ribbon",
+        "rindex", "roots", "rose", "rosser", "rotdim",
+        "rref", "run", "run_count", "rundemos", "run_test",
+        "runtests", "saveas", "saveaudio", "saveobj",
+        "savepath", "scatter", "secd", "semilogx",
+        "semilogxerr", "semilogy", "semilogyerr",
+        "setaudio", "setdiff", "setfield", "setxor",
+        "shading", "shift", "shiftdim", "sign_test",
+        "sinc", "sind", "sinetone", "sinewave", "skewness",
+        "slice", "sombrero", "sortrows", "spaugment",
+        "spconvert", "spdiags", "spearman", "spectral_adf",
+        "spectral_xdf", "specular", "speed", "spencer",
+        "speye", "spfun", "sphere", "spinmap", "spline",
+        "spones", "sprand", "sprandn", "sprandsym",
+        "spring", "spstats", "spy", "sqp", "stairs",
+        "statistics", "std", "stdnormal_cdf",
+        "stdnormal_inv", "stdnormal_pdf", "stdnormal_rnd",
+        "stem", "stft", "strcat", "strchr", "strjust",
+        "strmatch", "strread", "strsplit", "strtok",
+        "strtrim", "strtrunc", "structfun", "studentize",
+        "subplot", "subsindex", "subspace", "substr",
+        "substruct", "summer", "surf", "surface", "surfc",
+        "surfl", "surfnorm", "svds", "swapbytes",
+        "sylvester_matrix", "symvar", "synthesis", "table",
+        "tand", "tar", "tcdf", "tempdir", "tempname",
+        "test", "text", "textread", "textscan", "tinv",
+        "title", "toeplitz", "tpdf", "trace", "trapz",
+        "treelayout", "treeplot", "triangle_lw",
+        "triangle_sw", "tril", "trimesh", "triplequad",
+        "triplot", "trisurf", "triu", "trnd", "tsearchn",
+        "t_test", "t_test_regression", "type", "unidcdf",
+        "unidinv", "unidpdf", "unidrnd", "unifcdf",
+        "unifinv", "unifpdf", "unifrnd", "union", "unique",
+        "unix", "unmkpp", "unpack", "untabify", "untar",
+        "unwrap", "unzip", "u_test", "validatestring",
+        "vander", "var", "var_test", "vech", "ver",
+        "version", "view", "voronoi", "voronoin",
+        "waitforbuttonpress", "wavread", "wavwrite",
+        "wblcdf", "wblinv", "wblpdf", "wblrnd", "weekday",
+        "welch_test", "what", "white", "whitebg",
+        "wienrnd", "wilcoxon_test", "wilkinson", "winter",
+        "xlabel", "xlim", "ylabel", "yulewalker", "zip",
+        "zlabel", "z_test")
+
+    loadable_kw = (
+        "airy", "amd", "balance", "besselh", "besseli",
+        "besselj", "besselk", "bessely", "bitpack",
+        "bsxfun", "builtin", "ccolamd", "cellfun",
+        "cellslices", "chol", "choldelete", "cholinsert",
+        "cholinv", "cholshift", "cholupdate", "colamd",
+        "colloc", "convhulln", "convn", "csymamd",
+        "cummax", "cummin", "daspk", "daspk_options",
+        "dasrt", "dasrt_options", "dassl", "dassl_options",
+        "dbclear", "dbdown", "dbstack", "dbstatus",
+        "dbstop", "dbtype", "dbup", "dbwhere", "det",
+        "dlmread", "dmperm", "dot", "eig", "eigs",
+        "endgrent", "endpwent", "etree", "fft", "fftn",
+        "fftw", "filter", "find", "full", "gcd",
+        "getgrent", "getgrgid", "getgrnam", "getpwent",
+        "getpwnam", "getpwuid", "getrusage", "givens",
+        "gmtime", "gnuplot_binary", "hess", "ifft",
+        "ifftn", "inv", "isdebugmode", "issparse", "kron",
+        "localtime", "lookup", "lsode", "lsode_options",
+        "lu", "luinc", "luupdate", "matrix_type", "max",
+        "min", "mktime", "pinv", "qr", "qrdelete",
+        "qrinsert", "qrshift", "qrupdate", "quad",
+        "quad_options", "qz", "rand", "rande", "randg",
+        "randn", "randp", "randperm", "rcond", "regexp",
+        "regexpi", "regexprep", "schur", "setgrent",
+        "setpwent", "sort", "spalloc", "sparse", "spparms",
+        "sprank", "sqrtm", "strfind", "strftime",
+        "strptime", "strrep", "svd", "svd_driver", "syl",
+        "symamd", "symbfact", "symrcm", "time", "tsearch",
+        "typecast", "urlread", "urlwrite")
+
+    mapping_kw = (
+        "abs", "acos", "acosh", "acot", "acoth", "acsc",
+        "acsch", "angle", "arg", "asec", "asech", "asin",
+        "asinh", "atan", "atanh", "beta", "betainc",
+        "betaln", "bincoeff", "cbrt", "ceil", "conj", "cos",
+        "cosh", "cot", "coth", "csc", "csch", "erf", "erfc",
+        "erfcx", "erfinv", "exp", "finite", "fix", "floor",
+        "fmod", "gamma", "gammainc", "gammaln", "imag",
+        "isalnum", "isalpha", "isascii", "iscntrl",
+        "isdigit", "isfinite", "isgraph", "isinf",
+        "islower", "isna", "isnan", "isprint", "ispunct",
+        "isspace", "isupper", "isxdigit", "lcm", "lgamma",
+        "log", "lower", "mod", "real", "rem", "round",
+        "roundb", "sec", "sech", "sign", "sin", "sinh",
+        "sqrt", "tan", "tanh", "toascii", "tolower", "xor")
+
+    builtin_consts = (
+        "EDITOR", "EXEC_PATH", "I", "IMAGE_PATH", "NA",
+        "OCTAVE_HOME", "OCTAVE_VERSION", "PAGER",
+        "PAGER_FLAGS", "SEEK_CUR", "SEEK_END", "SEEK_SET",
+        "SIG", "S_ISBLK", "S_ISCHR", "S_ISDIR", "S_ISFIFO",
+        "S_ISLNK", "S_ISREG", "S_ISSOCK", "WCONTINUE",
+        "WCOREDUMP", "WEXITSTATUS", "WIFCONTINUED",
+        "WIFEXITED", "WIFSIGNALED", "WIFSTOPPED", "WNOHANG",
+        "WSTOPSIG", "WTERMSIG", "WUNTRACED")
+
+    tokens = {
+        'root': [
+            (r'%\{\s*\n', Comment.Multiline, 'percentblockcomment'),
+            (r'#\{\s*\n', Comment.Multiline, 'hashblockcomment'),
+            (r'[%#].*$', Comment),
+            (r'^\s*function\b', Keyword, 'deffunc'),
+
+            # from 'iskeyword' on hg changeset 8cc154f45e37
+            (words((
+                '__FILE__', '__LINE__', 'break', 'case', 'catch', 'classdef',
+                'continue', 'do', 'else', 'elseif', 'end', 'end_try_catch',
+                'end_unwind_protect', 'endclassdef', 'endevents', 'endfor',
+                'endfunction', 'endif', 'endmethods', 'endproperties', 'endswitch',
+                'endwhile', 'events', 'for', 'function', 'get', 'global', 'if',
+                'methods', 'otherwise', 'persistent', 'properties', 'return',
+                'set', 'static', 'switch', 'try', 'until', 'unwind_protect',
+                'unwind_protect_cleanup', 'while'), suffix=r'\b'),
+             Keyword),
+
+            (words(builtin_kw + command_kw + function_kw + loadable_kw + mapping_kw,
+                   suffix=r'\b'),  Name.Builtin),
+
+            (words(builtin_consts, suffix=r'\b'), Name.Constant),
+
+            # operators in Octave but not Matlab:
+            (r'-=|!=|!|/=|--', Operator),
+            # operators:
+            (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
+            # operators in Octave but not Matlab requiring escape for re:
+            (r'\*=|\+=|\^=|\/=|\\=|\*\*|\+\+|\.\*\*', Operator),
+            # operators requiring escape for re:
+            (r'\.\*|\*|\+|\.\^|\^|\.\\|\.\/|\/|\\', Operator),
+
+
+            # punctuation:
+            (r'[\[\](){}:@.,]', Punctuation),
+            (r'=|:|;', Punctuation),
+
+            (r'"[^"]*"', String),
+
+            (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+            (r'\d+', Number.Integer),
+
+            # quote can be transpose, instead of string:
+            # (not great, but handles common cases...)
+            (r'(?<=[\w)\].])\'+', Operator),
+            (r'(?|<=|>=|&&|&|~|\|\|?', Operator),
+            # operators requiring escape for re:
+            (r'\.\*|\*|\+|\.\^|\^|\.\\|\.\/|\/|\\', Operator),
+
+            # punctuation:
+            (r'[\[\](){}@.,=:;]+', Punctuation),
+
+            (r'"[^"]*"', String),
+
+            # quote can be transpose, instead of string:
+            # (not great, but handles common cases...)
+            (r'(?<=[\w)\].])\'+', Operator),
+            (r'(?', r'<', r'|', r'!', r"'")
+
+    operator_words = ('and', 'or', 'not')
+
+    tokens = {
+        'root': [
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'\(|\)|\[|\]|\{|\}', Punctuation),
+            (r'[,;$]', Punctuation),
+            (words (constants), Name.Constant),
+            (words (keywords), Keyword),
+            (words (operators), Operator),
+            (words (operator_words), Operator.Word),
+            (r'''(?x)
+              ((?:[a-zA-Z_#][\w#]*|`[^`]*`)
+              (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*)(\s*)([(])''',
+             bygroups(Name.Function, Text.Whitespace, Punctuation)),
+            (r'''(?x)
+              (?:[a-zA-Z_#%][\w#%]*|`[^`]*`)
+              (?:::[a-zA-Z_#%][\w#%]*|`[^`]*`)*''', Name.Variable),
+            (r'[-+]?(\d*\.\d+([bdefls][-+]?\d+)?|\d+(\.\d*)?[bdefls][-+]?\d+)', Number.Float),
+            (r'[-+]?\d+', Number.Integer),
+            (r'\s+', Text.Whitespace),
+            (r'.', Text)
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ]
+    }
+
+    def analyse_text (text):
+        strength = 0.0
+        # Input expression terminator.
+        if re.search (r'\$\s*$', text, re.MULTILINE):
+            strength += 0.05
+        # Function definition operator.
+        if ':=' in text:
+            strength += 0.02
+        return strength
diff --git a/lib/pygments/lexers/meson.py b/lib/pygments/lexers/meson.py
new file mode 100644
index 0000000..6f2c6da
--- /dev/null
+++ b/lib/pygments/lexers/meson.py
@@ -0,0 +1,139 @@
+"""
+    pygments.lexers.meson
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Pygments lexer for the Meson build system
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Name, Number, Punctuation, Operator, \
+    Keyword, String, Whitespace
+
+__all__ = ['MesonLexer']
+
+
+class MesonLexer(RegexLexer):
+    """Meson language lexer.
+
+    The grammar definition use to transcribe the syntax was retrieved from
+    https://mesonbuild.com/Syntax.html#grammar for version 0.58.
+    Some of those definitions are improperly transcribed, so the Meson++
+    implementation was also checked: https://github.com/dcbaker/meson-plus-plus.
+    """
+
+    # TODO String interpolation @VARNAME@ inner matches
+    # TODO keyword_arg: value inner matches
+
+    name = 'Meson'
+    url = 'https://mesonbuild.com/'
+    aliases = ['meson', 'meson.build']
+    filenames = ['meson.build', 'meson_options.txt']
+    mimetypes = ['text/x-meson']
+    version_added = '2.10'
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment),
+            (r"'''.*'''", String.Single),
+            (r'[1-9][0-9]*', Number.Integer),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[a-fA-F0-9]+', Number.Hex),
+            include('string'),
+            include('keywords'),
+            include('expr'),
+            (r'[a-zA-Z_][a-zA-Z_0-9]*', Name),
+            (r'\s+', Whitespace),
+        ],
+        'string': [
+            (r"[']{3}([']{0,2}([^\\']|\\(.|\n)))*[']{3}", String),
+            (r"'.*?(?`_.
+    """
+
+    name = "MCFunction"
+    url = "https://minecraft.wiki/w/Commands"
+    aliases = ["mcfunction", "mcf"]
+    filenames = ["*.mcfunction"]
+    mimetypes = ["text/mcfunction"]
+    version_added = '2.12'
+
+    # Used to denotate the start of a block comment, borrowed from Github's mcfunction
+    _block_comment_prefix = "[>!]"
+
+    tokens = {
+        "root": [
+            include("names"),
+            include("comments"),
+            include("literals"),
+            include("whitespace"),
+            include("property"),
+            include("operators"),
+            include("selectors"),
+        ],
+
+        "names": [
+            # The start of a command (either beginning of line OR after the run keyword)
+            #  We don't encode a list of keywords since mods, plugins, or even pre-processors
+            #  may add new commands, so we have a 'close-enough' regex which catches them.
+            (r"^(\s*)([a-z_]+)", bygroups(Whitespace, Name.Builtin)),
+            (r"(?<=run)\s+[a-z_]+", Name.Builtin),
+
+            # UUID
+            (r"\b[0-9a-fA-F]+(?:-[0-9a-fA-F]+){4}\b", Name.Variable),
+            include("resource-name"),
+            # normal command names and scoreboards
+            #  there's no way to know the differences unfortuntely
+            (r"[A-Za-z_][\w.#%$]+", Keyword.Constant),
+            (r"[#%$][\w.#%$]+", Name.Variable.Magic),
+        ],
+
+        "resource-name": [
+            # resource names have to be lowercase
+            (r"#?[a-z_][a-z_.-]*:[a-z0-9_./-]+", Name.Function),
+            # similar to above except optional `:``
+            #  a `/` must be present "somewhere"
+            (r"#?[a-z0-9_\.\-]+\/[a-z0-9_\.\-\/]+", Name.Function),
+        ],
+
+        "whitespace": [
+            (r"\s+", Whitespace),
+        ],
+
+        "comments": [
+            (rf"^\s*(#{_block_comment_prefix})", Comment.Multiline,
+             ("comments.block", "comments.block.emphasized")),
+            (r"#.*$", Comment.Single),
+        ],
+        "comments.block": [
+            (rf"^\s*#{_block_comment_prefix}", Comment.Multiline,
+             "comments.block.emphasized"),
+            (r"^\s*#", Comment.Multiline, "comments.block.normal"),
+            default("#pop"),
+        ],
+        "comments.block.normal": [
+            include("comments.block.special"),
+            (r"\S+", Comment.Multiline),
+            (r"\n", Text, "#pop"),
+            include("whitespace"),
+        ],
+        "comments.block.emphasized": [
+            include("comments.block.special"),
+            (r"\S+", String.Doc),
+            (r"\n", Text, "#pop"),
+            include("whitespace"),
+        ],
+        "comments.block.special": [
+            # Params
+            (r"@\S+", Name.Decorator),
+
+            include("resource-name"),
+
+            # Scoreboard player names
+            (r"[#%$][\w.#%$]+", Name.Variable.Magic),
+        ],
+
+        "operators": [
+            (r"[\-~%^?!+*<>\\/|&=.]", Operator),
+        ],
+
+        "literals": [
+            (r"\.\.", Literal),
+            (r"(true|false)", Keyword.Pseudo),
+
+            # these are like unquoted strings and appear in many places
+            (r"[A-Za-z_]+", Name.Variable.Class),
+
+            (r"[0-7]b", Number.Byte),
+            (r"[+-]?\d*\.?\d+([eE]?[+-]?\d+)?[df]?\b", Number.Float),
+            (r"[+-]?\d+\b", Number.Integer),
+            (r'"', String.Double, "literals.string-double"),
+            (r"'", String.Single, "literals.string-single"),
+        ],
+        "literals.string-double": [
+            (r"\\.", String.Escape),
+            (r'[^\\"\n]+', String.Double),
+            (r'"', String.Double, "#pop"),
+        ],
+        "literals.string-single": [
+            (r"\\.", String.Escape),
+            (r"[^\\'\n]+", String.Single),
+            (r"'", String.Single, "#pop"),
+        ],
+
+        "selectors": [
+            (r"@[a-z]", Name.Variable),
+        ],
+
+
+        ## Generic Property Container
+        # There are several, differing instances where the language accepts
+        #  specific contained keys or contained key, value pairings.
+        #
+        # Property Maps:
+        # - Starts with either `[` or `{`
+        # - Key separated by `:` or `=`
+        # - Deliminated by `,`
+        #
+        # Property Lists:
+        # - Starts with `[`
+        # - Deliminated by `,`
+        #
+        # For simplicity, these patterns match a generic, nestable structure
+        #  which follow a key, value pattern. For normal lists, there's only keys.
+        # This allow some "illegal" structures, but we'll accept those for
+        #  sake of simplicity
+        #
+        # Examples:
+        # - `[facing=up, powered=true]` (blockstate)
+        # - `[name="hello world", nbt={key: 1b}]` (selector + nbt)
+        # - `[{"text": "value"}, "literal"]` (json)
+        ##
+        "property": [
+            # This state gets included in root and also several substates
+            # We do this to shortcut the starting of new properties
+            #  within other properties. Lists can have sublists and compounds
+            #  and values can start a new property (see the `difficult_1.txt`
+            #  snippet).
+            (r"\{", Punctuation, ("property.curly", "property.key")),
+            (r"\[", Punctuation, ("property.square", "property.key")),
+        ],
+        "property.curly": [
+            include("whitespace"),
+            include("property"),
+            (r"\}", Punctuation, "#pop"),
+        ],
+        "property.square": [
+            include("whitespace"),
+            include("property"),
+            (r"\]", Punctuation, "#pop"),
+
+            # lists can have sequences of items
+            (r",", Punctuation),
+        ],
+        "property.key": [
+            include("whitespace"),
+
+            # resource names (for advancements)
+            #  can omit `:` to default `minecraft:`
+            # must check if there is a future equals sign if `:` is in the name
+            (r"#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+(?=\s*\=)", Name.Attribute, "property.delimiter"),
+            (r"#?[a-z_][a-z0-9_\.\-/]+", Name.Attribute, "property.delimiter"),
+
+            # unquoted NBT key
+            (r"[A-Za-z_\-\+]+", Name.Attribute, "property.delimiter"),
+
+            # quoted JSON or NBT key
+            (r'"', Name.Attribute, "property.delimiter", "literals.string-double"),
+            (r"'", Name.Attribute, "property.delimiter", "literals.string-single"),
+
+            # index for a list
+            (r"-?\d+", Number.Integer, "property.delimiter"),
+
+            default("#pop"),
+        ],
+        "property.key.string-double": [
+            (r"\\.", String.Escape),
+            (r'[^\\"\n]+', Name.Attribute),
+            (r'"', Name.Attribute, "#pop"),
+        ],
+        "property.key.string-single": [
+            (r"\\.", String.Escape),
+            (r"[^\\'\n]+", Name.Attribute),
+            (r"'", Name.Attribute, "#pop"),
+        ],
+        "property.delimiter": [
+            include("whitespace"),
+
+            (r"[:=]!?", Punctuation, "property.value"),
+            (r",", Punctuation),
+
+            default("#pop"),
+        ],
+        "property.value": [
+            include("whitespace"),
+
+            # unquoted resource names are valid literals here
+            (r"#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+", Name.Tag),
+            (r"#?[a-z_][a-z0-9_\.\-/]+", Name.Tag),
+
+            include("literals"),
+            include("property"),
+
+            default("#pop"),
+        ],
+    }
+
+
+class MCSchemaLexer(RegexLexer):
+    """Lexer for Minecraft Add-ons data Schemas, an interface structure standard used in Minecraft
+    """
+
+    name = 'MCSchema'
+    url = 'https://learn.microsoft.com/en-us/minecraft/creator/reference/content/schemasreference/'
+    aliases = ['mcschema']
+    filenames = ['*.mcschema']
+    mimetypes = ['text/mcschema']
+    version_added = '2.14'
+
+    tokens = {
+        'commentsandwhitespace': [
+            (r'\s+', Whitespace),
+            (r'//.*?$', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
+        ],
+        'slashstartsregex': [
+            include('commentsandwhitespace'),
+            (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+             r'([gimuysd]+\b|\B)', String.Regex, '#pop'),
+            (r'(?=/)', Text, ('#pop', 'badregex')),
+            default('#pop')
+        ],
+        'badregex': [
+            (r'\n', Whitespace, '#pop')
+        ],
+        'singlestring': [
+            (r'\\.', String.Escape),
+            (r"'", String.Single, '#pop'),
+            (r"[^\\']+", String.Single),
+        ],
+        'doublestring': [
+            (r'\\.', String.Escape),
+            (r'"', String.Double, '#pop'),
+            (r'[^\\"]+', String.Double),
+        ],
+        'root': [
+            (r'^(?=\s|/|', Comment, '#pop'),
+            (r'[^\-]+|-', Comment),
+        ],
+    }
+
+
+class ReasonLexer(RegexLexer):
+    """
+    For the ReasonML language.
+    """
+
+    name = 'ReasonML'
+    url = 'https://reasonml.github.io/'
+    aliases = ['reasonml', 'reason']
+    filenames = ['*.re', '*.rei']
+    mimetypes = ['text/x-reasonml']
+    version_added = '2.6'
+
+    keywords = (
+        'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done', 'downto',
+        'else', 'end', 'exception', 'external', 'false', 'for', 'fun', 'esfun',
+        'function', 'functor', 'if', 'in', 'include', 'inherit', 'initializer', 'lazy',
+        'let', 'switch', 'module', 'pub', 'mutable', 'new', 'nonrec', 'object', 'of',
+        'open', 'pri', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try',
+        'type', 'val', 'virtual', 'when', 'while', 'with',
+    )
+    keyopts = (
+        '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-',
+        r'-\.', '=>', r'\.', r'\.\.', r'\.\.\.', ':', '::', ':=', ':>', ';', ';;', '<',
+        '<-', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>',
+        r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|\|', r'\|]', r'\}', '~'
+    )
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    word_operators = ('and', 'asr', 'land', 'lor', 'lsl', 'lsr', 'lxor', 'mod', 'or')
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Class),
+            (r'//.*?\n', Comment.Single),
+            (r'\/\*(?!/)', Comment.Multiline, 'comment'),
+            (r'\b({})\b'.format('|'.join(keywords)), Keyword),
+            (r'({})'.format('|'.join(keyopts[::-1])), Operator.Word),
+            (rf'({infix_syms}|{prefix_syms})?{operators}', Operator),
+            (r'\b({})\b'.format('|'.join(word_operators)), Operator.Word),
+            (r'\b({})\b'.format('|'.join(primitives)), Keyword.Type),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Bin),
+            (r'\d[\d_]*', Number.Integer),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword),
+
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'\/\*', Comment.Multiline, '#push'),
+            (r'\*\/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name.Class, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+class FStarLexer(RegexLexer):
+    """
+    For the F* language.
+    """
+
+    name = 'FStar'
+    url = 'https://www.fstar-lang.org/'
+    aliases = ['fstar']
+    filenames = ['*.fst', '*.fsti']
+    mimetypes = ['text/x-fstar']
+    version_added = '2.7'
+
+    keywords = (
+        'abstract', 'attributes', 'noeq', 'unopteq', 'and'
+        'begin', 'by', 'default', 'effect', 'else', 'end', 'ensures',
+        'exception', 'exists', 'false', 'forall', 'fun', 'function', 'if',
+        'in', 'include', 'inline', 'inline_for_extraction', 'irreducible',
+        'logic', 'match', 'module', 'mutable', 'new', 'new_effect', 'noextract',
+        'of', 'open', 'opaque', 'private', 'range_of', 'reifiable',
+        'reify', 'reflectable', 'requires', 'set_range_of', 'sub_effect',
+        'synth', 'then', 'total', 'true', 'try', 'type', 'unfold', 'unfoldable',
+        'val', 'when', 'with', 'not'
+    )
+    decl_keywords = ('let', 'rec')
+    assume_keywords = ('assume', 'admit', 'assert', 'calc')
+    keyopts = (
+        r'~', r'-', r'/\\', r'\\/', r'<:', r'<@', r'\(\|', r'\|\)', r'#', r'u#',
+        r'&', r'\(', r'\)', r'\(\)', r',', r'~>', r'->', r'<-', r'<--', r'<==>',
+        r'==>', r'\.', r'\?', r'\?\.', r'\.\[', r'\.\(', r'\.\(\|', r'\.\[\|',
+        r'\{:pattern', r':', r'::', r':=', r';', r';;', r'=', r'%\[', r'!\{',
+        r'\[', r'\[@', r'\[\|', r'\|>', r'\]', r'\|\]', r'\{', r'\|', r'\}', r'\$'
+    )
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|False|True|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Class),
+            (r'\(\*(?![)])', Comment, 'comment'),
+            (r'\/\/.+$', Comment),
+            (r'\b({})\b'.format('|'.join(keywords)), Keyword),
+            (r'\b({})\b'.format('|'.join(assume_keywords)), Name.Exception),
+            (r'\b({})\b'.format('|'.join(decl_keywords)), Keyword.Declaration),
+            (r'({})'.format('|'.join(keyopts[::-1])), Operator),
+            (rf'({infix_syms}|{prefix_syms})?{operators}', Operator),
+            (r'\b({})\b'.format('|'.join(primitives)), Keyword.Type),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Bin),
+            (r'\d[\d_]*', Number.Integer),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword),  # a stray quote is another syntax element
+            (r"\`([\w\'.]+)\`", Operator.Word),  # for infix applications
+            (r"\`", Keyword),  # for quoting
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^(*)]+', Comment),
+            (r'\(\*', Comment, '#push'),
+            (r'\*\)', Comment, '#pop'),
+            (r'[(*)]', Comment),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name.Class, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
diff --git a/lib/pygments/lexers/modeling.py b/lib/pygments/lexers/modeling.py
new file mode 100644
index 0000000..d681e7f
--- /dev/null
+++ b/lib/pygments/lexers/modeling.py
@@ -0,0 +1,366 @@
+"""
+    pygments.lexers.modeling
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for modeling languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers import _stan_builtins
+
+__all__ = ['ModelicaLexer', 'BugsLexer', 'JagsLexer', 'StanLexer']
+
+
+class ModelicaLexer(RegexLexer):
+    """
+    For Modelica source code.
+    """
+    name = 'Modelica'
+    url = 'http://www.modelica.org/'
+    aliases = ['modelica']
+    filenames = ['*.mo']
+    mimetypes = ['text/x-modelica']
+    version_added = '1.1'
+
+    flags = re.DOTALL | re.MULTILINE
+
+    _name = r"(?:'(?:[^\\']|\\.)+'|[a-zA-Z_]\w*)"
+
+    tokens = {
+        'whitespace': [
+            (r'[\s\ufeff]+', Text),
+            (r'//[^\n]*\n?', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
+        ],
+        'root': [
+            include('whitespace'),
+            (r'"', String.Double, 'string'),
+            (r'[()\[\]{},;]+', Punctuation),
+            (r'\.?[*^/+-]|\.|<>|[<>:=]=?', Operator),
+            (r'\d+(\.?\d*[eE][-+]?\d+|\.\d*)', Number.Float),
+            (r'\d+', Number.Integer),
+            (r'(abs|acos|actualStream|array|asin|assert|AssertionLevel|atan|'
+             r'atan2|backSample|Boolean|cardinality|cat|ceil|change|Clock|'
+             r'Connections|cos|cosh|cross|delay|diagonal|div|edge|exp|'
+             r'ExternalObject|fill|floor|getInstanceName|hold|homotopy|'
+             r'identity|inStream|integer|Integer|interval|inverse|isPresent|'
+             r'linspace|log|log10|matrix|max|min|mod|ndims|noClock|noEvent|'
+             r'ones|outerProduct|pre|previous|product|Real|reinit|rem|rooted|'
+             r'sample|scalar|semiLinear|shiftSample|sign|sin|sinh|size|skew|'
+             r'smooth|spatialDistribution|sqrt|StateSelect|String|subSample|'
+             r'sum|superSample|symmetric|tan|tanh|terminal|terminate|time|'
+             r'transpose|vector|zeros)\b', Name.Builtin),
+            (r'(algorithm|annotation|break|connect|constant|constrainedby|der|'
+             r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|'
+             r'equation|exit|expandable|extends|external|firstTick|final|flow|for|if|'
+             r'import|impure|in|initial|inner|input|interval|loop|nondiscrete|outer|'
+             r'output|parameter|partial|protected|public|pure|redeclare|'
+             r'replaceable|return|stream|then|when|while)\b',
+             Keyword.Reserved),
+            (r'(and|not|or)\b', Operator.Word),
+            (r'(block|class|connector|end|function|model|operator|package|'
+             r'record|type)\b', Keyword.Reserved, 'class'),
+            (r'(false|true)\b', Keyword.Constant),
+            (r'within\b', Keyword.Reserved, 'package-prefix'),
+            (_name, Name)
+        ],
+        'class': [
+            include('whitespace'),
+            (r'(function|record)\b', Keyword.Reserved),
+            (r'(if|for|when|while)\b', Keyword.Reserved, '#pop'),
+            (_name, Name.Class, '#pop'),
+            default('#pop')
+        ],
+        'package-prefix': [
+            include('whitespace'),
+            (_name, Name.Namespace, '#pop'),
+            default('#pop')
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'\\[\'"?\\abfnrtv]', String.Escape),
+            (r'(?i)<\s*html\s*>([^\\"]|\\.)+?(<\s*/\s*html\s*>|(?="))',
+             using(HtmlLexer)),
+            (r'<|\\?[^"\\<]+', String.Double)
+        ]
+    }
+
+
+class BugsLexer(RegexLexer):
+    """
+    Pygments Lexer for OpenBugs and WinBugs
+    models.
+    """
+
+    name = 'BUGS'
+    aliases = ['bugs', 'winbugs', 'openbugs']
+    filenames = ['*.bug']
+    url = 'https://www.mrc-bsu.cam.ac.uk/software/bugs/openbugs'
+    version_added = '1.6'
+
+    _FUNCTIONS = (
+        # Scalar functions
+        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
+        'cloglog', 'cos', 'cosh', 'cumulative', 'cut', 'density', 'deviance',
+        'equals', 'expr', 'gammap', 'ilogit', 'icloglog', 'integral', 'log',
+        'logfact', 'loggam', 'logit', 'max', 'min', 'phi', 'post.p.value',
+        'pow', 'prior.p.value', 'probit', 'replicate.post', 'replicate.prior',
+        'round', 'sin', 'sinh', 'solution', 'sqrt', 'step', 'tan', 'tanh',
+        'trunc',
+        # Vector functions
+        'inprod', 'interp.lin', 'inverse', 'logdet', 'mean', 'eigen.vals',
+        'ode', 'prod', 'p.valueM', 'rank', 'ranked', 'replicate.postM',
+        'sd', 'sort', 'sum',
+        # Special
+        'D', 'I', 'F', 'T', 'C')
+    """ OpenBUGS built-in functions
+
+    From http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAII
+
+    This also includes
+
+    - T, C, I : Truncation and censoring.
+      ``T`` and ``C`` are in OpenBUGS. ``I`` in WinBUGS.
+    - D : ODE
+    - F : Functional http://www.openbugs.info/Examples/Functionals.html
+
+    """
+
+    _DISTRIBUTIONS = ('dbern', 'dbin', 'dcat', 'dnegbin', 'dpois',
+                      'dhyper', 'dbeta', 'dchisqr', 'ddexp', 'dexp',
+                      'dflat', 'dgamma', 'dgev', 'df', 'dggamma', 'dgpar',
+                      'dloglik', 'dlnorm', 'dlogis', 'dnorm', 'dpar',
+                      'dt', 'dunif', 'dweib', 'dmulti', 'ddirch', 'dmnorm',
+                      'dmt', 'dwish')
+    """ OpenBUGS built-in distributions
+
+    Functions from
+    http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAI
+    """
+
+    tokens = {
+        'whitespace': [
+            (r"\s+", Text),
+        ],
+        'comments': [
+            # Comments
+            (r'#.*$', Comment.Single),
+        ],
+        'root': [
+            # Comments
+            include('comments'),
+            include('whitespace'),
+            # Block start
+            (r'(model)(\s+)(\{)',
+             bygroups(Keyword.Namespace, Text, Punctuation)),
+            # Reserved Words
+            (r'(for|in)(?![\w.])', Keyword.Reserved),
+            # Built-in Functions
+            (r'({})(?=\s*\()'.format(r'|'.join(_FUNCTIONS + _DISTRIBUTIONS)),
+             Name.Builtin),
+            # Regular variable names
+            (r'[A-Za-z][\w.]*', Name),
+            # Number Literals
+            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
+            # Punctuation
+            (r'\[|\]|\(|\)|:|,|;', Punctuation),
+            # Assignment operators
+            # SLexer makes these tokens Operators.
+            (r'<-|~', Operator),
+            # Infix and prefix operators
+            (r'\+|-|\*|/', Operator),
+            # Block
+            (r'[{}]', Punctuation),
+        ]
+    }
+
+    def analyse_text(text):
+        if re.search(r"^\s*model\s*{", text, re.M):
+            return 0.7
+        else:
+            return 0.0
+
+
+class JagsLexer(RegexLexer):
+    """
+    Pygments Lexer for JAGS.
+    """
+
+    name = 'JAGS'
+    aliases = ['jags']
+    filenames = ['*.jag', '*.bug']
+    url = 'https://mcmc-jags.sourceforge.io'
+    version_added = '1.6'
+
+    # JAGS
+    _FUNCTIONS = (
+        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
+        'cos', 'cosh', 'cloglog',
+        'equals', 'exp', 'icloglog', 'ifelse', 'ilogit', 'log', 'logfact',
+        'loggam', 'logit', 'phi', 'pow', 'probit', 'round', 'sin', 'sinh',
+        'sqrt', 'step', 'tan', 'tanh', 'trunc', 'inprod', 'interp.lin',
+        'logdet', 'max', 'mean', 'min', 'prod', 'sum', 'sd', 'inverse',
+        'rank', 'sort', 't', 'acos', 'acosh', 'asin', 'asinh', 'atan',
+        # Truncation/Censoring (should I include)
+        'T', 'I')
+    # Distributions with density, probability and quartile functions
+    _DISTRIBUTIONS = tuple(f'[dpq]{x}' for x in
+                           ('bern', 'beta', 'dchiqsqr', 'ddexp', 'dexp',
+                            'df', 'gamma', 'gen.gamma', 'logis', 'lnorm',
+                            'negbin', 'nchisqr', 'norm', 'par', 'pois', 'weib'))
+    # Other distributions without density and probability
+    _OTHER_DISTRIBUTIONS = (
+        'dt', 'dunif', 'dbetabin', 'dbern', 'dbin', 'dcat', 'dhyper',
+        'ddirch', 'dmnorm', 'dwish', 'dmt', 'dmulti', 'dbinom', 'dchisq',
+        'dnbinom', 'dweibull', 'ddirich')
+
+    tokens = {
+        'whitespace': [
+            (r"\s+", Text),
+        ],
+        'names': [
+            # Regular variable names
+            (r'[a-zA-Z][\w.]*\b', Name),
+        ],
+        'comments': [
+            # do not use stateful comments
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            # Comments
+            (r'#.*$', Comment.Single),
+        ],
+        'root': [
+            # Comments
+            include('comments'),
+            include('whitespace'),
+            # Block start
+            (r'(model|data)(\s+)(\{)',
+             bygroups(Keyword.Namespace, Text, Punctuation)),
+            (r'var(?![\w.])', Keyword.Declaration),
+            # Reserved Words
+            (r'(for|in)(?![\w.])', Keyword.Reserved),
+            # Builtins
+            # Need to use lookahead because . is a valid char
+            (r'({})(?=\s*\()'.format(r'|'.join(_FUNCTIONS
+                                          + _DISTRIBUTIONS
+                                          + _OTHER_DISTRIBUTIONS)),
+             Name.Builtin),
+            # Names
+            include('names'),
+            # Number Literals
+            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
+            (r'\[|\]|\(|\)|:|,|;', Punctuation),
+            # Assignment operators
+            (r'<-|~', Operator),
+            # # JAGS includes many more than OpenBUGS
+            (r'\+|-|\*|\/|\|\|[&]{2}|[<>=]=?|\^|%.*?%', Operator),
+            (r'[{}]', Punctuation),
+        ]
+    }
+
+    def analyse_text(text):
+        if re.search(r'^\s*model\s*\{', text, re.M):
+            if re.search(r'^\s*data\s*\{', text, re.M):
+                return 0.9
+            elif re.search(r'^\s*var', text, re.M):
+                return 0.9
+            else:
+                return 0.3
+        else:
+            return 0
+
+
+class StanLexer(RegexLexer):
+    """Pygments Lexer for Stan models.
+
+    The Stan modeling language is specified in the *Stan Modeling Language
+    User's Guide and Reference Manual, v2.17.0*,
+    `pdf `__.
+    """
+
+    name = 'Stan'
+    aliases = ['stan']
+    filenames = ['*.stan']
+    url = 'https://mc-stan.org'
+    version_added = '1.6'
+
+    tokens = {
+        'whitespace': [
+            (r"\s+", Text),
+        ],
+        'comments': [
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            # Comments
+            (r'(//|#).*$', Comment.Single),
+        ],
+        'root': [
+            (r'"[^"]*"', String),
+            # Comments
+            include('comments'),
+            # block start
+            include('whitespace'),
+            # Block start
+            (r'({})(\s*)(\{{)'.format(r'|'.join(('functions', 'data', r'transformed\s+?data',
+                        'parameters', r'transformed\s+parameters',
+                        'model', r'generated\s+quantities'))),
+             bygroups(Keyword.Namespace, Text, Punctuation)),
+            # target keyword
+            (r'target\s*\+=', Keyword),
+            # Reserved Words
+            (r'({})\b'.format(r'|'.join(_stan_builtins.KEYWORDS)), Keyword),
+            # Truncation
+            (r'T(?=\s*\[)', Keyword),
+            # Data types
+            (r'({})\b'.format(r'|'.join(_stan_builtins.TYPES)), Keyword.Type),
+             # < should be punctuation, but elsewhere I can't tell if it is in
+             # a range constraint
+            (r'(<)(\s*)(upper|lower|offset|multiplier)(\s*)(=)',
+             bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)),
+            (r'(,)(\s*)(upper)(\s*)(=)',
+             bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)),
+            # Punctuation
+            (r"[;,\[\]()]", Punctuation),
+            # Builtin
+            (r'({})(?=\s*\()'.format('|'.join(_stan_builtins.FUNCTIONS)), Name.Builtin),
+            (r'(~)(\s*)({})(?=\s*\()'.format('|'.join(_stan_builtins.DISTRIBUTIONS)),
+                bygroups(Operator, Whitespace, Name.Builtin)),
+            # Special names ending in __, like lp__
+            (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo),
+            (r'({})\b'.format(r'|'.join(_stan_builtins.RESERVED)), Keyword.Reserved),
+            # user-defined functions
+            (r'[A-Za-z]\w*(?=\s*\()]', Name.Function),
+            # Imaginary Literals
+            (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?i', Number.Float),
+            (r'\.[0-9]+([eE][+-]?[0-9]+)?i', Number.Float),
+            (r'[0-9]+i', Number.Float),
+            # Real Literals
+            (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float),
+            # Integer Literals
+            (r'[0-9]+', Number.Integer),
+            # Regular variable names
+            (r'[A-Za-z]\w*\b', Name),
+            # Assignment operators
+            (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator),
+            # Infix, prefix and postfix operators (and = )
+            (r"\+|-|\.?\*|\.?/|\\|'|\.?\^|!=?|<=?|>=?|\|\||&&|%|\?|:|%/%|!", Operator),
+            # Block delimiters
+            (r'[{}]', Punctuation),
+            # Distribution |
+            (r'\|', Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        if re.search(r'^\s*parameters\s*\{', text, re.M):
+            return 1.0
+        else:
+            return 0.0
diff --git a/lib/pygments/lexers/modula2.py b/lib/pygments/lexers/modula2.py
new file mode 100644
index 0000000..713f472
--- /dev/null
+++ b/lib/pygments/lexers/modula2.py
@@ -0,0 +1,1579 @@
+"""
+    pygments.lexers.modula2
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Multi-Dialect Lexer for Modula-2.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.util import get_bool_opt, get_list_opt
+from pygments.token import Text, Comment, Operator, Keyword, Name, \
+    String, Number, Punctuation, Error
+
+__all__ = ['Modula2Lexer']
+
+
+# Multi-Dialect Modula-2 Lexer
+class Modula2Lexer(RegexLexer):
+    """
+    For Modula-2 source code.
+
+    The Modula-2 lexer supports several dialects.  By default, it operates in
+    fallback mode, recognising the *combined* literals, punctuation symbols
+    and operators of all supported dialects, and the *combined* reserved words
+    and builtins of PIM Modula-2, ISO Modula-2 and Modula-2 R10, while not
+    differentiating between library defined identifiers.
+
+    To select a specific dialect, a dialect option may be passed
+    or a dialect tag may be embedded into a source file.
+
+    Dialect Options:
+
+    `m2pim`
+        Select PIM Modula-2 dialect.
+    `m2iso`
+        Select ISO Modula-2 dialect.
+    `m2r10`
+        Select Modula-2 R10 dialect.
+    `objm2`
+        Select Objective Modula-2 dialect.
+
+    The PIM and ISO dialect options may be qualified with a language extension.
+
+    Language Extensions:
+
+    `+aglet`
+        Select Aglet Modula-2 extensions, available with m2iso.
+    `+gm2`
+        Select GNU Modula-2 extensions, available with m2pim.
+    `+p1`
+        Select p1 Modula-2 extensions, available with m2iso.
+    `+xds`
+        Select XDS Modula-2 extensions, available with m2iso.
+
+
+    Passing a Dialect Option via Unix Commandline Interface
+
+    Dialect options may be passed to the lexer using the `dialect` key.
+    Only one such option should be passed. If multiple dialect options are
+    passed, the first valid option is used, any subsequent options are ignored.
+
+    Examples:
+
+    `$ pygmentize -O full,dialect=m2iso -f html -o /path/to/output /path/to/input`
+        Use ISO dialect to render input to HTML output
+    `$ pygmentize -O full,dialect=m2iso+p1 -f rtf -o /path/to/output /path/to/input`
+        Use ISO dialect with p1 extensions to render input to RTF output
+
+
+    Embedding a Dialect Option within a source file
+
+    A dialect option may be embedded in a source file in form of a dialect
+    tag, a specially formatted comment that specifies a dialect option.
+
+    Dialect Tag EBNF::
+
+       dialectTag :
+           OpeningCommentDelim Prefix dialectOption ClosingCommentDelim ;
+
+       dialectOption :
+           'm2pim' | 'm2iso' | 'm2r10' | 'objm2' |
+           'm2iso+aglet' | 'm2pim+gm2' | 'm2iso+p1' | 'm2iso+xds' ;
+
+       Prefix : '!' ;
+
+       OpeningCommentDelim : '(*' ;
+
+       ClosingCommentDelim : '*)' ;
+
+    No whitespace is permitted between the tokens of a dialect tag.
+
+    In the event that a source file contains multiple dialect tags, the first
+    tag that contains a valid dialect option will be used and any subsequent
+    dialect tags will be ignored.  Ideally, a dialect tag should be placed
+    at the beginning of a source file.
+
+    An embedded dialect tag overrides a dialect option set via command line.
+
+    Examples:
+
+    ``(*!m2r10*) DEFINITION MODULE Foobar; ...``
+        Use Modula2 R10 dialect to render this source file.
+    ``(*!m2pim+gm2*) DEFINITION MODULE Bazbam; ...``
+        Use PIM dialect with GNU extensions to render this source file.
+
+
+    Algol Publication Mode:
+
+    In Algol publication mode, source text is rendered for publication of
+    algorithms in scientific papers and academic texts, following the format
+    of the Revised Algol-60 Language Report.  It is activated by passing
+    one of two corresponding styles as an option:
+
+    `algol`
+        render reserved words lowercase underline boldface
+        and builtins lowercase boldface italic
+    `algol_nu`
+        render reserved words lowercase boldface (no underlining)
+        and builtins lowercase boldface italic
+
+    The lexer automatically performs the required lowercase conversion when
+    this mode is activated.
+
+    Example:
+
+    ``$ pygmentize -O full,style=algol -f latex -o /path/to/output /path/to/input``
+        Render input file in Algol publication mode to LaTeX output.
+
+
+    Rendering Mode of First Class ADT Identifiers:
+
+    The rendering of standard library first class ADT identifiers is controlled
+    by option flag "treat_stdlib_adts_as_builtins".
+
+    When this option is turned on, standard library ADT identifiers are rendered
+    as builtins.  When it is turned off, they are rendered as ordinary library
+    identifiers.
+
+    `treat_stdlib_adts_as_builtins` (default: On)
+
+    The option is useful for dialects that support ADTs as first class objects
+    and provide ADTs in the standard library that would otherwise be built-in.
+
+    At present, only Modula-2 R10 supports library ADTs as first class objects
+    and therefore, no ADT identifiers are defined for any other dialects.
+
+    Example:
+
+    ``$ pygmentize -O full,dialect=m2r10,treat_stdlib_adts_as_builtins=Off ...``
+        Render standard library ADTs as ordinary library types.
+
+    .. versionchanged:: 2.1
+       Added multi-dialect support.
+    """
+    name = 'Modula-2'
+    url = 'http://www.modula2.org/'
+    aliases = ['modula2', 'm2']
+    filenames = ['*.def', '*.mod']
+    mimetypes = ['text/x-modula2']
+    version_added = '1.3'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    tokens = {
+        'whitespace': [
+            (r'\n+', Text),  # blank lines
+            (r'\s+', Text),  # whitespace
+        ],
+        'dialecttags': [
+            # PIM Dialect Tag
+            (r'\(\*!m2pim\*\)', Comment.Special),
+            # ISO Dialect Tag
+            (r'\(\*!m2iso\*\)', Comment.Special),
+            # M2R10 Dialect Tag
+            (r'\(\*!m2r10\*\)', Comment.Special),
+            # ObjM2 Dialect Tag
+            (r'\(\*!objm2\*\)', Comment.Special),
+            # Aglet Extensions Dialect Tag
+            (r'\(\*!m2iso\+aglet\*\)', Comment.Special),
+            # GNU Extensions Dialect Tag
+            (r'\(\*!m2pim\+gm2\*\)', Comment.Special),
+            # p1 Extensions Dialect Tag
+            (r'\(\*!m2iso\+p1\*\)', Comment.Special),
+            # XDS Extensions Dialect Tag
+            (r'\(\*!m2iso\+xds\*\)', Comment.Special),
+        ],
+        'identifiers': [
+            (r'([a-zA-Z_$][\w$]*)', Name),
+        ],
+        'prefixed_number_literals': [
+            #
+            # Base-2, whole number
+            (r'0b[01]+(\'[01]+)*', Number.Bin),
+            #
+            # Base-16, whole number
+            (r'0[ux][0-9A-F]+(\'[0-9A-F]+)*', Number.Hex),
+        ],
+        'plain_number_literals': [
+            #
+            # Base-10, real number with exponent
+            (r'[0-9]+(\'[0-9]+)*'  # integral part
+             r'\.[0-9]+(\'[0-9]+)*'  # fractional part
+             r'[eE][+-]?[0-9]+(\'[0-9]+)*',  # exponent
+             Number.Float),
+            #
+            # Base-10, real number without exponent
+            (r'[0-9]+(\'[0-9]+)*'  # integral part
+             r'\.[0-9]+(\'[0-9]+)*',  # fractional part
+             Number.Float),
+            #
+            # Base-10, whole number
+            (r'[0-9]+(\'[0-9]+)*', Number.Integer),
+        ],
+        'suffixed_number_literals': [
+            #
+            # Base-8, whole number
+            (r'[0-7]+B', Number.Oct),
+            #
+            # Base-8, character code
+            (r'[0-7]+C', Number.Oct),
+            #
+            # Base-16, number
+            (r'[0-9A-F]+H', Number.Hex),
+        ],
+        'string_literals': [
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+        ],
+        'digraph_operators': [
+            # Dot Product Operator
+            (r'\*\.', Operator),
+            # Array Concatenation Operator
+            (r'\+>', Operator),  # M2R10 + ObjM2
+            # Inequality Operator
+            (r'<>', Operator),  # ISO + PIM
+            # Less-Or-Equal, Subset
+            (r'<=', Operator),
+            # Greater-Or-Equal, Superset
+            (r'>=', Operator),
+            # Identity Operator
+            (r'==', Operator),  # M2R10 + ObjM2
+            # Type Conversion Operator
+            (r'::', Operator),  # M2R10 + ObjM2
+            # Assignment Symbol
+            (r':=', Operator),
+            # Postfix Increment Mutator
+            (r'\+\+', Operator),  # M2R10 + ObjM2
+            # Postfix Decrement Mutator
+            (r'--', Operator),  # M2R10 + ObjM2
+        ],
+        'unigraph_operators': [
+            # Arithmetic Operators
+            (r'[+-]', Operator),
+            (r'[*/]', Operator),
+            # ISO 80000-2 compliant Set Difference Operator
+            (r'\\', Operator),  # M2R10 + ObjM2
+            # Relational Operators
+            (r'[=#<>]', Operator),
+            # Dereferencing Operator
+            (r'\^', Operator),
+            # Dereferencing Operator Synonym
+            (r'@', Operator),  # ISO
+            # Logical AND Operator Synonym
+            (r'&', Operator),  # PIM + ISO
+            # Logical NOT Operator Synonym
+            (r'~', Operator),  # PIM + ISO
+            # Smalltalk Message Prefix
+            (r'`', Operator),  # ObjM2
+        ],
+        'digraph_punctuation': [
+            # Range Constructor
+            (r'\.\.', Punctuation),
+            # Opening Chevron Bracket
+            (r'<<', Punctuation),  # M2R10 + ISO
+            # Closing Chevron Bracket
+            (r'>>', Punctuation),  # M2R10 + ISO
+            # Blueprint Punctuation
+            (r'->', Punctuation),  # M2R10 + ISO
+            # Distinguish |# and # in M2 R10
+            (r'\|#', Punctuation),
+            # Distinguish ## and # in M2 R10
+            (r'##', Punctuation),
+            # Distinguish |* and * in M2 R10
+            (r'\|\*', Punctuation),
+        ],
+        'unigraph_punctuation': [
+            # Common Punctuation
+            (r'[()\[\]{},.:;|]', Punctuation),
+            # Case Label Separator Synonym
+            (r'!', Punctuation),  # ISO
+            # Blueprint Punctuation
+            (r'\?', Punctuation),  # M2R10 + ObjM2
+        ],
+        'comments': [
+            # Single Line Comment
+            (r'^//.*?\n', Comment.Single),  # M2R10 + ObjM2
+            # Block Comment
+            (r'\(\*([^$].*?)\*\)', Comment.Multiline),
+            # Template Block Comment
+            (r'/\*(.*?)\*/', Comment.Multiline),  # M2R10 + ObjM2
+        ],
+        'pragmas': [
+            # ISO Style Pragmas
+            (r'<\*.*?\*>', Comment.Preproc),  # ISO, M2R10 + ObjM2
+            # Pascal Style Pragmas
+            (r'\(\*\$.*?\*\)', Comment.Preproc),  # PIM
+        ],
+        'root': [
+            include('whitespace'),
+            include('dialecttags'),
+            include('pragmas'),
+            include('comments'),
+            include('identifiers'),
+            include('suffixed_number_literals'),  # PIM + ISO
+            include('prefixed_number_literals'),  # M2R10 + ObjM2
+            include('plain_number_literals'),
+            include('string_literals'),
+            include('digraph_punctuation'),
+            include('digraph_operators'),
+            include('unigraph_punctuation'),
+            include('unigraph_operators'),
+        ]
+    }
+
+#  C o m m o n   D a t a s e t s
+
+    # Common Reserved Words Dataset
+    common_reserved_words = (
+        # 37 common reserved words
+        'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', 'DIV',
+        'DO', 'ELSE', 'ELSIF', 'END', 'EXIT', 'FOR', 'FROM', 'IF',
+        'IMPLEMENTATION', 'IMPORT', 'IN', 'LOOP', 'MOD', 'MODULE', 'NOT',
+        'OF', 'OR', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN',
+        'SET', 'THEN', 'TO', 'TYPE', 'UNTIL', 'VAR', 'WHILE',
+    )
+
+    # Common Builtins Dataset
+    common_builtins = (
+        # 16 common builtins
+        'ABS', 'BOOLEAN', 'CARDINAL', 'CHAR', 'CHR', 'FALSE', 'INTEGER',
+        'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NIL', 'ODD', 'ORD', 'REAL',
+        'TRUE',
+    )
+
+    # Common Pseudo-Module Builtins Dataset
+    common_pseudo_builtins = (
+        # 4 common pseudo builtins
+        'ADDRESS', 'BYTE', 'WORD', 'ADR'
+    )
+
+#  P I M   M o d u l a - 2   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for PIM Modula-2
+    pim_lexemes_to_reject = (
+        '!', '`', '@', '$', '%', '?', '\\', '==', '++', '--', '::', '*.',
+        '+>', '->', '<<', '>>', '|#', '##',
+    )
+
+    # PIM Modula-2 Additional Reserved Words Dataset
+    pim_additional_reserved_words = (
+        # 3 additional reserved words
+        'EXPORT', 'QUALIFIED', 'WITH',
+    )
+
+    # PIM Modula-2 Additional Builtins Dataset
+    pim_additional_builtins = (
+        # 16 additional builtins
+        'BITSET', 'CAP', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT', 'HALT', 'HIGH',
+        'INC', 'INCL', 'NEW', 'NIL', 'PROC', 'SIZE', 'TRUNC', 'VAL',
+    )
+
+    # PIM Modula-2 Additional Pseudo-Module Builtins Dataset
+    pim_additional_pseudo_builtins = (
+        # 5 additional pseudo builtins
+        'SYSTEM', 'PROCESS', 'TSIZE', 'NEWPROCESS', 'TRANSFER',
+    )
+
+#  I S O   M o d u l a - 2   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for ISO Modula-2
+    iso_lexemes_to_reject = (
+        '`', '$', '%', '?', '\\', '==', '++', '--', '::', '*.', '+>', '->',
+        '<<', '>>', '|#', '##',
+    )
+
+    # ISO Modula-2 Additional Reserved Words Dataset
+    iso_additional_reserved_words = (
+        # 9 additional reserved words (ISO 10514-1)
+        'EXCEPT', 'EXPORT', 'FINALLY', 'FORWARD', 'PACKEDSET', 'QUALIFIED',
+        'REM', 'RETRY', 'WITH',
+        # 10 additional reserved words (ISO 10514-2 & ISO 10514-3)
+        'ABSTRACT', 'AS', 'CLASS', 'GUARD', 'INHERIT', 'OVERRIDE', 'READONLY',
+        'REVEAL', 'TRACED', 'UNSAFEGUARDED',
+    )
+
+    # ISO Modula-2 Additional Builtins Dataset
+    iso_additional_builtins = (
+        # 26 additional builtins (ISO 10514-1)
+        'BITSET', 'CAP', 'CMPLX', 'COMPLEX', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT',
+        'HALT', 'HIGH', 'IM', 'INC', 'INCL', 'INT', 'INTERRUPTIBLE',  'LENGTH',
+        'LFLOAT', 'LONGCOMPLEX', 'NEW', 'PROC', 'PROTECTION', 'RE', 'SIZE',
+        'TRUNC', 'UNINTERRUBTIBLE', 'VAL',
+        # 5 additional builtins (ISO 10514-2 & ISO 10514-3)
+        'CREATE', 'DESTROY', 'EMPTY', 'ISMEMBER', 'SELF',
+    )
+
+    # ISO Modula-2 Additional Pseudo-Module Builtins Dataset
+    iso_additional_pseudo_builtins = (
+        # 14 additional builtins (SYSTEM)
+        'SYSTEM', 'BITSPERLOC', 'LOCSPERBYTE', 'LOCSPERWORD', 'LOC',
+        'ADDADR', 'SUBADR', 'DIFADR', 'MAKEADR', 'ADR',
+        'ROTATE', 'SHIFT', 'CAST', 'TSIZE',
+        # 13 additional builtins (COROUTINES)
+        'COROUTINES', 'ATTACH', 'COROUTINE', 'CURRENT', 'DETACH', 'HANDLER',
+        'INTERRUPTSOURCE', 'IOTRANSFER', 'IsATTACHED', 'LISTEN',
+        'NEWCOROUTINE', 'PROT', 'TRANSFER',
+        # 9 additional builtins (EXCEPTIONS)
+        'EXCEPTIONS', 'AllocateSource', 'CurrentNumber', 'ExceptionNumber',
+        'ExceptionSource', 'GetMessage', 'IsCurrentSource',
+        'IsExceptionalExecution', 'RAISE',
+        # 3 additional builtins (TERMINATION)
+        'TERMINATION', 'IsTerminating', 'HasHalted',
+        # 4 additional builtins (M2EXCEPTION)
+        'M2EXCEPTION', 'M2Exceptions', 'M2Exception', 'IsM2Exception',
+        'indexException', 'rangeException', 'caseSelectException',
+        'invalidLocation', 'functionException', 'wholeValueException',
+        'wholeDivException', 'realValueException', 'realDivException',
+        'complexValueException', 'complexDivException', 'protException',
+        'sysException', 'coException', 'exException',
+    )
+
+#  M o d u l a - 2   R 1 0   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for Modula-2 R10
+    m2r10_lexemes_to_reject = (
+        '!', '`', '@', '$', '%', '&', '<>',
+    )
+
+    # Modula-2 R10 reserved words in addition to the common set
+    m2r10_additional_reserved_words = (
+        # 12 additional reserved words
+        'ALIAS', 'ARGLIST', 'BLUEPRINT', 'COPY', 'GENLIB', 'INDETERMINATE',
+        'NEW', 'NONE', 'OPAQUE', 'REFERENTIAL', 'RELEASE', 'RETAIN',
+        # 2 additional reserved words with symbolic assembly option
+        'ASM', 'REG',
+    )
+
+    # Modula-2 R10 builtins in addition to the common set
+    m2r10_additional_builtins = (
+        # 26 additional builtins
+        'CARDINAL', 'COUNT', 'EMPTY', 'EXISTS', 'INSERT', 'LENGTH', 'LONGCARD',
+        'OCTET', 'PTR', 'PRED', 'READ', 'READNEW', 'REMOVE', 'RETRIEVE', 'SORT',
+        'STORE', 'SUBSET', 'SUCC', 'TLIMIT', 'TMAX', 'TMIN', 'TRUE', 'TSIZE',
+        'UNICHAR', 'WRITE', 'WRITEF',
+    )
+
+    # Modula-2 R10 Additional Pseudo-Module Builtins Dataset
+    m2r10_additional_pseudo_builtins = (
+        # 13 additional builtins (TPROPERTIES)
+        'TPROPERTIES', 'PROPERTY', 'LITERAL', 'TPROPERTY', 'TLITERAL',
+        'TBUILTIN', 'TDYN', 'TREFC', 'TNIL', 'TBASE', 'TPRECISION',
+        'TMAXEXP', 'TMINEXP',
+        # 4 additional builtins (CONVERSION)
+        'CONVERSION', 'TSXFSIZE', 'SXF', 'VAL',
+        # 35 additional builtins (UNSAFE)
+        'UNSAFE', 'CAST', 'INTRINSIC', 'AVAIL', 'ADD', 'SUB', 'ADDC', 'SUBC',
+        'FETCHADD', 'FETCHSUB', 'SHL', 'SHR', 'ASHR', 'ROTL', 'ROTR', 'ROTLC',
+        'ROTRC', 'BWNOT', 'BWAND', 'BWOR', 'BWXOR', 'BWNAND', 'BWNOR',
+        'SETBIT', 'TESTBIT', 'LSBIT', 'MSBIT', 'CSBITS', 'BAIL', 'HALT',
+        'TODO', 'FFI', 'ADDR', 'VARGLIST', 'VARGC',
+        # 11 additional builtins (ATOMIC)
+        'ATOMIC', 'INTRINSIC', 'AVAIL', 'SWAP', 'CAS', 'INC', 'DEC', 'BWAND',
+        'BWNAND', 'BWOR', 'BWXOR',
+        # 7 additional builtins (COMPILER)
+        'COMPILER', 'DEBUG', 'MODNAME', 'PROCNAME', 'LINENUM', 'DEFAULT',
+        'HASH',
+        # 5 additional builtins (ASSEMBLER)
+        'ASSEMBLER', 'REGISTER', 'SETREG', 'GETREG', 'CODE',
+    )
+
+#  O b j e c t i v e   M o d u l a - 2   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for Objective Modula-2
+    objm2_lexemes_to_reject = (
+        '!', '$', '%', '&', '<>',
+    )
+
+    # Objective Modula-2 Extensions
+    # reserved words in addition to Modula-2 R10
+    objm2_additional_reserved_words = (
+        # 16 additional reserved words
+        'BYCOPY', 'BYREF', 'CLASS', 'CONTINUE', 'CRITICAL', 'INOUT', 'METHOD',
+        'ON', 'OPTIONAL', 'OUT', 'PRIVATE', 'PROTECTED', 'PROTOCOL', 'PUBLIC',
+        'SUPER', 'TRY',
+    )
+
+    # Objective Modula-2 Extensions
+    # builtins in addition to Modula-2 R10
+    objm2_additional_builtins = (
+        # 3 additional builtins
+        'OBJECT', 'NO', 'YES',
+    )
+
+    # Objective Modula-2 Extensions
+    # pseudo-module builtins in addition to Modula-2 R10
+    objm2_additional_pseudo_builtins = (
+        # None
+    )
+
+#  A g l e t   M o d u l a - 2   D a t a s e t s
+
+    # Aglet Extensions
+    # reserved words in addition to ISO Modula-2
+    aglet_additional_reserved_words = (
+        # None
+    )
+
+    # Aglet Extensions
+    # builtins in addition to ISO Modula-2
+    aglet_additional_builtins = (
+        # 9 additional builtins
+        'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16',
+        'CARDINAL32', 'INTEGER8', 'INTEGER16', 'INTEGER32',
+    )
+
+    # Aglet Modula-2 Extensions
+    # pseudo-module builtins in addition to ISO Modula-2
+    aglet_additional_pseudo_builtins = (
+        # None
+    )
+
+#  G N U   M o d u l a - 2   D a t a s e t s
+
+    # GNU Extensions
+    # reserved words in addition to PIM Modula-2
+    gm2_additional_reserved_words = (
+        # 10 additional reserved words
+        'ASM', '__ATTRIBUTE__', '__BUILTIN__', '__COLUMN__', '__DATE__',
+        '__FILE__', '__FUNCTION__', '__LINE__', '__MODULE__', 'VOLATILE',
+    )
+
+    # GNU Extensions
+    # builtins in addition to PIM Modula-2
+    gm2_additional_builtins = (
+        # 21 additional builtins
+        'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16',
+        'CARDINAL32', 'CARDINAL64', 'COMPLEX32', 'COMPLEX64', 'COMPLEX96',
+        'COMPLEX128', 'INTEGER8', 'INTEGER16', 'INTEGER32', 'INTEGER64',
+        'REAL8', 'REAL16', 'REAL32', 'REAL96', 'REAL128', 'THROW',
+    )
+
+    # GNU Extensions
+    # pseudo-module builtins in addition to PIM Modula-2
+    gm2_additional_pseudo_builtins = (
+        # None
+    )
+
+#  p 1   M o d u l a - 2   D a t a s e t s
+
+    # p1 Extensions
+    # reserved words in addition to ISO Modula-2
+    p1_additional_reserved_words = (
+        # None
+    )
+
+    # p1 Extensions
+    # builtins in addition to ISO Modula-2
+    p1_additional_builtins = (
+        # None
+    )
+
+    # p1 Modula-2 Extensions
+    # pseudo-module builtins in addition to ISO Modula-2
+    p1_additional_pseudo_builtins = (
+        # 1 additional builtin
+        'BCD',
+    )
+
+#  X D S   M o d u l a - 2   D a t a s e t s
+
+    # XDS Extensions
+    # reserved words in addition to ISO Modula-2
+    xds_additional_reserved_words = (
+        # 1 additional reserved word
+        'SEQ',
+    )
+
+    # XDS Extensions
+    # builtins in addition to ISO Modula-2
+    xds_additional_builtins = (
+        # 9 additional builtins
+        'ASH', 'ASSERT', 'DIFFADR_TYPE', 'ENTIER', 'INDEX', 'LEN',
+        'LONGCARD', 'SHORTCARD', 'SHORTINT',
+    )
+
+    # XDS Modula-2 Extensions
+    # pseudo-module builtins in addition to ISO Modula-2
+    xds_additional_pseudo_builtins = (
+        # 22 additional builtins (SYSTEM)
+        'PROCESS', 'NEWPROCESS', 'BOOL8', 'BOOL16', 'BOOL32', 'CARD8',
+        'CARD16', 'CARD32', 'INT8', 'INT16', 'INT32', 'REF', 'MOVE',
+        'FILL', 'GET', 'PUT', 'CC', 'int', 'unsigned', 'size_t', 'void'
+        # 3 additional builtins (COMPILER)
+        'COMPILER', 'OPTION', 'EQUATION'
+    )
+
+#  P I M   S t a n d a r d   L i b r a r y   D a t a s e t s
+
+    # PIM Modula-2 Standard Library Modules Dataset
+    pim_stdlib_module_identifiers = (
+        'Terminal', 'FileSystem', 'InOut', 'RealInOut', 'MathLib0', 'Storage',
+    )
+
+    # PIM Modula-2 Standard Library Types Dataset
+    pim_stdlib_type_identifiers = (
+        'Flag', 'FlagSet', 'Response', 'Command', 'Lock', 'Permission',
+        'MediumType', 'File', 'FileProc', 'DirectoryProc', 'FileCommand',
+        'DirectoryCommand',
+    )
+
+    # PIM Modula-2 Standard Library Procedures Dataset
+    pim_stdlib_proc_identifiers = (
+        'Read', 'BusyRead', 'ReadAgain', 'Write', 'WriteString', 'WriteLn',
+        'Create', 'Lookup', 'Close', 'Delete', 'Rename', 'SetRead', 'SetWrite',
+        'SetModify', 'SetOpen', 'Doio', 'SetPos', 'GetPos', 'Length', 'Reset',
+        'Again', 'ReadWord', 'WriteWord', 'ReadChar', 'WriteChar',
+        'CreateMedium', 'DeleteMedium', 'AssignName', 'DeassignName',
+        'ReadMedium', 'LookupMedium', 'OpenInput', 'OpenOutput', 'CloseInput',
+        'CloseOutput', 'ReadString', 'ReadInt', 'ReadCard', 'ReadWrd',
+        'WriteInt', 'WriteCard', 'WriteOct', 'WriteHex', 'WriteWrd',
+        'ReadReal', 'WriteReal', 'WriteFixPt', 'WriteRealOct', 'sqrt', 'exp',
+        'ln', 'sin', 'cos', 'arctan', 'entier', 'ALLOCATE', 'DEALLOCATE',
+    )
+
+    # PIM Modula-2 Standard Library Variables Dataset
+    pim_stdlib_var_identifiers = (
+        'Done', 'termCH', 'in', 'out'
+    )
+
+    # PIM Modula-2 Standard Library Constants Dataset
+    pim_stdlib_const_identifiers = (
+        'EOL',
+    )
+
+#  I S O   S t a n d a r d   L i b r a r y   D a t a s e t s
+
+    # ISO Modula-2 Standard Library Modules Dataset
+    iso_stdlib_module_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Types Dataset
+    iso_stdlib_type_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Procedures Dataset
+    iso_stdlib_proc_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Variables Dataset
+    iso_stdlib_var_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Constants Dataset
+    iso_stdlib_const_identifiers = (
+        # TO DO
+    )
+
+#  M 2   R 1 0   S t a n d a r d   L i b r a r y   D a t a s e t s
+
+    # Modula-2 R10 Standard Library ADTs Dataset
+    m2r10_stdlib_adt_identifiers = (
+        'BCD', 'LONGBCD', 'BITSET', 'SHORTBITSET', 'LONGBITSET',
+        'LONGLONGBITSET', 'COMPLEX', 'LONGCOMPLEX', 'SHORTCARD', 'LONGLONGCARD',
+        'SHORTINT', 'LONGLONGINT', 'POSINT', 'SHORTPOSINT', 'LONGPOSINT',
+        'LONGLONGPOSINT', 'BITSET8', 'BITSET16', 'BITSET32', 'BITSET64',
+        'BITSET128', 'BS8', 'BS16', 'BS32', 'BS64', 'BS128', 'CARDINAL8',
+        'CARDINAL16', 'CARDINAL32', 'CARDINAL64', 'CARDINAL128', 'CARD8',
+        'CARD16', 'CARD32', 'CARD64', 'CARD128', 'INTEGER8', 'INTEGER16',
+        'INTEGER32', 'INTEGER64', 'INTEGER128', 'INT8', 'INT16', 'INT32',
+        'INT64', 'INT128', 'STRING', 'UNISTRING',
+    )
+
+    # Modula-2 R10 Standard Library Blueprints Dataset
+    m2r10_stdlib_blueprint_identifiers = (
+        'ProtoRoot', 'ProtoComputational', 'ProtoNumeric', 'ProtoScalar',
+        'ProtoNonScalar', 'ProtoCardinal', 'ProtoInteger', 'ProtoReal',
+        'ProtoComplex', 'ProtoVector', 'ProtoTuple', 'ProtoCompArray',
+        'ProtoCollection', 'ProtoStaticArray', 'ProtoStaticSet',
+        'ProtoStaticString', 'ProtoArray', 'ProtoString', 'ProtoSet',
+        'ProtoMultiSet', 'ProtoDictionary', 'ProtoMultiDict', 'ProtoExtension',
+        'ProtoIO', 'ProtoCardMath', 'ProtoIntMath', 'ProtoRealMath',
+    )
+
+    # Modula-2 R10 Standard Library Modules Dataset
+    m2r10_stdlib_module_identifiers = (
+        'ASCII', 'BooleanIO', 'CharIO', 'UnicharIO', 'OctetIO',
+        'CardinalIO', 'LongCardIO', 'IntegerIO', 'LongIntIO', 'RealIO',
+        'LongRealIO', 'BCDIO', 'LongBCDIO', 'CardMath', 'LongCardMath',
+        'IntMath', 'LongIntMath', 'RealMath', 'LongRealMath', 'BCDMath',
+        'LongBCDMath', 'FileIO', 'FileSystem', 'Storage', 'IOSupport',
+    )
+
+    # Modula-2 R10 Standard Library Types Dataset
+    m2r10_stdlib_type_identifiers = (
+        'File', 'Status',
+        # TO BE COMPLETED
+    )
+
+    # Modula-2 R10 Standard Library Procedures Dataset
+    m2r10_stdlib_proc_identifiers = (
+        'ALLOCATE', 'DEALLOCATE', 'SIZE',
+        # TO BE COMPLETED
+    )
+
+    # Modula-2 R10 Standard Library Variables Dataset
+    m2r10_stdlib_var_identifiers = (
+        'stdIn', 'stdOut', 'stdErr',
+    )
+
+    # Modula-2 R10 Standard Library Constants Dataset
+    m2r10_stdlib_const_identifiers = (
+        'pi', 'tau',
+    )
+
+#  D i a l e c t s
+
+    # Dialect modes
+    dialects = (
+        'unknown',
+        'm2pim', 'm2iso', 'm2r10', 'objm2',
+        'm2iso+aglet', 'm2pim+gm2', 'm2iso+p1', 'm2iso+xds',
+    )
+
+#   D a t a b a s e s
+
+    # Lexemes to Mark as Errors Database
+    lexemes_to_reject_db = {
+        # Lexemes to reject for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Lexemes to reject for PIM Modula-2
+        'm2pim': (
+            pim_lexemes_to_reject,
+        ),
+        # Lexemes to reject for ISO Modula-2
+        'm2iso': (
+            iso_lexemes_to_reject,
+        ),
+        # Lexemes to reject for Modula-2 R10
+        'm2r10': (
+            m2r10_lexemes_to_reject,
+        ),
+        # Lexemes to reject for Objective Modula-2
+        'objm2': (
+            objm2_lexemes_to_reject,
+        ),
+        # Lexemes to reject for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_lexemes_to_reject,
+        ),
+        # Lexemes to reject for GNU Modula-2
+        'm2pim+gm2': (
+            pim_lexemes_to_reject,
+        ),
+        # Lexemes to reject for p1 Modula-2
+        'm2iso+p1': (
+            iso_lexemes_to_reject,
+        ),
+        # Lexemes to reject for XDS Modula-2
+        'm2iso+xds': (
+            iso_lexemes_to_reject,
+        ),
+    }
+
+    # Reserved Words Database
+    reserved_words_db = {
+        # Reserved words for unknown dialect
+        'unknown': (
+            common_reserved_words,
+            pim_additional_reserved_words,
+            iso_additional_reserved_words,
+            m2r10_additional_reserved_words,
+        ),
+
+        # Reserved words for PIM Modula-2
+        'm2pim': (
+            common_reserved_words,
+            pim_additional_reserved_words,
+        ),
+
+        # Reserved words for Modula-2 R10
+        'm2iso': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+        ),
+
+        # Reserved words for ISO Modula-2
+        'm2r10': (
+            common_reserved_words,
+            m2r10_additional_reserved_words,
+        ),
+
+        # Reserved words for Objective Modula-2
+        'objm2': (
+            common_reserved_words,
+            m2r10_additional_reserved_words,
+            objm2_additional_reserved_words,
+        ),
+
+        # Reserved words for Aglet Modula-2 Extensions
+        'm2iso+aglet': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+            aglet_additional_reserved_words,
+        ),
+
+        # Reserved words for GNU Modula-2 Extensions
+        'm2pim+gm2': (
+            common_reserved_words,
+            pim_additional_reserved_words,
+            gm2_additional_reserved_words,
+        ),
+
+        # Reserved words for p1 Modula-2 Extensions
+        'm2iso+p1': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+            p1_additional_reserved_words,
+        ),
+
+        # Reserved words for XDS Modula-2 Extensions
+        'm2iso+xds': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+            xds_additional_reserved_words,
+        ),
+    }
+
+    # Builtins Database
+    builtins_db = {
+        # Builtins for unknown dialect
+        'unknown': (
+            common_builtins,
+            pim_additional_builtins,
+            iso_additional_builtins,
+            m2r10_additional_builtins,
+        ),
+
+        # Builtins for PIM Modula-2
+        'm2pim': (
+            common_builtins,
+            pim_additional_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2iso': (
+            common_builtins,
+            iso_additional_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2r10': (
+            common_builtins,
+            m2r10_additional_builtins,
+        ),
+
+        # Builtins for Objective Modula-2
+        'objm2': (
+            common_builtins,
+            m2r10_additional_builtins,
+            objm2_additional_builtins,
+        ),
+
+        # Builtins for Aglet Modula-2 Extensions
+        'm2iso+aglet': (
+            common_builtins,
+            iso_additional_builtins,
+            aglet_additional_builtins,
+        ),
+
+        # Builtins for GNU Modula-2 Extensions
+        'm2pim+gm2': (
+            common_builtins,
+            pim_additional_builtins,
+            gm2_additional_builtins,
+        ),
+
+        # Builtins for p1 Modula-2 Extensions
+        'm2iso+p1': (
+            common_builtins,
+            iso_additional_builtins,
+            p1_additional_builtins,
+        ),
+
+        # Builtins for XDS Modula-2 Extensions
+        'm2iso+xds': (
+            common_builtins,
+            iso_additional_builtins,
+            xds_additional_builtins,
+        ),
+    }
+
+    # Pseudo-Module Builtins Database
+    pseudo_builtins_db = {
+        # Builtins for unknown dialect
+        'unknown': (
+            common_pseudo_builtins,
+            pim_additional_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            m2r10_additional_pseudo_builtins,
+        ),
+
+        # Builtins for PIM Modula-2
+        'm2pim': (
+            common_pseudo_builtins,
+            pim_additional_pseudo_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2iso': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2r10': (
+            common_pseudo_builtins,
+            m2r10_additional_pseudo_builtins,
+        ),
+
+        # Builtins for Objective Modula-2
+        'objm2': (
+            common_pseudo_builtins,
+            m2r10_additional_pseudo_builtins,
+            objm2_additional_pseudo_builtins,
+        ),
+
+        # Builtins for Aglet Modula-2 Extensions
+        'm2iso+aglet': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            aglet_additional_pseudo_builtins,
+        ),
+
+        # Builtins for GNU Modula-2 Extensions
+        'm2pim+gm2': (
+            common_pseudo_builtins,
+            pim_additional_pseudo_builtins,
+            gm2_additional_pseudo_builtins,
+        ),
+
+        # Builtins for p1 Modula-2 Extensions
+        'm2iso+p1': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            p1_additional_pseudo_builtins,
+        ),
+
+        # Builtins for XDS Modula-2 Extensions
+        'm2iso+xds': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            xds_additional_pseudo_builtins,
+        ),
+    }
+
+    # Standard Library ADTs Database
+    stdlib_adts_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library ADTs for PIM Modula-2
+        'm2pim': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for ISO Modula-2
+        'm2iso': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_adt_identifiers,
+        ),
+
+        # Standard Library ADTs for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_adt_identifiers,
+        ),
+
+        # Standard Library ADTs for Aglet Modula-2
+        'm2iso+aglet': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for GNU Modula-2
+        'm2pim+gm2': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for p1 Modula-2
+        'm2iso+p1': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for XDS Modula-2
+        'm2iso+xds': (
+            # No first class library types
+        ),
+    }
+
+    # Standard Library Modules Database
+    stdlib_modules_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Modules for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_blueprint_identifiers,
+            m2r10_stdlib_module_identifiers,
+            m2r10_stdlib_adt_identifiers,
+        ),
+
+        # Standard Library Modules for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_blueprint_identifiers,
+            m2r10_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_module_identifiers,
+        ),
+    }
+
+    # Standard Library Types Database
+    stdlib_types_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Types for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_type_identifiers,
+        ),
+    }
+
+    # Standard Library Procedures Database
+    stdlib_procedures_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Procedures for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_proc_identifiers,
+        ),
+    }
+
+    # Standard Library Variables Database
+    stdlib_variables_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Variables for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_var_identifiers,
+        ),
+    }
+
+    # Standard Library Constants Database
+    stdlib_constants_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Constants for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_const_identifiers,
+        ),
+    }
+
+#   M e t h o d s
+
+    # initialise a lexer instance
+    def __init__(self, **options):
+        #
+        # check dialect options
+        #
+        dialects = get_list_opt(options, 'dialect', [])
+        #
+        for dialect_option in dialects:
+            if dialect_option in self.dialects[1:-1]:
+                # valid dialect option found
+                self.set_dialect(dialect_option)
+                break
+        #
+        # Fallback Mode (DEFAULT)
+        else:
+            # no valid dialect option
+            self.set_dialect('unknown')
+        #
+        self.dialect_set_by_tag = False
+        #
+        # check style options
+        #
+        styles = get_list_opt(options, 'style', [])
+        #
+        # use lowercase mode for Algol style
+        if 'algol' in styles or 'algol_nu' in styles:
+            self.algol_publication_mode = True
+        else:
+            self.algol_publication_mode = False
+        #
+        # Check option flags
+        #
+        self.treat_stdlib_adts_as_builtins = get_bool_opt(
+            options, 'treat_stdlib_adts_as_builtins', True)
+        #
+        # call superclass initialiser
+        RegexLexer.__init__(self, **options)
+
+    # Set lexer to a specified dialect
+    def set_dialect(self, dialect_id):
+        #
+        # if __debug__:
+        #    print 'entered set_dialect with arg: ', dialect_id
+        #
+        # check dialect name against known dialects
+        if dialect_id not in self.dialects:
+            dialect = 'unknown'  # default
+        else:
+            dialect = dialect_id
+        #
+        # compose lexemes to reject set
+        lexemes_to_reject_set = set()
+        # add each list of reject lexemes for this dialect
+        for list in self.lexemes_to_reject_db[dialect]:
+            lexemes_to_reject_set.update(set(list))
+        #
+        # compose reserved words set
+        reswords_set = set()
+        # add each list of reserved words for this dialect
+        for list in self.reserved_words_db[dialect]:
+            reswords_set.update(set(list))
+        #
+        # compose builtins set
+        builtins_set = set()
+        # add each list of builtins for this dialect excluding reserved words
+        for list in self.builtins_db[dialect]:
+            builtins_set.update(set(list).difference(reswords_set))
+        #
+        # compose pseudo-builtins set
+        pseudo_builtins_set = set()
+        # add each list of builtins for this dialect excluding reserved words
+        for list in self.pseudo_builtins_db[dialect]:
+            pseudo_builtins_set.update(set(list).difference(reswords_set))
+        #
+        # compose ADTs set
+        adts_set = set()
+        # add each list of ADTs for this dialect excluding reserved words
+        for list in self.stdlib_adts_db[dialect]:
+            adts_set.update(set(list).difference(reswords_set))
+        #
+        # compose modules set
+        modules_set = set()
+        # add each list of builtins for this dialect excluding builtins
+        for list in self.stdlib_modules_db[dialect]:
+            modules_set.update(set(list).difference(builtins_set))
+        #
+        # compose types set
+        types_set = set()
+        # add each list of types for this dialect excluding builtins
+        for list in self.stdlib_types_db[dialect]:
+            types_set.update(set(list).difference(builtins_set))
+        #
+        # compose procedures set
+        procedures_set = set()
+        # add each list of procedures for this dialect excluding builtins
+        for list in self.stdlib_procedures_db[dialect]:
+            procedures_set.update(set(list).difference(builtins_set))
+        #
+        # compose variables set
+        variables_set = set()
+        # add each list of variables for this dialect excluding builtins
+        for list in self.stdlib_variables_db[dialect]:
+            variables_set.update(set(list).difference(builtins_set))
+        #
+        # compose constants set
+        constants_set = set()
+        # add each list of constants for this dialect excluding builtins
+        for list in self.stdlib_constants_db[dialect]:
+            constants_set.update(set(list).difference(builtins_set))
+        #
+        # update lexer state
+        self.dialect = dialect
+        self.lexemes_to_reject = lexemes_to_reject_set
+        self.reserved_words = reswords_set
+        self.builtins = builtins_set
+        self.pseudo_builtins = pseudo_builtins_set
+        self.adts = adts_set
+        self.modules = modules_set
+        self.types = types_set
+        self.procedures = procedures_set
+        self.variables = variables_set
+        self.constants = constants_set
+        #
+        # if __debug__:
+        #    print 'exiting set_dialect'
+        #    print ' self.dialect: ', self.dialect
+        #    print ' self.lexemes_to_reject: ', self.lexemes_to_reject
+        #    print ' self.reserved_words: ', self.reserved_words
+        #    print ' self.builtins: ', self.builtins
+        #    print ' self.pseudo_builtins: ', self.pseudo_builtins
+        #    print ' self.adts: ', self.adts
+        #    print ' self.modules: ', self.modules
+        #    print ' self.types: ', self.types
+        #    print ' self.procedures: ', self.procedures
+        #    print ' self.variables: ', self.variables
+        #    print ' self.types: ', self.types
+        #    print ' self.constants: ', self.constants
+
+    # Extracts a dialect name from a dialect tag comment string  and checks
+    # the extracted name against known dialects.  If a match is found,  the
+    # matching name is returned, otherwise dialect id 'unknown' is returned
+    def get_dialect_from_dialect_tag(self, dialect_tag):
+        #
+        # if __debug__:
+        #    print 'entered get_dialect_from_dialect_tag with arg: ', dialect_tag
+        #
+        # constants
+        left_tag_delim = '(*!'
+        right_tag_delim = '*)'
+        left_tag_delim_len = len(left_tag_delim)
+        right_tag_delim_len = len(right_tag_delim)
+        indicator_start = left_tag_delim_len
+        indicator_end = -(right_tag_delim_len)
+        #
+        # check comment string for dialect indicator
+        if len(dialect_tag) > (left_tag_delim_len + right_tag_delim_len) \
+           and dialect_tag.startswith(left_tag_delim) \
+           and dialect_tag.endswith(right_tag_delim):
+            #
+            # if __debug__:
+            #    print 'dialect tag found'
+            #
+            # extract dialect indicator
+            indicator = dialect_tag[indicator_start:indicator_end]
+            #
+            # if __debug__:
+            #    print 'extracted: ', indicator
+            #
+            # check against known dialects
+            for index in range(1, len(self.dialects)):
+                #
+                # if __debug__:
+                #    print 'dialects[', index, ']: ', self.dialects[index]
+                #
+                if indicator == self.dialects[index]:
+                    #
+                    # if __debug__:
+                    #    print 'matching dialect found'
+                    #
+                    # indicator matches known dialect
+                    return indicator
+            else:
+                # indicator does not match any dialect
+                return 'unknown'  # default
+        else:
+            # invalid indicator string
+            return 'unknown'  # default
+
+    # intercept the token stream, modify token attributes and return them
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in RegexLexer.get_tokens_unprocessed(self, text):
+            #
+            # check for dialect tag if dialect has not been set by tag
+            if not self.dialect_set_by_tag and token == Comment.Special:
+                indicated_dialect = self.get_dialect_from_dialect_tag(value)
+                if indicated_dialect != 'unknown':
+                    # token is a dialect indicator
+                    # reset reserved words and builtins
+                    self.set_dialect(indicated_dialect)
+                    self.dialect_set_by_tag = True
+            #
+            # check for reserved words, predefined and stdlib identifiers
+            if token is Name:
+                if value in self.reserved_words:
+                    token = Keyword.Reserved
+                    if self.algol_publication_mode:
+                        value = value.lower()
+                #
+                elif value in self.builtins:
+                    token = Name.Builtin
+                    if self.algol_publication_mode:
+                        value = value.lower()
+                #
+                elif value in self.pseudo_builtins:
+                    token = Name.Builtin.Pseudo
+                    if self.algol_publication_mode:
+                        value = value.lower()
+                #
+                elif value in self.adts:
+                    if not self.treat_stdlib_adts_as_builtins:
+                        token = Name.Namespace
+                    else:
+                        token = Name.Builtin.Pseudo
+                        if self.algol_publication_mode:
+                            value = value.lower()
+                #
+                elif value in self.modules:
+                    token = Name.Namespace
+                #
+                elif value in self.types:
+                    token = Name.Class
+                #
+                elif value in self.procedures:
+                    token = Name.Function
+                #
+                elif value in self.variables:
+                    token = Name.Variable
+                #
+                elif value in self.constants:
+                    token = Name.Constant
+            #
+            elif token in Number:
+                #
+                # mark prefix number literals as error for PIM and ISO dialects
+                if self.dialect not in ('unknown', 'm2r10', 'objm2'):
+                    if "'" in value or value[0:2] in ('0b', '0x', '0u'):
+                        token = Error
+                #
+                elif self.dialect in ('m2r10', 'objm2'):
+                    # mark base-8 number literals as errors for M2 R10 and ObjM2
+                    if token is Number.Oct:
+                        token = Error
+                    # mark suffix base-16 literals as errors for M2 R10 and ObjM2
+                    elif token is Number.Hex and 'H' in value:
+                        token = Error
+                    # mark real numbers with E as errors for M2 R10 and ObjM2
+                    elif token is Number.Float and 'E' in value:
+                        token = Error
+            #
+            elif token in Comment:
+                #
+                # mark single line comment as error for PIM and ISO dialects
+                if token is Comment.Single:
+                    if self.dialect not in ('unknown', 'm2r10', 'objm2'):
+                        token = Error
+                #
+                if token is Comment.Preproc:
+                    # mark ISO pragma as error for PIM dialects
+                    if value.startswith('<*') and \
+                       self.dialect.startswith('m2pim'):
+                        token = Error
+                    # mark PIM pragma as comment for other dialects
+                    elif value.startswith('(*$') and \
+                            self.dialect != 'unknown' and \
+                            not self.dialect.startswith('m2pim'):
+                        token = Comment.Multiline
+            #
+            else:  # token is neither Name nor Comment
+                #
+                # mark lexemes matching the dialect's error token set as errors
+                if value in self.lexemes_to_reject:
+                    token = Error
+                #
+                # substitute lexemes when in Algol mode
+                if self.algol_publication_mode:
+                    if value == '#':
+                        value = '≠'
+                    elif value == '<=':
+                        value = '≤'
+                    elif value == '>=':
+                        value = '≥'
+                    elif value == '==':
+                        value = '≡'
+                    elif value == '*.':
+                        value = '•'
+
+            # return result
+            yield index, token, value
+
+    def analyse_text(text):
+        """It's Pascal-like, but does not use FUNCTION -- uses PROCEDURE
+        instead."""
+
+        # Check if this looks like Pascal, if not, bail out early
+        if not ('(*' in text and '*)' in text and ':=' in text):
+            return
+
+        result = 0
+        # Procedure is in Modula2
+        if re.search(r'\bPROCEDURE\b', text):
+            result += 0.6
+
+        # FUNCTION is only valid in Pascal, but not in Modula2
+        if re.search(r'\bFUNCTION\b', text):
+            result = 0.0
+
+        return result
diff --git a/lib/pygments/lexers/mojo.py b/lib/pygments/lexers/mojo.py
new file mode 100644
index 0000000..4df18c4
--- /dev/null
+++ b/lib/pygments/lexers/mojo.py
@@ -0,0 +1,707 @@
+"""
+    pygments.lexers.mojo
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Mojo and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import keyword
+
+from pygments import unistring as uni
+from pygments.lexer import (
+    RegexLexer,
+    bygroups,
+    combined,
+    default,
+    include,
+    this,
+    using,
+    words,
+)
+from pygments.token import (
+    Comment,
+    # Error,
+    Keyword,
+    Name,
+    Number,
+    Operator,
+    Punctuation,
+    String,
+    Text,
+    Whitespace,
+)
+from pygments.util import shebang_matches
+
+__all__ = ["MojoLexer"]
+
+
+class MojoLexer(RegexLexer):
+    """
+    For Mojo source code (version 24.2.1).
+    """
+
+    name = "Mojo"
+    url = "https://docs.modular.com/mojo/"
+    aliases = ["mojo", "🔥"]
+    filenames = [
+        "*.mojo",
+        "*.🔥",
+    ]
+    mimetypes = [
+        "text/x-mojo",
+        "application/x-mojo",
+    ]
+    version_added = "2.18"
+
+    uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*"
+
+    def innerstring_rules(ttype):
+        return [
+            # the old style '%s' % (...) string formatting (still valid in Py3)
+            (
+                r"%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?"
+                "[hlL]?[E-GXc-giorsaux%]",
+                String.Interpol,
+            ),
+            # the new style '{}'.format(...) string formatting
+            (
+                r"\{"
+                r"((\w+)((\.\w+)|(\[[^\]]+\]))*)?"  # field name
+                r"(\![sra])?"  # conversion
+                r"(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?"
+                r"\}",
+                String.Interpol,
+            ),
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"%{\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # unhandled string formatting sign
+            (r"%|(\{{1,2})", ttype),
+            # newlines are an error (use "nl" state)
+        ]
+
+    def fstring_rules(ttype):
+        return [
+            # Assuming that a '}' is the closing brace after format specifier.
+            # Sadly, this means that we won't detect syntax error. But it's
+            # more important to parse correct syntax correctly, than to
+            # highlight invalid syntax.
+            (r"\}", String.Interpol),
+            (r"\{", String.Interpol, "expr-inside-fstring"),
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"{}\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # newlines are an error (use "nl" state)
+        ]
+
+    tokens = {
+        "root": [
+            (r"\s+", Whitespace),
+            (
+                r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
+                bygroups(Whitespace, String.Affix, String.Doc),
+            ),
+            (
+                r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
+                bygroups(Whitespace, String.Affix, String.Doc),
+            ),
+            (r"\A#!.+$", Comment.Hashbang),
+            (r"#.*$", Comment.Single),
+            (r"\\\n", Whitespace),
+            (r"\\", Whitespace),
+            include("keywords"),
+            include("soft-keywords"),
+            # In the original PR, all the below here used ((?:\s|\\\s)+) to
+            # designate whitespace, but I can't find any example of this being
+            # needed in the example file, so we're replacing it with `\s+`.
+            (
+                r"(alias)(\s+)",
+                bygroups(Keyword, Whitespace),
+                "varname",  # TODO varname the right fit?
+            ),
+            (r"(var)(\s+)", bygroups(Keyword, Whitespace), "varname"),
+            (r"(def)(\s+)", bygroups(Keyword, Whitespace), "funcname"),
+            (r"(fn)(\s+)", bygroups(Keyword, Whitespace), "funcname"),
+            (
+                r"(class)(\s+)",
+                bygroups(Keyword, Whitespace),
+                "classname",
+            ),  # not implemented yet
+            (r"(struct)(\s+)", bygroups(Keyword, Whitespace), "structname"),
+            (r"(trait)(\s+)", bygroups(Keyword, Whitespace), "structname"),
+            (r"(from)(\s+)", bygroups(Keyword.Namespace, Whitespace), "fromimport"),
+            (r"(import)(\s+)", bygroups(Keyword.Namespace, Whitespace), "import"),
+            include("expr"),
+        ],
+        "expr": [
+            # raw f-strings
+            (
+                '(?i)(rf|fr)(""")',
+                bygroups(String.Affix, String.Double),
+                combined("rfstringescape", "tdqf"),
+            ),
+            (
+                "(?i)(rf|fr)(''')",
+                bygroups(String.Affix, String.Single),
+                combined("rfstringescape", "tsqf"),
+            ),
+            (
+                '(?i)(rf|fr)(")',
+                bygroups(String.Affix, String.Double),
+                combined("rfstringescape", "dqf"),
+            ),
+            (
+                "(?i)(rf|fr)(')",
+                bygroups(String.Affix, String.Single),
+                combined("rfstringescape", "sqf"),
+            ),
+            # non-raw f-strings
+            (
+                '([fF])(""")',
+                bygroups(String.Affix, String.Double),
+                combined("fstringescape", "tdqf"),
+            ),
+            (
+                "([fF])(''')",
+                bygroups(String.Affix, String.Single),
+                combined("fstringescape", "tsqf"),
+            ),
+            (
+                '([fF])(")',
+                bygroups(String.Affix, String.Double),
+                combined("fstringescape", "dqf"),
+            ),
+            (
+                "([fF])(')",
+                bygroups(String.Affix, String.Single),
+                combined("fstringescape", "sqf"),
+            ),
+            # raw bytes and strings
+            ('(?i)(rb|br|r)(""")', bygroups(String.Affix, String.Double), "tdqs"),
+            ("(?i)(rb|br|r)(''')", bygroups(String.Affix, String.Single), "tsqs"),
+            ('(?i)(rb|br|r)(")', bygroups(String.Affix, String.Double), "dqs"),
+            ("(?i)(rb|br|r)(')", bygroups(String.Affix, String.Single), "sqs"),
+            # non-raw strings
+            (
+                '([uU]?)(""")',
+                bygroups(String.Affix, String.Double),
+                combined("stringescape", "tdqs"),
+            ),
+            (
+                "([uU]?)(''')",
+                bygroups(String.Affix, String.Single),
+                combined("stringescape", "tsqs"),
+            ),
+            (
+                '([uU]?)(")',
+                bygroups(String.Affix, String.Double),
+                combined("stringescape", "dqs"),
+            ),
+            (
+                "([uU]?)(')",
+                bygroups(String.Affix, String.Single),
+                combined("stringescape", "sqs"),
+            ),
+            # non-raw bytes
+            (
+                '([bB])(""")',
+                bygroups(String.Affix, String.Double),
+                combined("bytesescape", "tdqs"),
+            ),
+            (
+                "([bB])(''')",
+                bygroups(String.Affix, String.Single),
+                combined("bytesescape", "tsqs"),
+            ),
+            (
+                '([bB])(")',
+                bygroups(String.Affix, String.Double),
+                combined("bytesescape", "dqs"),
+            ),
+            (
+                "([bB])(')",
+                bygroups(String.Affix, String.Single),
+                combined("bytesescape", "sqs"),
+            ),
+            (r"[^\S\n]+", Text),
+            include("numbers"),
+            (r"!=|==|<<|>>|:=|[-~+/*%=<>&^|.]", Operator),
+            (r"([]{}:\(\),;[])+", Punctuation),
+            (r"(in|is|and|or|not)\b", Operator.Word),
+            include("expr-keywords"),
+            include("builtins"),
+            include("magicfuncs"),
+            include("magicvars"),
+            include("name"),
+        ],
+        "expr-inside-fstring": [
+            (r"[{([]", Punctuation, "expr-inside-fstring-inner"),
+            # without format specifier
+            (
+                r"(=\s*)?"  # debug (https://bugs.python.org/issue36817)
+                r"(\![sraf])?"  # conversion
+                r"\}",
+                String.Interpol,
+                "#pop",
+            ),
+            # with format specifier
+            # we'll catch the remaining '}' in the outer scope
+            (
+                r"(=\s*)?"  # debug (https://bugs.python.org/issue36817)
+                r"(\![sraf])?"  # conversion
+                r":",
+                String.Interpol,
+                "#pop",
+            ),
+            (r"\s+", Whitespace),  # allow new lines
+            include("expr"),
+        ],
+        "expr-inside-fstring-inner": [
+            (r"[{([]", Punctuation, "expr-inside-fstring-inner"),
+            (r"[])}]", Punctuation, "#pop"),
+            (r"\s+", Whitespace),  # allow new lines
+            include("expr"),
+        ],
+        "expr-keywords": [
+            # Based on https://docs.python.org/3/reference/expressions.html
+            (
+                words(
+                    (
+                        "async for",  # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with
+                        "async with",  # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with
+                        "await",
+                        "else",
+                        "for",
+                        "if",
+                        "lambda",
+                        "yield",
+                        "yield from",
+                    ),
+                    suffix=r"\b",
+                ),
+                Keyword,
+            ),
+            (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant),
+        ],
+        "keywords": [
+            (
+                words(
+                    (
+                        "assert",
+                        "async",
+                        "await",
+                        "borrowed",
+                        "break",
+                        "continue",
+                        "del",
+                        "elif",
+                        "else",
+                        "except",
+                        "finally",
+                        "for",
+                        "global",
+                        "if",
+                        "lambda",
+                        "pass",
+                        "raise",
+                        "nonlocal",
+                        "return",
+                        "try",
+                        "while",
+                        "yield",
+                        "yield from",
+                        "as",
+                        "with",
+                    ),
+                    suffix=r"\b",
+                ),
+                Keyword,
+            ),
+            (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant),
+        ],
+        "soft-keywords": [
+            # `match`, `case` and `_` soft keywords
+            (
+                r"(^[ \t]*)"  # at beginning of line + possible indentation
+                r"(match|case)\b"  # a possible keyword
+                r"(?![ \t]*(?:"  # not followed by...
+                r"[:,;=^&|@~)\]}]|(?:" +  # characters and keywords that mean this isn't
+                # pattern matching (but None/True/False is ok)
+                r"|".join(k for k in keyword.kwlist if k[0].islower())
+                + r")\b))",
+                bygroups(Whitespace, Keyword),
+                "soft-keywords-inner",
+            ),
+        ],
+        "soft-keywords-inner": [
+            # optional `_` keyword
+            (r"(\s+)([^\n_]*)(_\b)", bygroups(Whitespace, using(this), Keyword)),
+            default("#pop"),
+        ],
+        "builtins": [
+            (
+                words(
+                    (
+                        "__import__",
+                        "abs",
+                        "aiter",
+                        "all",
+                        "any",
+                        "bin",
+                        "bool",
+                        "bytearray",
+                        "breakpoint",
+                        "bytes",
+                        "callable",
+                        "chr",
+                        "classmethod",
+                        "compile",
+                        "complex",
+                        "delattr",
+                        "dict",
+                        "dir",
+                        "divmod",
+                        "enumerate",
+                        "eval",
+                        "filter",
+                        "float",
+                        "format",
+                        "frozenset",
+                        "getattr",
+                        "globals",
+                        "hasattr",
+                        "hash",
+                        "hex",
+                        "id",
+                        "input",
+                        "int",
+                        "isinstance",
+                        "issubclass",
+                        "iter",
+                        "len",
+                        "list",
+                        "locals",
+                        "map",
+                        "max",
+                        "memoryview",
+                        "min",
+                        "next",
+                        "object",
+                        "oct",
+                        "open",
+                        "ord",
+                        "pow",
+                        "print",
+                        "property",
+                        "range",
+                        "repr",
+                        "reversed",
+                        "round",
+                        "set",
+                        "setattr",
+                        "slice",
+                        "sorted",
+                        "staticmethod",
+                        "str",
+                        "sum",
+                        "super",
+                        "tuple",
+                        "type",
+                        "vars",
+                        "zip",
+                        # Mojo builtin types: https://docs.modular.com/mojo/stdlib/builtin/
+                        "AnyType",
+                        "Coroutine",
+                        "DType",
+                        "Error",
+                        "Int",
+                        "List",
+                        "ListLiteral",
+                        "Scalar",
+                        "Int8",
+                        "UInt8",
+                        "Int16",
+                        "UInt16",
+                        "Int32",
+                        "UInt32",
+                        "Int64",
+                        "UInt64",
+                        "BFloat16",
+                        "Float16",
+                        "Float32",
+                        "Float64",
+                        "SIMD",
+                        "String",
+                        "Tensor",
+                        "Tuple",
+                        "Movable",
+                        "Copyable",
+                        "CollectionElement",
+                    ),
+                    prefix=r"(?>',
+    # Binary augmented
+    '+=', '-=', '*=', '/=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>=',
+    # Comparison
+    '==', '!=', '<', '<=', '>', '>=', '<=>',
+    # Patterns and assignment
+    ':=', '?', '=~', '!~', '=>',
+    # Calls and sends
+    '.', '<-', '->',
+]
+_escape_pattern = (
+    r'(?:\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|'
+    r'\\["\'\\bftnr])')
+# _char = _escape_chars + [('.', String.Char)]
+_identifier = r'[_a-zA-Z]\w*'
+
+_constants = [
+    # Void constants
+    'null',
+    # Bool constants
+    'false', 'true',
+    # Double constants
+    'Infinity', 'NaN',
+    # Special objects
+    'M', 'Ref', 'throw', 'traceln',
+]
+
+_guards = [
+    'Any', 'Binding', 'Bool', 'Bytes', 'Char', 'DeepFrozen', 'Double',
+    'Empty', 'Int', 'List', 'Map', 'Near', 'NullOk', 'Same', 'Selfless',
+    'Set', 'Str', 'SubrangeGuard', 'Transparent', 'Void',
+]
+
+_safeScope = [
+    '_accumulateList', '_accumulateMap', '_auditedBy', '_bind',
+    '_booleanFlow', '_comparer', '_equalizer', '_iterForever', '_loop',
+    '_makeBytes', '_makeDouble', '_makeFinalSlot', '_makeInt', '_makeList',
+    '_makeMap', '_makeMessageDesc', '_makeOrderedSpace', '_makeParamDesc',
+    '_makeProtocolDesc', '_makeSourceSpan', '_makeString', '_makeVarSlot',
+    '_makeVerbFacet', '_mapExtract', '_matchSame', '_quasiMatcher',
+    '_slotToBinding', '_splitList', '_suchThat', '_switchFailed',
+    '_validateFor', 'b__quasiParser', 'eval', 'import', 'm__quasiParser',
+    'makeBrandPair', 'makeLazySlot', 'safeScope', 'simple__quasiParser',
+]
+
+
+class MonteLexer(RegexLexer):
+    """
+    Lexer for the Monte programming language.
+    """
+    name = 'Monte'
+    url = 'https://monte.readthedocs.io/'
+    aliases = ['monte']
+    filenames = ['*.mt']
+    version_added = '2.2'
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'#[^\n]*\n', Comment),
+
+            # Docstrings
+            # Apologies for the non-greedy matcher here.
+            (r'/\*\*.*?\*/', String.Doc),
+
+            # `var` declarations
+            (r'\bvar\b', Keyword.Declaration, 'var'),
+
+            # `interface` declarations
+            (r'\binterface\b', Keyword.Declaration, 'interface'),
+
+            # method declarations
+            (words(_methods, prefix='\\b', suffix='\\b'),
+             Keyword, 'method'),
+
+            # All other declarations
+            (words(_declarations, prefix='\\b', suffix='\\b'),
+             Keyword.Declaration),
+
+            # Keywords
+            (words(_keywords, prefix='\\b', suffix='\\b'), Keyword),
+
+            # Literals
+            ('[+-]?0x[_0-9a-fA-F]+', Number.Hex),
+            (r'[+-]?[_0-9]+\.[_0-9]*([eE][+-]?[_0-9]+)?', Number.Float),
+            ('[+-]?[_0-9]+', Number.Integer),
+            ("'", String.Double, 'char'),
+            ('"', String.Double, 'string'),
+
+            # Quasiliterals
+            ('`', String.Backtick, 'ql'),
+
+            # Operators
+            (words(_operators), Operator),
+
+            # Verb operators
+            (_identifier + '=', Operator.Word),
+
+            # Safe scope constants
+            (words(_constants, prefix='\\b', suffix='\\b'),
+             Keyword.Pseudo),
+
+            # Safe scope guards
+            (words(_guards, prefix='\\b', suffix='\\b'), Keyword.Type),
+
+            # All other safe scope names
+            (words(_safeScope, prefix='\\b', suffix='\\b'),
+             Name.Builtin),
+
+            # Identifiers
+            (_identifier, Name),
+
+            # Punctuation
+            (r'\(|\)|\{|\}|\[|\]|:|,', Punctuation),
+
+            # Whitespace
+            (' +', Whitespace),
+
+            # Definite lexer errors
+            ('=', Error),
+        ],
+        'char': [
+            # It is definitely an error to have a char of width == 0.
+            ("'", Error, 'root'),
+            (_escape_pattern, String.Escape, 'charEnd'),
+            ('.', String.Char, 'charEnd'),
+        ],
+        'charEnd': [
+            ("'", String.Char, '#pop:2'),
+            # It is definitely an error to have a char of width > 1.
+            ('.', Error),
+        ],
+        # The state of things coming into an interface.
+        'interface': [
+            (' +', Whitespace),
+            (_identifier, Name.Class, '#pop'),
+            include('root'),
+        ],
+        # The state of things coming into a method.
+        'method': [
+            (' +', Whitespace),
+            (_identifier, Name.Function, '#pop'),
+            include('root'),
+        ],
+        'string': [
+            ('"', String.Double, 'root'),
+            (_escape_pattern, String.Escape),
+            (r'\n', String.Double),
+            ('.', String.Double),
+        ],
+        'ql': [
+            ('`', String.Backtick, 'root'),
+            (r'\$' + _escape_pattern, String.Escape),
+            (r'\$\$', String.Escape),
+            (r'@@', String.Escape),
+            (r'\$\{', String.Interpol, 'qlNest'),
+            (r'@\{', String.Interpol, 'qlNest'),
+            (r'\$' + _identifier, Name),
+            ('@' + _identifier, Name),
+            ('.', String.Backtick),
+        ],
+        'qlNest': [
+            (r'\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        # The state of things immediately following `var`.
+        'var': [
+            (' +', Whitespace),
+            (_identifier, Name.Variable, '#pop'),
+            include('root'),
+        ],
+    }
diff --git a/lib/pygments/lexers/mosel.py b/lib/pygments/lexers/mosel.py
new file mode 100644
index 0000000..426c9a1
--- /dev/null
+++ b/lib/pygments/lexers/mosel.py
@@ -0,0 +1,447 @@
+"""
+    pygments.lexers.mosel
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the mosel language.
+    http://www.fico.com/en/products/fico-xpress-optimization
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['MoselLexer']
+
+FUNCTIONS = (
+    # core functions
+    '_',
+    'abs',
+    'arctan',
+    'asproc',
+    'assert',
+    'bitflip',
+    'bitneg',
+    'bitset',
+    'bitshift',
+    'bittest',
+    'bitval',
+    'ceil',
+    'cos',
+    'create',
+    'currentdate',
+    'currenttime',
+    'cutelt',
+    'cutfirst',
+    'cuthead',
+    'cutlast',
+    'cuttail',
+    'datablock',
+    'delcell',
+    'exists',
+    'exit',
+    'exp',
+    'exportprob',
+    'fclose',
+    'fflush',
+    'finalize',
+    'findfirst',
+    'findlast',
+    'floor',
+    'fopen',
+    'fselect',
+    'fskipline',
+    'fwrite',
+    'fwrite_',
+    'fwriteln',
+    'fwriteln_',
+    'getact',
+    'getcoeff',
+    'getcoeffs',
+    'getdual',
+    'getelt',
+    'getfid',
+    'getfirst',
+    'getfname',
+    'gethead',
+    'getlast',
+    'getobjval',
+    'getparam',
+    'getrcost',
+    'getreadcnt',
+    'getreverse',
+    'getsize',
+    'getslack',
+    'getsol',
+    'gettail',
+    'gettype',
+    'getvars',
+    'isdynamic',
+    'iseof',
+    'isfinite',
+    'ishidden',
+    'isinf',
+    'isnan',
+    'isodd',
+    'ln',
+    'localsetparam',
+    'log',
+    'makesos1',
+    'makesos2',
+    'maxlist',
+    'memoryuse',
+    'minlist',
+    'newmuid',
+    'publish',
+    'random',
+    'read',
+    'readln',
+    'reset',
+    'restoreparam',
+    'reverse',
+    'round',
+    'setcoeff',
+    'sethidden',
+    'setioerr',
+    'setmatherr',
+    'setname',
+    'setparam',
+    'setrandseed',
+    'setrange',
+    'settype',
+    'sin',
+    'splithead',
+    'splittail',
+    'sqrt',
+    'strfmt',
+    'substr',
+    'timestamp',
+    'unpublish',
+    'versionnum',
+    'versionstr',
+    'write',
+    'write_',
+    'writeln',
+    'writeln_',
+
+    # mosel exam mmxprs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u
+    'addcut',
+    'addcuts',
+    'addmipsol',
+    'basisstability',
+    'calcsolinfo',
+    'clearmipdir',
+    'clearmodcut',
+    'command',
+    'copysoltoinit',
+    'crossoverlpsol',
+    'defdelayedrows',
+    'defsecurevecs',
+    'delcuts',
+    'dropcuts',
+    'estimatemarginals',
+    'fixglobal',
+    'flushmsgq',
+    'getbstat',
+    'getcnlist',
+    'getcplist',
+    'getdualray',
+    'getiis',
+    'getiissense',
+    'getiistype',
+    'getinfcause',
+    'getinfeas',
+    'getlb',
+    'getlct',
+    'getleft',
+    'getloadedlinctrs',
+    'getloadedmpvars',
+    'getname',
+    'getprimalray',
+    'getprobstat',
+    'getrange',
+    'getright',
+    'getsensrng',
+    'getsize',
+    'getsol',
+    'gettype',
+    'getub',
+    'getvars',
+    'gety',
+    'hasfeature',
+    'implies',
+    'indicator',
+    'initglobal',
+    'ishidden',
+    'isiisvalid',
+    'isintegral',
+    'loadbasis',
+    'loadcuts',
+    'loadlpsol',
+    'loadmipsol',
+    'loadprob',
+    'maximise',
+    'maximize',
+    'minimise',
+    'minimize',
+    'postsolve',
+    'readbasis',
+    'readdirs',
+    'readsol',
+    'refinemipsol',
+    'rejectintsol',
+    'repairinfeas',
+    'repairinfeas_deprec',
+    'resetbasis',
+    'resetiis',
+    'resetsol',
+    'savebasis',
+    'savemipsol',
+    'savesol',
+    'savestate',
+    'selectsol',
+    'setarchconsistency',
+    'setbstat',
+    'setcallback',
+    'setcbcutoff',
+    'setgndata',
+    'sethidden',
+    'setlb',
+    'setmipdir',
+    'setmodcut',
+    'setsol',
+    'setub',
+    'setucbdata',
+    'stopoptimise',
+    'stopoptimize',
+    'storecut',
+    'storecuts',
+    'unloadprob',
+    'uselastbarsol',
+    'writebasis',
+    'writedirs',
+    'writeprob',
+    'writesol',
+    'xor',
+    'xprs_addctr',
+    'xprs_addindic',
+
+    # mosel exam mmsystem | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u
+    'addmonths',
+    'copytext',
+    'cuttext',
+    'deltext',
+    'endswith',
+    'erase',
+    'expandpath',
+    'fcopy',
+    'fdelete',
+    'findfiles',
+    'findtext',
+    'fmove',
+    'formattext',
+    'getasnumber',
+    'getchar',
+    'getcwd',
+    'getdate',
+    'getday',
+    'getdaynum',
+    'getdays',
+    'getdirsep',
+    'getdsoparam',
+    'getendparse',
+    'getenv',
+    'getfsize',
+    'getfstat',
+    'getftime',
+    'gethour',
+    'getminute',
+    'getmonth',
+    'getmsec',
+    'getoserrmsg',
+    'getoserror',
+    'getpathsep',
+    'getqtype',
+    'getsecond',
+    'getsepchar',
+    'getsize',
+    'getstart',
+    'getsucc',
+    'getsysinfo',
+    'getsysstat',
+    'gettime',
+    'gettmpdir',
+    'gettrim',
+    'getweekday',
+    'getyear',
+    'inserttext',
+    'isvalid',
+    'jointext',
+    'makedir',
+    'makepath',
+    'newtar',
+    'newzip',
+    'nextfield',
+    'openpipe',
+    'parseextn',
+    'parseint',
+    'parsereal',
+    'parsetext',
+    'pastetext',
+    'pathmatch',
+    'pathsplit',
+    'qsort',
+    'quote',
+    'readtextline',
+    'regmatch',
+    'regreplace',
+    'removedir',
+    'removefiles',
+    'setchar',
+    'setdate',
+    'setday',
+    'setdsoparam',
+    'setendparse',
+    'setenv',
+    'sethour',
+    'setminute',
+    'setmonth',
+    'setmsec',
+    'setoserror',
+    'setqtype',
+    'setsecond',
+    'setsepchar',
+    'setstart',
+    'setsucc',
+    'settime',
+    'settrim',
+    'setyear',
+    'sleep',
+    'splittext',
+    'startswith',
+    'system',
+    'tarlist',
+    'textfmt',
+    'tolower',
+    'toupper',
+    'trim',
+    'untar',
+    'unzip',
+    'ziplist',
+
+    # mosel exam mmjobs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u
+    'canceltimer',
+    'clearaliases',
+    'compile',
+    'connect',
+    'detach',
+    'disconnect',
+    'dropnextevent',
+    'findxsrvs',
+    'getaliases',
+    'getannidents',
+    'getannotations',
+    'getbanner',
+    'getclass',
+    'getdsoprop',
+    'getdsopropnum',
+    'getexitcode',
+    'getfromgid',
+    'getfromid',
+    'getfromuid',
+    'getgid',
+    'gethostalias',
+    'getid',
+    'getmodprop',
+    'getmodpropnum',
+    'getnextevent',
+    'getnode',
+    'getrmtid',
+    'getstatus',
+    'getsysinfo',
+    'gettimer',
+    'getuid',
+    'getvalue',
+    'isqueueempty',
+    'load',
+    'nullevent',
+    'peeknextevent',
+    'resetmodpar',
+    'run',
+    'send',
+    'setcontrol',
+    'setdefstream',
+    'setgid',
+    'sethostalias',
+    'setmodpar',
+    'settimer',
+    'setuid',
+    'setworkdir',
+    'stop',
+    'unload',
+    'wait',
+    'waitexpired',
+    'waitfor',
+    'waitforend',
+)
+
+
+class MoselLexer(RegexLexer):
+    """
+    For the Mosel optimization language.
+    """
+    name = 'Mosel'
+    aliases = ['mosel']
+    filenames = ['*.mos']
+    url = 'https://www.fico.com/fico-xpress-optimization/docs/latest/mosel/mosel_lang/dhtml/moselreflang.html'
+    version_added = '2.6'
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'\s+', Text.Whitespace),
+            (r'!.*?\n', Comment.Single),
+            (r'\(!(.|\n)*?!\)', Comment.Multiline),
+            (words((
+                'and', 'as', 'break', 'case', 'count', 'declarations', 'do',
+                'dynamic', 'elif', 'else', 'end-', 'end', 'evaluation', 'false',
+                'forall', 'forward', 'from', 'function', 'hashmap', 'if',
+                'imports', 'include', 'initialisations', 'initializations', 'inter',
+                'max', 'min', 'model', 'namespace', 'next', 'not', 'nsgroup',
+                'nssearch', 'of', 'options', 'or', 'package', 'parameters',
+                'procedure', 'public', 'prod', 'record', 'repeat', 'requirements',
+                'return', 'sum', 'then', 'to', 'true', 'union', 'until', 'uses',
+                'version', 'while', 'with'), prefix=r'\b', suffix=r'\b'),
+             Keyword.Builtin),
+            (words((
+                'range', 'array', 'set', 'list', 'mpvar', 'mpproblem', 'linctr',
+                'nlctr', 'integer', 'string', 'real', 'boolean', 'text', 'time',
+                'date', 'datetime', 'returned', 'Model', 'Mosel', 'counter',
+                'xmldoc', 'is_sos1', 'is_sos2', 'is_integer', 'is_binary',
+                'is_continuous', 'is_free', 'is_semcont', 'is_semint',
+                'is_partint'), prefix=r'\b', suffix=r'\b'),
+             Keyword.Type),
+            (r'(\+|\-|\*|/|=|<=|>=|\||\^|<|>|<>|\.\.|\.|:=|::|:|in|mod|div)',
+             Operator),
+            (r'[()\[\]{},;]+', Punctuation),
+            (words(FUNCTIONS,  prefix=r'\b', suffix=r'\b'), Name.Function),
+            (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
+            (r'\d+([eE][+-]?\d+)?', Number.Integer),
+            (r'[+-]?Infinity', Number.Integer),
+            (r'0[xX][0-9a-fA-F]+', Number),
+            (r'"', String.Double, 'double_quote'),
+            (r'\'', String.Single, 'single_quote'),
+            (r'(\w+|(\.(?!\.)))', Text),
+        ],
+        'single_quote': [
+            (r'\'', String.Single, '#pop'),
+            (r'[^\']+', String.Single),
+        ],
+        'double_quote': [
+            (r'(\\"|\\[0-7]{1,3}\D|\\[abfnrtv]|\\\\)', String.Escape),
+            (r'\"', String.Double, '#pop'),
+            (r'[^"\\]+', String.Double),
+        ],
+    }
diff --git a/lib/pygments/lexers/ncl.py b/lib/pygments/lexers/ncl.py
new file mode 100644
index 0000000..d2f4760
--- /dev/null
+++ b/lib/pygments/lexers/ncl.py
@@ -0,0 +1,894 @@
+"""
+    pygments.lexers.ncl
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for NCAR Command Language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['NCLLexer']
+
+
+class NCLLexer(RegexLexer):
+    """
+    Lexer for NCL code.
+    """
+    name = 'NCL'
+    aliases = ['ncl']
+    filenames = ['*.ncl']
+    mimetypes = ['text/ncl']
+    url = 'https://www.ncl.ucar.edu'
+    version_added = '2.2'
+
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r';.*\n', Comment),
+            include('strings'),
+            include('core'),
+            (r'[a-zA-Z_]\w*', Name),
+            include('nums'),
+            (r'[\s]+', Text),
+        ],
+        'core': [
+            # Statements
+            (words((
+                'begin', 'break', 'continue', 'create', 'defaultapp', 'do',
+                'else', 'end', 'external', 'exit', 'True', 'False', 'file', 'function',
+                'getvalues', 'graphic', 'group', 'if', 'list', 'load', 'local',
+                'new', '_Missing', 'Missing', 'noparent', 'procedure',
+                'quit', 'QUIT', 'Quit', 'record', 'return', 'setvalues', 'stop',
+                'then', 'while'), prefix=r'\b', suffix=r'\s*\b'),
+             Keyword),
+
+            # Data Types
+            (words((
+                'ubyte', 'uint', 'uint64', 'ulong', 'string', 'byte',
+                'character', 'double', 'float', 'integer', 'int64', 'logical',
+                'long', 'short', 'ushort', 'enumeric', 'numeric', 'snumeric'),
+                prefix=r'\b', suffix=r'\s*\b'),
+             Keyword.Type),
+
+            # Operators
+            (r'[\%^*+\-/<>]', Operator),
+
+            # punctuation:
+            (r'[\[\]():@$!&|.,\\{}]', Punctuation),
+            (r'[=:]', Punctuation),
+
+            # Intrinsics
+            (words((
+                'abs', 'acos', 'addfile', 'addfiles', 'all', 'angmom_atm', 'any',
+                'area_conserve_remap', 'area_hi2lores', 'area_poly_sphere',
+                'asciiread', 'asciiwrite', 'asin', 'atan', 'atan2', 'attsetvalues',
+                'avg', 'betainc', 'bin_avg', 'bin_sum', 'bw_bandpass_filter',
+                'cancor', 'cbinread', 'cbinwrite', 'cd_calendar', 'cd_inv_calendar',
+                'cdfbin_p', 'cdfbin_pr', 'cdfbin_s', 'cdfbin_xn', 'cdfchi_p',
+                'cdfchi_x', 'cdfgam_p', 'cdfgam_x', 'cdfnor_p', 'cdfnor_x',
+                'cdft_p', 'cdft_t', 'ceil', 'center_finite_diff',
+                'center_finite_diff_n', 'cfftb', 'cfftf', 'cfftf_frq_reorder',
+                'charactertodouble', 'charactertofloat', 'charactertointeger',
+                'charactertolong', 'charactertoshort', 'charactertostring',
+                'chartodouble', 'chartofloat', 'chartoint', 'chartointeger',
+                'chartolong', 'chartoshort', 'chartostring', 'chiinv', 'clear',
+                'color_index_to_rgba', 'conform', 'conform_dims', 'cos', 'cosh',
+                'count_unique_values', 'covcorm', 'covcorm_xy', 'craybinnumrec',
+                'craybinrecread', 'create_graphic', 'csa1', 'csa1d', 'csa1s',
+                'csa1x', 'csa1xd', 'csa1xs', 'csa2', 'csa2d', 'csa2l', 'csa2ld',
+                'csa2ls', 'csa2lx', 'csa2lxd', 'csa2lxs', 'csa2s', 'csa2x',
+                'csa2xd', 'csa2xs', 'csa3', 'csa3d', 'csa3l', 'csa3ld', 'csa3ls',
+                'csa3lx', 'csa3lxd', 'csa3lxs', 'csa3s', 'csa3x', 'csa3xd',
+                'csa3xs', 'csc2s', 'csgetp', 'css2c', 'cssetp', 'cssgrid', 'csstri',
+                'csvoro', 'cumsum', 'cz2ccm', 'datatondc', 'day_of_week',
+                'day_of_year', 'days_in_month', 'default_fillvalue', 'delete',
+                'depth_to_pres', 'destroy', 'determinant', 'dewtemp_trh',
+                'dgeevx_lapack', 'dim_acumrun_n', 'dim_avg', 'dim_avg_n',
+                'dim_avg_wgt', 'dim_avg_wgt_n', 'dim_cumsum', 'dim_cumsum_n',
+                'dim_gamfit_n', 'dim_gbits', 'dim_max', 'dim_max_n', 'dim_median',
+                'dim_median_n', 'dim_min', 'dim_min_n', 'dim_num', 'dim_num_n',
+                'dim_numrun_n', 'dim_pqsort', 'dim_pqsort_n', 'dim_product',
+                'dim_product_n', 'dim_rmsd', 'dim_rmsd_n', 'dim_rmvmean',
+                'dim_rmvmean_n', 'dim_rmvmed', 'dim_rmvmed_n', 'dim_spi_n',
+                'dim_standardize', 'dim_standardize_n', 'dim_stat4', 'dim_stat4_n',
+                'dim_stddev', 'dim_stddev_n', 'dim_sum', 'dim_sum_n', 'dim_sum_wgt',
+                'dim_sum_wgt_n', 'dim_variance', 'dim_variance_n', 'dimsizes',
+                'doubletobyte', 'doubletochar', 'doubletocharacter',
+                'doubletofloat', 'doubletoint', 'doubletointeger', 'doubletolong',
+                'doubletoshort', 'dpres_hybrid_ccm', 'dpres_plevel', 'draw',
+                'draw_color_palette', 'dsgetp', 'dsgrid2', 'dsgrid2d', 'dsgrid2s',
+                'dsgrid3', 'dsgrid3d', 'dsgrid3s', 'dspnt2', 'dspnt2d', 'dspnt2s',
+                'dspnt3', 'dspnt3d', 'dspnt3s', 'dssetp', 'dtrend', 'dtrend_msg',
+                'dtrend_msg_n', 'dtrend_n', 'dtrend_quadratic',
+                'dtrend_quadratic_msg_n', 'dv2uvf', 'dv2uvg', 'dz_height',
+                'echo_off', 'echo_on', 'eof2data', 'eof_varimax', 'eofcor',
+                'eofcor_pcmsg', 'eofcor_ts', 'eofcov', 'eofcov_pcmsg', 'eofcov_ts',
+                'eofunc', 'eofunc_ts', 'eofunc_varimax', 'equiv_sample_size', 'erf',
+                'erfc', 'esacr', 'esacv', 'esccr', 'esccv', 'escorc', 'escorc_n',
+                'escovc', 'exit', 'exp', 'exp_tapersh', 'exp_tapersh_wgts',
+                'exp_tapershC', 'ezfftb', 'ezfftb_n', 'ezfftf', 'ezfftf_n',
+                'f2fosh', 'f2foshv', 'f2fsh', 'f2fshv', 'f2gsh', 'f2gshv', 'fabs',
+                'fbindirread', 'fbindirwrite', 'fbinnumrec', 'fbinread',
+                'fbinrecread', 'fbinrecwrite', 'fbinwrite', 'fft2db', 'fft2df',
+                'fftshift', 'fileattdef', 'filechunkdimdef', 'filedimdef',
+                'fileexists', 'filegrpdef', 'filevarattdef', 'filevarchunkdef',
+                'filevarcompressleveldef', 'filevardef', 'filevardimsizes',
+                'filwgts_lancos', 'filwgts_lanczos', 'filwgts_normal',
+                'floattobyte', 'floattochar', 'floattocharacter', 'floattoint',
+                'floattointeger', 'floattolong', 'floattoshort', 'floor',
+                'fluxEddy', 'fo2fsh', 'fo2fshv', 'fourier_info', 'frame', 'fspan',
+                'ftcurv', 'ftcurvd', 'ftcurvi', 'ftcurvp', 'ftcurvpi', 'ftcurvps',
+                'ftcurvs', 'ftest', 'ftgetp', 'ftkurv', 'ftkurvd', 'ftkurvp',
+                'ftkurvpd', 'ftsetp', 'ftsurf', 'g2fsh', 'g2fshv', 'g2gsh',
+                'g2gshv', 'gamma', 'gammainc', 'gaus', 'gaus_lobat',
+                'gaus_lobat_wgt', 'gc_aangle', 'gc_clkwise', 'gc_dangle',
+                'gc_inout', 'gc_latlon', 'gc_onarc', 'gc_pnt2gc', 'gc_qarea',
+                'gc_tarea', 'generate_2d_array', 'get_color_index',
+                'get_color_rgba', 'get_cpu_time', 'get_isolines', 'get_ncl_version',
+                'get_script_name', 'get_script_prefix_name', 'get_sphere_radius',
+                'get_unique_values', 'getbitsone', 'getenv', 'getfiledimsizes',
+                'getfilegrpnames', 'getfilepath', 'getfilevaratts',
+                'getfilevarchunkdimsizes', 'getfilevardims', 'getfilevardimsizes',
+                'getfilevarnames', 'getfilevartypes', 'getvaratts', 'getvardims',
+                'gradsf', 'gradsg', 'greg2jul', 'grid2triple', 'hlsrgb', 'hsvrgb',
+                'hydro', 'hyi2hyo', 'idsfft', 'igradsf', 'igradsg', 'ilapsf',
+                'ilapsg', 'ilapvf', 'ilapvg', 'ind', 'ind_resolve', 'int2p',
+                'int2p_n', 'integertobyte', 'integertochar', 'integertocharacter',
+                'integertoshort', 'inttobyte', 'inttochar', 'inttoshort',
+                'inverse_matrix', 'isatt', 'isbigendian', 'isbyte', 'ischar',
+                'iscoord', 'isdefined', 'isdim', 'isdimnamed', 'isdouble',
+                'isenumeric', 'isfile', 'isfilepresent', 'isfilevar',
+                'isfilevaratt', 'isfilevarcoord', 'isfilevardim', 'isfloat',
+                'isfunc', 'isgraphic', 'isint', 'isint64', 'isinteger',
+                'isleapyear', 'islogical', 'islong', 'ismissing', 'isnan_ieee',
+                'isnumeric', 'ispan', 'isproc', 'isshort', 'issnumeric', 'isstring',
+                'isubyte', 'isuint', 'isuint64', 'isulong', 'isunlimited',
+                'isunsigned', 'isushort', 'isvar', 'jul2greg', 'kmeans_as136',
+                'kolsm2_n', 'kron_product', 'lapsf', 'lapsg', 'lapvf', 'lapvg',
+                'latlon2utm', 'lclvl', 'lderuvf', 'lderuvg', 'linint1', 'linint1_n',
+                'linint2', 'linint2_points', 'linmsg', 'linmsg_n', 'linrood_latwgt',
+                'linrood_wgt', 'list_files', 'list_filevars', 'list_hlus',
+                'list_procfuncs', 'list_vars', 'ListAppend', 'ListCount',
+                'ListGetType', 'ListIndex', 'ListIndexFromName', 'ListPop',
+                'ListPush', 'ListSetType', 'loadscript', 'local_max', 'local_min',
+                'log', 'log10', 'longtobyte', 'longtochar', 'longtocharacter',
+                'longtoint', 'longtointeger', 'longtoshort', 'lspoly', 'lspoly_n',
+                'mask', 'max', 'maxind', 'min', 'minind', 'mixed_layer_depth',
+                'mixhum_ptd', 'mixhum_ptrh', 'mjo_cross_coh2pha',
+                'mjo_cross_segment', 'moc_globe_atl', 'monthday', 'natgrid',
+                'natgridd', 'natgrids', 'ncargpath', 'ncargversion', 'ndctodata',
+                'ndtooned', 'new', 'NewList', 'ngezlogo', 'nggcog', 'nggetp',
+                'nglogo', 'ngsetp', 'NhlAddAnnotation', 'NhlAddData',
+                'NhlAddOverlay', 'NhlAddPrimitive', 'NhlAppGetDefaultParentId',
+                'NhlChangeWorkstation', 'NhlClassName', 'NhlClearWorkstation',
+                'NhlDataPolygon', 'NhlDataPolyline', 'NhlDataPolymarker',
+                'NhlDataToNDC', 'NhlDestroy', 'NhlDraw', 'NhlFrame', 'NhlFreeColor',
+                'NhlGetBB', 'NhlGetClassResources', 'NhlGetErrorObjectId',
+                'NhlGetNamedColorIndex', 'NhlGetParentId',
+                'NhlGetParentWorkstation', 'NhlGetWorkspaceObjectId',
+                'NhlIsAllocatedColor', 'NhlIsApp', 'NhlIsDataComm', 'NhlIsDataItem',
+                'NhlIsDataSpec', 'NhlIsTransform', 'NhlIsView', 'NhlIsWorkstation',
+                'NhlName', 'NhlNDCPolygon', 'NhlNDCPolyline', 'NhlNDCPolymarker',
+                'NhlNDCToData', 'NhlNewColor', 'NhlNewDashPattern', 'NhlNewMarker',
+                'NhlPalGetDefined', 'NhlRemoveAnnotation', 'NhlRemoveData',
+                'NhlRemoveOverlay', 'NhlRemovePrimitive', 'NhlSetColor',
+                'NhlSetDashPattern', 'NhlSetMarker', 'NhlUpdateData',
+                'NhlUpdateWorkstation', 'nice_mnmxintvl', 'nngetaspectd',
+                'nngetaspects', 'nngetp', 'nngetsloped', 'nngetslopes', 'nngetwts',
+                'nngetwtsd', 'nnpnt', 'nnpntd', 'nnpntend', 'nnpntendd',
+                'nnpntinit', 'nnpntinitd', 'nnpntinits', 'nnpnts', 'nnsetp', 'num',
+                'obj_anal_ic', 'omega_ccm', 'onedtond', 'overlay', 'paleo_outline',
+                'pdfxy_bin', 'poisson_grid_fill', 'pop_remap', 'potmp_insitu_ocn',
+                'prcwater_dp', 'pres2hybrid', 'pres_hybrid_ccm', 'pres_sigma',
+                'print', 'print_table', 'printFileVarSummary', 'printVarSummary',
+                'product', 'pslec', 'pslhor', 'pslhyp', 'qsort', 'rand',
+                'random_chi', 'random_gamma', 'random_normal', 'random_setallseed',
+                'random_uniform', 'rcm2points', 'rcm2rgrid', 'rdsstoi',
+                'read_colormap_file', 'reg_multlin', 'regcoef', 'regCoef_n',
+                'regline', 'relhum', 'replace_ieeenan', 'reshape', 'reshape_ind',
+                'rgba_to_color_index', 'rgbhls', 'rgbhsv', 'rgbyiq', 'rgrid2rcm',
+                'rhomb_trunc', 'rip_cape_2d', 'rip_cape_3d', 'round', 'rtest',
+                'runave', 'runave_n', 'set_default_fillvalue', 'set_sphere_radius',
+                'setfileoption', 'sfvp2uvf', 'sfvp2uvg', 'shaec', 'shagc',
+                'shgetnp', 'shgetp', 'shgrid', 'shorttobyte', 'shorttochar',
+                'shorttocharacter', 'show_ascii', 'shsec', 'shsetp', 'shsgc',
+                'shsgc_R42', 'sigma2hybrid', 'simpeq', 'simpne', 'sin',
+                'sindex_yrmo', 'sinh', 'sizeof', 'sleep', 'smth9', 'snindex_yrmo',
+                'solve_linsys', 'span_color_indexes', 'span_color_rgba',
+                'sparse_matrix_mult', 'spcorr', 'spcorr_n', 'specx_anal',
+                'specxy_anal', 'spei', 'sprintf', 'sprinti', 'sqrt', 'sqsort',
+                'srand', 'stat2', 'stat4', 'stat_medrng', 'stat_trim',
+                'status_exit', 'stdatmus_p2tdz', 'stdatmus_z2tdp', 'stddev',
+                'str_capital', 'str_concat', 'str_fields_count', 'str_get_cols',
+                'str_get_dq', 'str_get_field', 'str_get_nl', 'str_get_sq',
+                'str_get_tab', 'str_index_of_substr', 'str_insert', 'str_is_blank',
+                'str_join', 'str_left_strip', 'str_lower', 'str_match',
+                'str_match_ic', 'str_match_ic_regex', 'str_match_ind',
+                'str_match_ind_ic', 'str_match_ind_ic_regex', 'str_match_ind_regex',
+                'str_match_regex', 'str_right_strip', 'str_split',
+                'str_split_by_length', 'str_split_csv', 'str_squeeze', 'str_strip',
+                'str_sub_str', 'str_switch', 'str_upper', 'stringtochar',
+                'stringtocharacter', 'stringtodouble', 'stringtofloat',
+                'stringtoint', 'stringtointeger', 'stringtolong', 'stringtoshort',
+                'strlen', 'student_t', 'sum', 'svd_lapack', 'svdcov', 'svdcov_sv',
+                'svdstd', 'svdstd_sv', 'system', 'systemfunc', 'tan', 'tanh',
+                'taper', 'taper_n', 'tdclrs', 'tdctri', 'tdcudp', 'tdcurv',
+                'tddtri', 'tdez2d', 'tdez3d', 'tdgetp', 'tdgrds', 'tdgrid',
+                'tdgtrs', 'tdinit', 'tditri', 'tdlbla', 'tdlblp', 'tdlbls',
+                'tdline', 'tdlndp', 'tdlnpa', 'tdlpdp', 'tdmtri', 'tdotri',
+                'tdpara', 'tdplch', 'tdprpa', 'tdprpi', 'tdprpt', 'tdsetp',
+                'tdsort', 'tdstri', 'tdstrs', 'tdttri', 'thornthwaite', 'tobyte',
+                'tochar', 'todouble', 'tofloat', 'toint', 'toint64', 'tointeger',
+                'tolong', 'toshort', 'tosigned', 'tostring', 'tostring_with_format',
+                'totype', 'toubyte', 'touint', 'touint64', 'toulong', 'tounsigned',
+                'toushort', 'trend_manken', 'tri_trunc', 'triple2grid',
+                'triple2grid2d', 'trop_wmo', 'ttest', 'typeof', 'undef',
+                'unique_string', 'update', 'ushorttoint', 'ut_calendar',
+                'ut_inv_calendar', 'utm2latlon', 'uv2dv_cfd', 'uv2dvf', 'uv2dvg',
+                'uv2sfvpf', 'uv2sfvpg', 'uv2vr_cfd', 'uv2vrdvf', 'uv2vrdvg',
+                'uv2vrf', 'uv2vrg', 'v5d_close', 'v5d_create', 'v5d_setLowLev',
+                'v5d_setUnits', 'v5d_write', 'v5d_write_var', 'variance', 'vhaec',
+                'vhagc', 'vhsec', 'vhsgc', 'vibeta', 'vinth2p', 'vinth2p_ecmwf',
+                'vinth2p_ecmwf_nodes', 'vinth2p_nodes', 'vintp2p_ecmwf', 'vr2uvf',
+                'vr2uvg', 'vrdv2uvf', 'vrdv2uvg', 'wavelet', 'wavelet_default',
+                'weibull', 'wgt_area_smooth', 'wgt_areaave', 'wgt_areaave2',
+                'wgt_arearmse', 'wgt_arearmse2', 'wgt_areasum2', 'wgt_runave',
+                'wgt_runave_n', 'wgt_vert_avg_beta', 'wgt_volave', 'wgt_volave_ccm',
+                'wgt_volrmse', 'wgt_volrmse_ccm', 'where', 'wk_smooth121', 'wmbarb',
+                'wmbarbmap', 'wmdrft', 'wmgetp', 'wmlabs', 'wmsetp', 'wmstnm',
+                'wmvect', 'wmvectmap', 'wmvlbl', 'wrf_avo', 'wrf_cape_2d',
+                'wrf_cape_3d', 'wrf_dbz', 'wrf_eth', 'wrf_helicity', 'wrf_ij_to_ll',
+                'wrf_interp_1d', 'wrf_interp_2d_xy', 'wrf_interp_3d_z',
+                'wrf_latlon_to_ij', 'wrf_ll_to_ij', 'wrf_omega', 'wrf_pvo',
+                'wrf_rh', 'wrf_slp', 'wrf_smooth_2d', 'wrf_td', 'wrf_tk',
+                'wrf_updraft_helicity', 'wrf_uvmet', 'wrf_virtual_temp',
+                'wrf_wetbulb', 'wrf_wps_close_int', 'wrf_wps_open_int',
+                'wrf_wps_rddata_int', 'wrf_wps_rdhead_int', 'wrf_wps_read_int',
+                'wrf_wps_write_int', 'write_matrix', 'write_table', 'yiqrgb',
+                'z2geouv', 'zonal_mpsi', 'addfiles_GetVar', 'advect_variable',
+                'area_conserve_remap_Wrap', 'area_hi2lores_Wrap',
+                'array_append_record', 'assignFillValue', 'byte2flt',
+                'byte2flt_hdf', 'calcDayAnomTLL', 'calcMonAnomLLLT',
+                'calcMonAnomLLT', 'calcMonAnomTLL', 'calcMonAnomTLLL',
+                'calculate_monthly_values', 'cd_convert', 'changeCase',
+                'changeCaseChar', 'clmDayTLL', 'clmDayTLLL', 'clmMon2clmDay',
+                'clmMonLLLT', 'clmMonLLT', 'clmMonTLL', 'clmMonTLLL', 'closest_val',
+                'copy_VarAtts', 'copy_VarCoords', 'copy_VarCoords_1',
+                'copy_VarCoords_2', 'copy_VarMeta', 'copyatt', 'crossp3',
+                'cshstringtolist', 'cssgrid_Wrap', 'dble2flt', 'decimalPlaces',
+                'delete_VarAtts', 'dim_avg_n_Wrap', 'dim_avg_wgt_n_Wrap',
+                'dim_avg_wgt_Wrap', 'dim_avg_Wrap', 'dim_cumsum_n_Wrap',
+                'dim_cumsum_Wrap', 'dim_max_n_Wrap', 'dim_min_n_Wrap',
+                'dim_rmsd_n_Wrap', 'dim_rmsd_Wrap', 'dim_rmvmean_n_Wrap',
+                'dim_rmvmean_Wrap', 'dim_rmvmed_n_Wrap', 'dim_rmvmed_Wrap',
+                'dim_standardize_n_Wrap', 'dim_standardize_Wrap',
+                'dim_stddev_n_Wrap', 'dim_stddev_Wrap', 'dim_sum_n_Wrap',
+                'dim_sum_wgt_n_Wrap', 'dim_sum_wgt_Wrap', 'dim_sum_Wrap',
+                'dim_variance_n_Wrap', 'dim_variance_Wrap', 'dpres_plevel_Wrap',
+                'dtrend_leftdim', 'dv2uvF_Wrap', 'dv2uvG_Wrap', 'eof_north',
+                'eofcor_Wrap', 'eofcov_Wrap', 'eofunc_north', 'eofunc_ts_Wrap',
+                'eofunc_varimax_reorder', 'eofunc_varimax_Wrap', 'eofunc_Wrap',
+                'epsZero', 'f2fosh_Wrap', 'f2foshv_Wrap', 'f2fsh_Wrap',
+                'f2fshv_Wrap', 'f2gsh_Wrap', 'f2gshv_Wrap', 'fbindirSwap',
+                'fbinseqSwap1', 'fbinseqSwap2', 'flt2dble', 'flt2string',
+                'fo2fsh_Wrap', 'fo2fshv_Wrap', 'g2fsh_Wrap', 'g2fshv_Wrap',
+                'g2gsh_Wrap', 'g2gshv_Wrap', 'generate_resample_indices',
+                'generate_sample_indices', 'generate_unique_indices',
+                'genNormalDist', 'get1Dindex', 'get1Dindex_Collapse',
+                'get1Dindex_Exclude', 'get_file_suffix', 'GetFillColor',
+                'GetFillColorIndex', 'getFillValue', 'getind_latlon2d',
+                'getVarDimNames', 'getVarFillValue', 'grib_stime2itime',
+                'hyi2hyo_Wrap', 'ilapsF_Wrap', 'ilapsG_Wrap', 'ind_nearest_coord',
+                'indStrSubset', 'int2dble', 'int2flt', 'int2p_n_Wrap', 'int2p_Wrap',
+                'isMonotonic', 'isStrSubset', 'latGau', 'latGauWgt', 'latGlobeF',
+                'latGlobeFo', 'latRegWgt', 'linint1_n_Wrap', 'linint1_Wrap',
+                'linint2_points_Wrap', 'linint2_Wrap', 'local_max_1d',
+                'local_min_1d', 'lonFlip', 'lonGlobeF', 'lonGlobeFo', 'lonPivot',
+                'merge_levels_sfc', 'mod', 'month_to_annual',
+                'month_to_annual_weighted', 'month_to_season', 'month_to_season12',
+                'month_to_seasonN', 'monthly_total_to_daily_mean', 'nameDim',
+                'natgrid_Wrap', 'NewCosWeight', 'niceLatLon2D', 'NormCosWgtGlobe',
+                'numAsciiCol', 'numAsciiRow', 'numeric2int',
+                'obj_anal_ic_deprecated', 'obj_anal_ic_Wrap', 'omega_ccm_driver',
+                'omega_to_w', 'oneDtostring', 'pack_values', 'pattern_cor', 'pdfx',
+                'pdfxy', 'pdfxy_conform', 'pot_temp', 'pot_vort_hybrid',
+                'pot_vort_isobaric', 'pres2hybrid_Wrap', 'print_clock',
+                'printMinMax', 'quadroots', 'rcm2points_Wrap', 'rcm2rgrid_Wrap',
+                'readAsciiHead', 'readAsciiTable', 'reg_multlin_stats',
+                'region_ind', 'regline_stats', 'relhum_ttd', 'replaceSingleChar',
+                'RGBtoCmap', 'rgrid2rcm_Wrap', 'rho_mwjf', 'rm_single_dims',
+                'rmAnnCycle1D', 'rmInsufData', 'rmMonAnnCycLLLT', 'rmMonAnnCycLLT',
+                'rmMonAnnCycTLL', 'runave_n_Wrap', 'runave_Wrap', 'short2flt',
+                'short2flt_hdf', 'shsgc_R42_Wrap', 'sign_f90', 'sign_matlab',
+                'smth9_Wrap', 'smthClmDayTLL', 'smthClmDayTLLL', 'SqrtCosWeight',
+                'stat_dispersion', 'static_stability', 'stdMonLLLT', 'stdMonLLT',
+                'stdMonTLL', 'stdMonTLLL', 'symMinMaxPlt', 'table_attach_columns',
+                'table_attach_rows', 'time_to_newtime', 'transpose',
+                'triple2grid_Wrap', 'ut_convert', 'uv2dvF_Wrap', 'uv2dvG_Wrap',
+                'uv2vrF_Wrap', 'uv2vrG_Wrap', 'vr2uvF_Wrap', 'vr2uvG_Wrap',
+                'w_to_omega', 'wallClockElapseTime', 'wave_number_spc',
+                'wgt_areaave_Wrap', 'wgt_runave_leftdim', 'wgt_runave_n_Wrap',
+                'wgt_runave_Wrap', 'wgt_vertical_n', 'wind_component',
+                'wind_direction', 'yyyyddd_to_yyyymmdd', 'yyyymm_time',
+                'yyyymm_to_yyyyfrac', 'yyyymmdd_time', 'yyyymmdd_to_yyyyddd',
+                'yyyymmdd_to_yyyyfrac', 'yyyymmddhh_time', 'yyyymmddhh_to_yyyyfrac',
+                'zonal_mpsi_Wrap', 'zonalAve', 'calendar_decode2', 'cd_string',
+                'kf_filter', 'run_cor', 'time_axis_labels', 'ut_string',
+                'wrf_contour', 'wrf_map', 'wrf_map_overlay', 'wrf_map_overlays',
+                'wrf_map_resources', 'wrf_map_zoom', 'wrf_overlay', 'wrf_overlays',
+                'wrf_user_getvar', 'wrf_user_ij_to_ll', 'wrf_user_intrp2d',
+                'wrf_user_intrp3d', 'wrf_user_latlon_to_ij', 'wrf_user_list_times',
+                'wrf_user_ll_to_ij', 'wrf_user_unstagger', 'wrf_user_vert_interp',
+                'wrf_vector', 'gsn_add_annotation', 'gsn_add_polygon',
+                'gsn_add_polyline', 'gsn_add_polymarker',
+                'gsn_add_shapefile_polygons', 'gsn_add_shapefile_polylines',
+                'gsn_add_shapefile_polymarkers', 'gsn_add_text', 'gsn_attach_plots',
+                'gsn_blank_plot', 'gsn_contour', 'gsn_contour_map',
+                'gsn_contour_shade', 'gsn_coordinates', 'gsn_create_labelbar',
+                'gsn_create_legend', 'gsn_create_text',
+                'gsn_csm_attach_zonal_means', 'gsn_csm_blank_plot',
+                'gsn_csm_contour', 'gsn_csm_contour_map', 'gsn_csm_contour_map_ce',
+                'gsn_csm_contour_map_overlay', 'gsn_csm_contour_map_polar',
+                'gsn_csm_hov', 'gsn_csm_lat_time', 'gsn_csm_map', 'gsn_csm_map_ce',
+                'gsn_csm_map_polar', 'gsn_csm_pres_hgt',
+                'gsn_csm_pres_hgt_streamline', 'gsn_csm_pres_hgt_vector',
+                'gsn_csm_streamline', 'gsn_csm_streamline_contour_map',
+                'gsn_csm_streamline_contour_map_ce',
+                'gsn_csm_streamline_contour_map_polar', 'gsn_csm_streamline_map',
+                'gsn_csm_streamline_map_ce', 'gsn_csm_streamline_map_polar',
+                'gsn_csm_streamline_scalar', 'gsn_csm_streamline_scalar_map',
+                'gsn_csm_streamline_scalar_map_ce',
+                'gsn_csm_streamline_scalar_map_polar', 'gsn_csm_time_lat',
+                'gsn_csm_vector', 'gsn_csm_vector_map', 'gsn_csm_vector_map_ce',
+                'gsn_csm_vector_map_polar', 'gsn_csm_vector_scalar',
+                'gsn_csm_vector_scalar_map', 'gsn_csm_vector_scalar_map_ce',
+                'gsn_csm_vector_scalar_map_polar', 'gsn_csm_x2y', 'gsn_csm_x2y2',
+                'gsn_csm_xy', 'gsn_csm_xy2', 'gsn_csm_xy3', 'gsn_csm_y',
+                'gsn_define_colormap', 'gsn_draw_colormap', 'gsn_draw_named_colors',
+                'gsn_histogram', 'gsn_labelbar_ndc', 'gsn_legend_ndc', 'gsn_map',
+                'gsn_merge_colormaps', 'gsn_open_wks', 'gsn_panel', 'gsn_polygon',
+                'gsn_polygon_ndc', 'gsn_polyline', 'gsn_polyline_ndc',
+                'gsn_polymarker', 'gsn_polymarker_ndc', 'gsn_retrieve_colormap',
+                'gsn_reverse_colormap', 'gsn_streamline', 'gsn_streamline_map',
+                'gsn_streamline_scalar', 'gsn_streamline_scalar_map', 'gsn_table',
+                'gsn_text', 'gsn_text_ndc', 'gsn_vector', 'gsn_vector_map',
+                'gsn_vector_scalar', 'gsn_vector_scalar_map', 'gsn_xy', 'gsn_y',
+                'hsv2rgb', 'maximize_output', 'namedcolor2rgb', 'namedcolor2rgba',
+                'reset_device_coordinates', 'span_named_colors'), prefix=r'\b'),
+             Name.Builtin),
+
+            # Resources
+            (words((
+                'amDataXF', 'amDataYF', 'amJust', 'amOn', 'amOrthogonalPosF',
+                'amParallelPosF', 'amResizeNotify', 'amSide', 'amTrackData',
+                'amViewId', 'amZone', 'appDefaultParent', 'appFileSuffix',
+                'appResources', 'appSysDir', 'appUsrDir', 'caCopyArrays',
+                'caXArray', 'caXCast', 'caXMaxV', 'caXMinV', 'caXMissingV',
+                'caYArray', 'caYCast', 'caYMaxV', 'caYMinV', 'caYMissingV',
+                'cnCellFillEdgeColor', 'cnCellFillMissingValEdgeColor',
+                'cnConpackParams', 'cnConstFEnableFill', 'cnConstFLabelAngleF',
+                'cnConstFLabelBackgroundColor', 'cnConstFLabelConstantSpacingF',
+                'cnConstFLabelFont', 'cnConstFLabelFontAspectF',
+                'cnConstFLabelFontColor', 'cnConstFLabelFontHeightF',
+                'cnConstFLabelFontQuality', 'cnConstFLabelFontThicknessF',
+                'cnConstFLabelFormat', 'cnConstFLabelFuncCode', 'cnConstFLabelJust',
+                'cnConstFLabelOn', 'cnConstFLabelOrthogonalPosF',
+                'cnConstFLabelParallelPosF', 'cnConstFLabelPerimColor',
+                'cnConstFLabelPerimOn', 'cnConstFLabelPerimSpaceF',
+                'cnConstFLabelPerimThicknessF', 'cnConstFLabelSide',
+                'cnConstFLabelString', 'cnConstFLabelTextDirection',
+                'cnConstFLabelZone', 'cnConstFUseInfoLabelRes',
+                'cnExplicitLabelBarLabelsOn', 'cnExplicitLegendLabelsOn',
+                'cnExplicitLineLabelsOn', 'cnFillBackgroundColor', 'cnFillColor',
+                'cnFillColors', 'cnFillDotSizeF', 'cnFillDrawOrder', 'cnFillMode',
+                'cnFillOn', 'cnFillOpacityF', 'cnFillPalette', 'cnFillPattern',
+                'cnFillPatterns', 'cnFillScaleF', 'cnFillScales', 'cnFixFillBleed',
+                'cnGridBoundFillColor', 'cnGridBoundFillPattern',
+                'cnGridBoundFillScaleF', 'cnGridBoundPerimColor',
+                'cnGridBoundPerimDashPattern', 'cnGridBoundPerimOn',
+                'cnGridBoundPerimThicknessF', 'cnHighLabelAngleF',
+                'cnHighLabelBackgroundColor', 'cnHighLabelConstantSpacingF',
+                'cnHighLabelCount', 'cnHighLabelFont', 'cnHighLabelFontAspectF',
+                'cnHighLabelFontColor', 'cnHighLabelFontHeightF',
+                'cnHighLabelFontQuality', 'cnHighLabelFontThicknessF',
+                'cnHighLabelFormat', 'cnHighLabelFuncCode', 'cnHighLabelPerimColor',
+                'cnHighLabelPerimOn', 'cnHighLabelPerimSpaceF',
+                'cnHighLabelPerimThicknessF', 'cnHighLabelString', 'cnHighLabelsOn',
+                'cnHighLowLabelOverlapMode', 'cnHighUseLineLabelRes',
+                'cnInfoLabelAngleF', 'cnInfoLabelBackgroundColor',
+                'cnInfoLabelConstantSpacingF', 'cnInfoLabelFont',
+                'cnInfoLabelFontAspectF', 'cnInfoLabelFontColor',
+                'cnInfoLabelFontHeightF', 'cnInfoLabelFontQuality',
+                'cnInfoLabelFontThicknessF', 'cnInfoLabelFormat',
+                'cnInfoLabelFuncCode', 'cnInfoLabelJust', 'cnInfoLabelOn',
+                'cnInfoLabelOrthogonalPosF', 'cnInfoLabelParallelPosF',
+                'cnInfoLabelPerimColor', 'cnInfoLabelPerimOn',
+                'cnInfoLabelPerimSpaceF', 'cnInfoLabelPerimThicknessF',
+                'cnInfoLabelSide', 'cnInfoLabelString', 'cnInfoLabelTextDirection',
+                'cnInfoLabelZone', 'cnLabelBarEndLabelsOn', 'cnLabelBarEndStyle',
+                'cnLabelDrawOrder', 'cnLabelMasking', 'cnLabelScaleFactorF',
+                'cnLabelScaleValueF', 'cnLabelScalingMode', 'cnLegendLevelFlags',
+                'cnLevelCount', 'cnLevelFlag', 'cnLevelFlags', 'cnLevelSelectionMode',
+                'cnLevelSpacingF', 'cnLevels', 'cnLineColor', 'cnLineColors',
+                'cnLineDashPattern', 'cnLineDashPatterns', 'cnLineDashSegLenF',
+                'cnLineDrawOrder', 'cnLineLabelAngleF', 'cnLineLabelBackgroundColor',
+                'cnLineLabelConstantSpacingF', 'cnLineLabelCount',
+                'cnLineLabelDensityF', 'cnLineLabelFont', 'cnLineLabelFontAspectF',
+                'cnLineLabelFontColor', 'cnLineLabelFontColors',
+                'cnLineLabelFontHeightF', 'cnLineLabelFontQuality',
+                'cnLineLabelFontThicknessF', 'cnLineLabelFormat',
+                'cnLineLabelFuncCode', 'cnLineLabelInterval', 'cnLineLabelPerimColor',
+                'cnLineLabelPerimOn', 'cnLineLabelPerimSpaceF',
+                'cnLineLabelPerimThicknessF', 'cnLineLabelPlacementMode',
+                'cnLineLabelStrings', 'cnLineLabelsOn', 'cnLinePalette',
+                'cnLineThicknessF', 'cnLineThicknesses', 'cnLinesOn',
+                'cnLowLabelAngleF', 'cnLowLabelBackgroundColor',
+                'cnLowLabelConstantSpacingF', 'cnLowLabelCount', 'cnLowLabelFont',
+                'cnLowLabelFontAspectF', 'cnLowLabelFontColor',
+                'cnLowLabelFontHeightF', 'cnLowLabelFontQuality',
+                'cnLowLabelFontThicknessF', 'cnLowLabelFormat', 'cnLowLabelFuncCode',
+                'cnLowLabelPerimColor', 'cnLowLabelPerimOn', 'cnLowLabelPerimSpaceF',
+                'cnLowLabelPerimThicknessF', 'cnLowLabelString', 'cnLowLabelsOn',
+                'cnLowUseHighLabelRes', 'cnMaxDataValueFormat', 'cnMaxLevelCount',
+                'cnMaxLevelValF', 'cnMaxPointDistanceF', 'cnMinLevelValF',
+                'cnMissingValFillColor', 'cnMissingValFillPattern',
+                'cnMissingValFillScaleF', 'cnMissingValPerimColor',
+                'cnMissingValPerimDashPattern', 'cnMissingValPerimGridBoundOn',
+                'cnMissingValPerimOn', 'cnMissingValPerimThicknessF',
+                'cnMonoFillColor', 'cnMonoFillPattern', 'cnMonoFillScale',
+                'cnMonoLevelFlag', 'cnMonoLineColor', 'cnMonoLineDashPattern',
+                'cnMonoLineLabelFontColor', 'cnMonoLineThickness', 'cnNoDataLabelOn',
+                'cnNoDataLabelString', 'cnOutOfRangeFillColor',
+                'cnOutOfRangeFillPattern', 'cnOutOfRangeFillScaleF',
+                'cnOutOfRangePerimColor', 'cnOutOfRangePerimDashPattern',
+                'cnOutOfRangePerimOn', 'cnOutOfRangePerimThicknessF',
+                'cnRasterCellSizeF', 'cnRasterMinCellSizeF', 'cnRasterModeOn',
+                'cnRasterSampleFactorF', 'cnRasterSmoothingOn', 'cnScalarFieldData',
+                'cnSmoothingDistanceF', 'cnSmoothingOn', 'cnSmoothingTensionF',
+                'cnSpanFillPalette', 'cnSpanLinePalette', 'ctCopyTables',
+                'ctXElementSize', 'ctXMaxV', 'ctXMinV', 'ctXMissingV', 'ctXTable',
+                'ctXTableLengths', 'ctXTableType', 'ctYElementSize', 'ctYMaxV',
+                'ctYMinV', 'ctYMissingV', 'ctYTable', 'ctYTableLengths',
+                'ctYTableType', 'dcDelayCompute', 'errBuffer',
+                'errFileName', 'errFilePtr', 'errLevel', 'errPrint', 'errUnitNumber',
+                'gsClipOn', 'gsColors', 'gsEdgeColor', 'gsEdgeDashPattern',
+                'gsEdgeDashSegLenF', 'gsEdgeThicknessF', 'gsEdgesOn',
+                'gsFillBackgroundColor', 'gsFillColor', 'gsFillDotSizeF',
+                'gsFillIndex', 'gsFillLineThicknessF', 'gsFillOpacityF',
+                'gsFillScaleF', 'gsFont', 'gsFontAspectF', 'gsFontColor',
+                'gsFontHeightF', 'gsFontOpacityF', 'gsFontQuality',
+                'gsFontThicknessF', 'gsLineColor', 'gsLineDashPattern',
+                'gsLineDashSegLenF', 'gsLineLabelConstantSpacingF', 'gsLineLabelFont',
+                'gsLineLabelFontAspectF', 'gsLineLabelFontColor',
+                'gsLineLabelFontHeightF', 'gsLineLabelFontQuality',
+                'gsLineLabelFontThicknessF', 'gsLineLabelFuncCode',
+                'gsLineLabelString', 'gsLineOpacityF', 'gsLineThicknessF',
+                'gsMarkerColor', 'gsMarkerIndex', 'gsMarkerOpacityF', 'gsMarkerSizeF',
+                'gsMarkerThicknessF', 'gsSegments', 'gsTextAngleF',
+                'gsTextConstantSpacingF', 'gsTextDirection', 'gsTextFuncCode',
+                'gsTextJustification', 'gsnAboveYRefLineBarColors',
+                'gsnAboveYRefLineBarFillScales', 'gsnAboveYRefLineBarPatterns',
+                'gsnAboveYRefLineColor', 'gsnAddCyclic', 'gsnAttachBorderOn',
+                'gsnAttachPlotsXAxis', 'gsnBelowYRefLineBarColors',
+                'gsnBelowYRefLineBarFillScales', 'gsnBelowYRefLineBarPatterns',
+                'gsnBelowYRefLineColor', 'gsnBoxMargin', 'gsnCenterString',
+                'gsnCenterStringFontColor', 'gsnCenterStringFontHeightF',
+                'gsnCenterStringFuncCode', 'gsnCenterStringOrthogonalPosF',
+                'gsnCenterStringParallelPosF', 'gsnContourLineThicknessesScale',
+                'gsnContourNegLineDashPattern', 'gsnContourPosLineDashPattern',
+                'gsnContourZeroLineThicknessF', 'gsnDebugWriteFileName', 'gsnDraw',
+                'gsnFrame', 'gsnHistogramBarWidthPercent', 'gsnHistogramBinIntervals',
+                'gsnHistogramBinMissing', 'gsnHistogramBinWidth',
+                'gsnHistogramClassIntervals', 'gsnHistogramCompare',
+                'gsnHistogramComputePercentages',
+                'gsnHistogramComputePercentagesNoMissing',
+                'gsnHistogramDiscreteBinValues', 'gsnHistogramDiscreteClassValues',
+                'gsnHistogramHorizontal', 'gsnHistogramMinMaxBinsOn',
+                'gsnHistogramNumberOfBins', 'gsnHistogramPercentSign',
+                'gsnHistogramSelectNiceIntervals', 'gsnLeftString',
+                'gsnLeftStringFontColor', 'gsnLeftStringFontHeightF',
+                'gsnLeftStringFuncCode', 'gsnLeftStringOrthogonalPosF',
+                'gsnLeftStringParallelPosF', 'gsnMajorLatSpacing',
+                'gsnMajorLonSpacing', 'gsnMaskLambertConformal',
+                'gsnMaskLambertConformalOutlineOn', 'gsnMaximize',
+                'gsnMinorLatSpacing', 'gsnMinorLonSpacing', 'gsnPanelBottom',
+                'gsnPanelCenter', 'gsnPanelDebug', 'gsnPanelFigureStrings',
+                'gsnPanelFigureStringsBackgroundFillColor',
+                'gsnPanelFigureStringsFontHeightF', 'gsnPanelFigureStringsJust',
+                'gsnPanelFigureStringsPerimOn', 'gsnPanelLabelBar', 'gsnPanelLeft',
+                'gsnPanelMainFont', 'gsnPanelMainFontColor',
+                'gsnPanelMainFontHeightF', 'gsnPanelMainString', 'gsnPanelRight',
+                'gsnPanelRowSpec', 'gsnPanelScalePlotIndex', 'gsnPanelTop',
+                'gsnPanelXF', 'gsnPanelXWhiteSpacePercent', 'gsnPanelYF',
+                'gsnPanelYWhiteSpacePercent', 'gsnPaperHeight', 'gsnPaperMargin',
+                'gsnPaperOrientation', 'gsnPaperWidth', 'gsnPolar',
+                'gsnPolarLabelDistance', 'gsnPolarLabelFont',
+                'gsnPolarLabelFontHeightF', 'gsnPolarLabelSpacing', 'gsnPolarTime',
+                'gsnPolarUT', 'gsnRightString', 'gsnRightStringFontColor',
+                'gsnRightStringFontHeightF', 'gsnRightStringFuncCode',
+                'gsnRightStringOrthogonalPosF', 'gsnRightStringParallelPosF',
+                'gsnScalarContour', 'gsnScale', 'gsnShape', 'gsnSpreadColorEnd',
+                'gsnSpreadColorStart', 'gsnSpreadColors', 'gsnStringFont',
+                'gsnStringFontColor', 'gsnStringFontHeightF', 'gsnStringFuncCode',
+                'gsnTickMarksOn', 'gsnXAxisIrregular2Linear', 'gsnXAxisIrregular2Log',
+                'gsnXRefLine', 'gsnXRefLineColor', 'gsnXRefLineDashPattern',
+                'gsnXRefLineThicknessF', 'gsnXYAboveFillColors', 'gsnXYBarChart',
+                'gsnXYBarChartBarWidth', 'gsnXYBarChartColors',
+                'gsnXYBarChartColors2', 'gsnXYBarChartFillDotSizeF',
+                'gsnXYBarChartFillLineThicknessF', 'gsnXYBarChartFillOpacityF',
+                'gsnXYBarChartFillScaleF', 'gsnXYBarChartOutlineOnly',
+                'gsnXYBarChartOutlineThicknessF', 'gsnXYBarChartPatterns',
+                'gsnXYBarChartPatterns2', 'gsnXYBelowFillColors', 'gsnXYFillColors',
+                'gsnXYFillOpacities', 'gsnXYLeftFillColors', 'gsnXYRightFillColors',
+                'gsnYAxisIrregular2Linear', 'gsnYAxisIrregular2Log', 'gsnYRefLine',
+                'gsnYRefLineColor', 'gsnYRefLineColors', 'gsnYRefLineDashPattern',
+                'gsnYRefLineDashPatterns', 'gsnYRefLineThicknessF',
+                'gsnYRefLineThicknesses', 'gsnZonalMean', 'gsnZonalMeanXMaxF',
+                'gsnZonalMeanXMinF', 'gsnZonalMeanYRefLine', 'lbAutoManage',
+                'lbBottomMarginF', 'lbBoxCount', 'lbBoxEndCapStyle', 'lbBoxFractions',
+                'lbBoxLineColor', 'lbBoxLineDashPattern', 'lbBoxLineDashSegLenF',
+                'lbBoxLineThicknessF', 'lbBoxLinesOn', 'lbBoxMajorExtentF',
+                'lbBoxMinorExtentF', 'lbBoxSeparatorLinesOn', 'lbBoxSizing',
+                'lbFillBackground', 'lbFillColor', 'lbFillColors', 'lbFillDotSizeF',
+                'lbFillLineThicknessF', 'lbFillPattern', 'lbFillPatterns',
+                'lbFillScaleF', 'lbFillScales', 'lbJustification', 'lbLabelAlignment',
+                'lbLabelAngleF', 'lbLabelAutoStride', 'lbLabelBarOn',
+                'lbLabelConstantSpacingF', 'lbLabelDirection', 'lbLabelFont',
+                'lbLabelFontAspectF', 'lbLabelFontColor', 'lbLabelFontHeightF',
+                'lbLabelFontQuality', 'lbLabelFontThicknessF', 'lbLabelFuncCode',
+                'lbLabelJust', 'lbLabelOffsetF', 'lbLabelPosition', 'lbLabelStride',
+                'lbLabelStrings', 'lbLabelsOn', 'lbLeftMarginF', 'lbMaxLabelLenF',
+                'lbMinLabelSpacingF', 'lbMonoFillColor', 'lbMonoFillPattern',
+                'lbMonoFillScale', 'lbOrientation', 'lbPerimColor',
+                'lbPerimDashPattern', 'lbPerimDashSegLenF', 'lbPerimFill',
+                'lbPerimFillColor', 'lbPerimOn', 'lbPerimThicknessF',
+                'lbRasterFillOn', 'lbRightMarginF', 'lbTitleAngleF',
+                'lbTitleConstantSpacingF', 'lbTitleDirection', 'lbTitleExtentF',
+                'lbTitleFont', 'lbTitleFontAspectF', 'lbTitleFontColor',
+                'lbTitleFontHeightF', 'lbTitleFontQuality', 'lbTitleFontThicknessF',
+                'lbTitleFuncCode', 'lbTitleJust', 'lbTitleOffsetF', 'lbTitleOn',
+                'lbTitlePosition', 'lbTitleString', 'lbTopMarginF', 'lgAutoManage',
+                'lgBottomMarginF', 'lgBoxBackground', 'lgBoxLineColor',
+                'lgBoxLineDashPattern', 'lgBoxLineDashSegLenF', 'lgBoxLineThicknessF',
+                'lgBoxLinesOn', 'lgBoxMajorExtentF', 'lgBoxMinorExtentF',
+                'lgDashIndex', 'lgDashIndexes', 'lgItemCount', 'lgItemOrder',
+                'lgItemPlacement', 'lgItemPositions', 'lgItemType', 'lgItemTypes',
+                'lgJustification', 'lgLabelAlignment', 'lgLabelAngleF',
+                'lgLabelAutoStride', 'lgLabelConstantSpacingF', 'lgLabelDirection',
+                'lgLabelFont', 'lgLabelFontAspectF', 'lgLabelFontColor',
+                'lgLabelFontHeightF', 'lgLabelFontQuality', 'lgLabelFontThicknessF',
+                'lgLabelFuncCode', 'lgLabelJust', 'lgLabelOffsetF', 'lgLabelPosition',
+                'lgLabelStride', 'lgLabelStrings', 'lgLabelsOn', 'lgLeftMarginF',
+                'lgLegendOn', 'lgLineColor', 'lgLineColors', 'lgLineDashSegLenF',
+                'lgLineDashSegLens', 'lgLineLabelConstantSpacingF', 'lgLineLabelFont',
+                'lgLineLabelFontAspectF', 'lgLineLabelFontColor',
+                'lgLineLabelFontColors', 'lgLineLabelFontHeightF',
+                'lgLineLabelFontHeights', 'lgLineLabelFontQuality',
+                'lgLineLabelFontThicknessF', 'lgLineLabelFuncCode',
+                'lgLineLabelStrings', 'lgLineLabelsOn', 'lgLineThicknessF',
+                'lgLineThicknesses', 'lgMarkerColor', 'lgMarkerColors',
+                'lgMarkerIndex', 'lgMarkerIndexes', 'lgMarkerSizeF', 'lgMarkerSizes',
+                'lgMarkerThicknessF', 'lgMarkerThicknesses', 'lgMonoDashIndex',
+                'lgMonoItemType', 'lgMonoLineColor', 'lgMonoLineDashSegLen',
+                'lgMonoLineLabelFontColor', 'lgMonoLineLabelFontHeight',
+                'lgMonoLineThickness', 'lgMonoMarkerColor', 'lgMonoMarkerIndex',
+                'lgMonoMarkerSize', 'lgMonoMarkerThickness', 'lgOrientation',
+                'lgPerimColor', 'lgPerimDashPattern', 'lgPerimDashSegLenF',
+                'lgPerimFill', 'lgPerimFillColor', 'lgPerimOn', 'lgPerimThicknessF',
+                'lgRightMarginF', 'lgTitleAngleF', 'lgTitleConstantSpacingF',
+                'lgTitleDirection', 'lgTitleExtentF', 'lgTitleFont',
+                'lgTitleFontAspectF', 'lgTitleFontColor', 'lgTitleFontHeightF',
+                'lgTitleFontQuality', 'lgTitleFontThicknessF', 'lgTitleFuncCode',
+                'lgTitleJust', 'lgTitleOffsetF', 'lgTitleOn', 'lgTitlePosition',
+                'lgTitleString', 'lgTopMarginF', 'mpAreaGroupCount',
+                'mpAreaMaskingOn', 'mpAreaNames', 'mpAreaTypes', 'mpBottomAngleF',
+                'mpBottomMapPosF', 'mpBottomNDCF', 'mpBottomNPCF',
+                'mpBottomPointLatF', 'mpBottomPointLonF', 'mpBottomWindowF',
+                'mpCenterLatF', 'mpCenterLonF', 'mpCenterRotF', 'mpCountyLineColor',
+                'mpCountyLineDashPattern', 'mpCountyLineDashSegLenF',
+                'mpCountyLineThicknessF', 'mpDataBaseVersion', 'mpDataResolution',
+                'mpDataSetName', 'mpDefaultFillColor', 'mpDefaultFillPattern',
+                'mpDefaultFillScaleF', 'mpDynamicAreaGroups', 'mpEllipticalBoundary',
+                'mpFillAreaSpecifiers', 'mpFillBoundarySets', 'mpFillColor',
+                'mpFillColors', 'mpFillColors-default', 'mpFillDotSizeF',
+                'mpFillDrawOrder', 'mpFillOn', 'mpFillPatternBackground',
+                'mpFillPattern', 'mpFillPatterns', 'mpFillPatterns-default',
+                'mpFillScaleF', 'mpFillScales', 'mpFillScales-default',
+                'mpFixedAreaGroups', 'mpGeophysicalLineColor',
+                'mpGeophysicalLineDashPattern', 'mpGeophysicalLineDashSegLenF',
+                'mpGeophysicalLineThicknessF', 'mpGreatCircleLinesOn',
+                'mpGridAndLimbDrawOrder', 'mpGridAndLimbOn', 'mpGridLatSpacingF',
+                'mpGridLineColor', 'mpGridLineDashPattern', 'mpGridLineDashSegLenF',
+                'mpGridLineThicknessF', 'mpGridLonSpacingF', 'mpGridMaskMode',
+                'mpGridMaxLatF', 'mpGridPolarLonSpacingF', 'mpGridSpacingF',
+                'mpInlandWaterFillColor', 'mpInlandWaterFillPattern',
+                'mpInlandWaterFillScaleF', 'mpLabelDrawOrder', 'mpLabelFontColor',
+                'mpLabelFontHeightF', 'mpLabelsOn', 'mpLambertMeridianF',
+                'mpLambertParallel1F', 'mpLambertParallel2F', 'mpLandFillColor',
+                'mpLandFillPattern', 'mpLandFillScaleF', 'mpLeftAngleF',
+                'mpLeftCornerLatF', 'mpLeftCornerLonF', 'mpLeftMapPosF',
+                'mpLeftNDCF', 'mpLeftNPCF', 'mpLeftPointLatF',
+                'mpLeftPointLonF', 'mpLeftWindowF', 'mpLimbLineColor',
+                'mpLimbLineDashPattern', 'mpLimbLineDashSegLenF',
+                'mpLimbLineThicknessF', 'mpLimitMode', 'mpMaskAreaSpecifiers',
+                'mpMaskOutlineSpecifiers', 'mpMaxLatF', 'mpMaxLonF',
+                'mpMinLatF', 'mpMinLonF', 'mpMonoFillColor', 'mpMonoFillPattern',
+                'mpMonoFillScale', 'mpNationalLineColor', 'mpNationalLineDashPattern',
+                'mpNationalLineThicknessF', 'mpOceanFillColor', 'mpOceanFillPattern',
+                'mpOceanFillScaleF', 'mpOutlineBoundarySets', 'mpOutlineDrawOrder',
+                'mpOutlineMaskingOn', 'mpOutlineOn', 'mpOutlineSpecifiers',
+                'mpPerimDrawOrder', 'mpPerimLineColor', 'mpPerimLineDashPattern',
+                'mpPerimLineDashSegLenF', 'mpPerimLineThicknessF', 'mpPerimOn',
+                'mpPolyMode', 'mpProjection', 'mpProvincialLineColor',
+                'mpProvincialLineDashPattern', 'mpProvincialLineDashSegLenF',
+                'mpProvincialLineThicknessF', 'mpRelativeCenterLat',
+                'mpRelativeCenterLon', 'mpRightAngleF', 'mpRightCornerLatF',
+                'mpRightCornerLonF', 'mpRightMapPosF', 'mpRightNDCF',
+                'mpRightNPCF', 'mpRightPointLatF', 'mpRightPointLonF',
+                'mpRightWindowF', 'mpSatelliteAngle1F', 'mpSatelliteAngle2F',
+                'mpSatelliteDistF', 'mpShapeMode', 'mpSpecifiedFillColors',
+                'mpSpecifiedFillDirectIndexing', 'mpSpecifiedFillPatterns',
+                'mpSpecifiedFillPriority', 'mpSpecifiedFillScales',
+                'mpTopAngleF', 'mpTopMapPosF', 'mpTopNDCF', 'mpTopNPCF',
+                'mpTopPointLatF', 'mpTopPointLonF', 'mpTopWindowF',
+                'mpUSStateLineColor', 'mpUSStateLineDashPattern',
+                'mpUSStateLineDashSegLenF', 'mpUSStateLineThicknessF',
+                'pmAnnoManagers', 'pmAnnoViews', 'pmLabelBarDisplayMode',
+                'pmLabelBarHeightF', 'pmLabelBarKeepAspect', 'pmLabelBarOrthogonalPosF',
+                'pmLabelBarParallelPosF', 'pmLabelBarSide', 'pmLabelBarWidthF',
+                'pmLabelBarZone', 'pmLegendDisplayMode', 'pmLegendHeightF',
+                'pmLegendKeepAspect', 'pmLegendOrthogonalPosF',
+                'pmLegendParallelPosF', 'pmLegendSide', 'pmLegendWidthF',
+                'pmLegendZone', 'pmOverlaySequenceIds', 'pmTickMarkDisplayMode',
+                'pmTickMarkZone', 'pmTitleDisplayMode', 'pmTitleZone',
+                'prGraphicStyle', 'prPolyType', 'prXArray', 'prYArray',
+                'sfCopyData', 'sfDataArray', 'sfDataMaxV', 'sfDataMinV',
+                'sfElementNodes', 'sfExchangeDimensions', 'sfFirstNodeIndex',
+                'sfMissingValueV', 'sfXArray', 'sfXCActualEndF', 'sfXCActualStartF',
+                'sfXCEndIndex', 'sfXCEndSubsetV', 'sfXCEndV', 'sfXCStartIndex',
+                'sfXCStartSubsetV', 'sfXCStartV', 'sfXCStride', 'sfXCellBounds',
+                'sfYArray', 'sfYCActualEndF', 'sfYCActualStartF', 'sfYCEndIndex',
+                'sfYCEndSubsetV', 'sfYCEndV', 'sfYCStartIndex', 'sfYCStartSubsetV',
+                'sfYCStartV', 'sfYCStride', 'sfYCellBounds', 'stArrowLengthF',
+                'stArrowStride', 'stCrossoverCheckCount',
+                'stExplicitLabelBarLabelsOn', 'stLabelBarEndLabelsOn',
+                'stLabelFormat', 'stLengthCheckCount', 'stLevelColors',
+                'stLevelCount', 'stLevelPalette', 'stLevelSelectionMode',
+                'stLevelSpacingF', 'stLevels', 'stLineColor', 'stLineOpacityF',
+                'stLineStartStride', 'stLineThicknessF', 'stMapDirection',
+                'stMaxLevelCount', 'stMaxLevelValF', 'stMinArrowSpacingF',
+                'stMinDistanceF', 'stMinLevelValF', 'stMinLineSpacingF',
+                'stMinStepFactorF', 'stMonoLineColor', 'stNoDataLabelOn',
+                'stNoDataLabelString', 'stScalarFieldData', 'stScalarMissingValColor',
+                'stSpanLevelPalette', 'stStepSizeF', 'stStreamlineDrawOrder',
+                'stUseScalarArray', 'stVectorFieldData', 'stZeroFLabelAngleF',
+                'stZeroFLabelBackgroundColor', 'stZeroFLabelConstantSpacingF',
+                'stZeroFLabelFont', 'stZeroFLabelFontAspectF',
+                'stZeroFLabelFontColor', 'stZeroFLabelFontHeightF',
+                'stZeroFLabelFontQuality', 'stZeroFLabelFontThicknessF',
+                'stZeroFLabelFuncCode', 'stZeroFLabelJust', 'stZeroFLabelOn',
+                'stZeroFLabelOrthogonalPosF', 'stZeroFLabelParallelPosF',
+                'stZeroFLabelPerimColor', 'stZeroFLabelPerimOn',
+                'stZeroFLabelPerimSpaceF', 'stZeroFLabelPerimThicknessF',
+                'stZeroFLabelSide', 'stZeroFLabelString', 'stZeroFLabelTextDirection',
+                'stZeroFLabelZone', 'tfDoNDCOverlay', 'tfPlotManagerOn',
+                'tfPolyDrawList', 'tfPolyDrawOrder', 'tiDeltaF', 'tiMainAngleF',
+                'tiMainConstantSpacingF', 'tiMainDirection', 'tiMainFont',
+                'tiMainFontAspectF', 'tiMainFontColor', 'tiMainFontHeightF',
+                'tiMainFontQuality', 'tiMainFontThicknessF', 'tiMainFuncCode',
+                'tiMainJust', 'tiMainOffsetXF', 'tiMainOffsetYF', 'tiMainOn',
+                'tiMainPosition', 'tiMainSide', 'tiMainString', 'tiUseMainAttributes',
+                'tiXAxisAngleF', 'tiXAxisConstantSpacingF', 'tiXAxisDirection',
+                'tiXAxisFont', 'tiXAxisFontAspectF', 'tiXAxisFontColor',
+                'tiXAxisFontHeightF', 'tiXAxisFontQuality', 'tiXAxisFontThicknessF',
+                'tiXAxisFuncCode', 'tiXAxisJust', 'tiXAxisOffsetXF',
+                'tiXAxisOffsetYF', 'tiXAxisOn', 'tiXAxisPosition', 'tiXAxisSide',
+                'tiXAxisString', 'tiYAxisAngleF', 'tiYAxisConstantSpacingF',
+                'tiYAxisDirection', 'tiYAxisFont', 'tiYAxisFontAspectF',
+                'tiYAxisFontColor', 'tiYAxisFontHeightF', 'tiYAxisFontQuality',
+                'tiYAxisFontThicknessF', 'tiYAxisFuncCode', 'tiYAxisJust',
+                'tiYAxisOffsetXF', 'tiYAxisOffsetYF', 'tiYAxisOn', 'tiYAxisPosition',
+                'tiYAxisSide', 'tiYAxisString', 'tmBorderLineColor',
+                'tmBorderThicknessF', 'tmEqualizeXYSizes', 'tmLabelAutoStride',
+                'tmSciNoteCutoff', 'tmXBAutoPrecision', 'tmXBBorderOn',
+                'tmXBDataLeftF', 'tmXBDataRightF', 'tmXBFormat', 'tmXBIrrTensionF',
+                'tmXBIrregularPoints', 'tmXBLabelAngleF', 'tmXBLabelConstantSpacingF',
+                'tmXBLabelDeltaF', 'tmXBLabelDirection', 'tmXBLabelFont',
+                'tmXBLabelFontAspectF', 'tmXBLabelFontColor', 'tmXBLabelFontHeightF',
+                'tmXBLabelFontQuality', 'tmXBLabelFontThicknessF',
+                'tmXBLabelFuncCode', 'tmXBLabelJust', 'tmXBLabelStride', 'tmXBLabels',
+                'tmXBLabelsOn', 'tmXBMajorLengthF', 'tmXBMajorLineColor',
+                'tmXBMajorOutwardLengthF', 'tmXBMajorThicknessF', 'tmXBMaxLabelLenF',
+                'tmXBMaxTicks', 'tmXBMinLabelSpacingF', 'tmXBMinorLengthF',
+                'tmXBMinorLineColor', 'tmXBMinorOn', 'tmXBMinorOutwardLengthF',
+                'tmXBMinorPerMajor', 'tmXBMinorThicknessF', 'tmXBMinorValues',
+                'tmXBMode', 'tmXBOn', 'tmXBPrecision', 'tmXBStyle', 'tmXBTickEndF',
+                'tmXBTickSpacingF', 'tmXBTickStartF', 'tmXBValues', 'tmXMajorGrid',
+                'tmXMajorGridLineColor', 'tmXMajorGridLineDashPattern',
+                'tmXMajorGridThicknessF', 'tmXMinorGrid', 'tmXMinorGridLineColor',
+                'tmXMinorGridLineDashPattern', 'tmXMinorGridThicknessF',
+                'tmXTAutoPrecision', 'tmXTBorderOn', 'tmXTDataLeftF',
+                'tmXTDataRightF', 'tmXTFormat', 'tmXTIrrTensionF',
+                'tmXTIrregularPoints', 'tmXTLabelAngleF', 'tmXTLabelConstantSpacingF',
+                'tmXTLabelDeltaF', 'tmXTLabelDirection', 'tmXTLabelFont',
+                'tmXTLabelFontAspectF', 'tmXTLabelFontColor', 'tmXTLabelFontHeightF',
+                'tmXTLabelFontQuality', 'tmXTLabelFontThicknessF',
+                'tmXTLabelFuncCode', 'tmXTLabelJust', 'tmXTLabelStride', 'tmXTLabels',
+                'tmXTLabelsOn', 'tmXTMajorLengthF', 'tmXTMajorLineColor',
+                'tmXTMajorOutwardLengthF', 'tmXTMajorThicknessF', 'tmXTMaxLabelLenF',
+                'tmXTMaxTicks', 'tmXTMinLabelSpacingF', 'tmXTMinorLengthF',
+                'tmXTMinorLineColor', 'tmXTMinorOn', 'tmXTMinorOutwardLengthF',
+                'tmXTMinorPerMajor', 'tmXTMinorThicknessF', 'tmXTMinorValues',
+                'tmXTMode', 'tmXTOn', 'tmXTPrecision', 'tmXTStyle', 'tmXTTickEndF',
+                'tmXTTickSpacingF', 'tmXTTickStartF', 'tmXTValues', 'tmXUseBottom',
+                'tmYLAutoPrecision', 'tmYLBorderOn', 'tmYLDataBottomF',
+                'tmYLDataTopF', 'tmYLFormat', 'tmYLIrrTensionF',
+                'tmYLIrregularPoints', 'tmYLLabelAngleF', 'tmYLLabelConstantSpacingF',
+                'tmYLLabelDeltaF', 'tmYLLabelDirection', 'tmYLLabelFont',
+                'tmYLLabelFontAspectF', 'tmYLLabelFontColor', 'tmYLLabelFontHeightF',
+                'tmYLLabelFontQuality', 'tmYLLabelFontThicknessF',
+                'tmYLLabelFuncCode', 'tmYLLabelJust', 'tmYLLabelStride', 'tmYLLabels',
+                'tmYLLabelsOn', 'tmYLMajorLengthF', 'tmYLMajorLineColor',
+                'tmYLMajorOutwardLengthF', 'tmYLMajorThicknessF', 'tmYLMaxLabelLenF',
+                'tmYLMaxTicks', 'tmYLMinLabelSpacingF', 'tmYLMinorLengthF',
+                'tmYLMinorLineColor', 'tmYLMinorOn', 'tmYLMinorOutwardLengthF',
+                'tmYLMinorPerMajor', 'tmYLMinorThicknessF', 'tmYLMinorValues',
+                'tmYLMode', 'tmYLOn', 'tmYLPrecision', 'tmYLStyle', 'tmYLTickEndF',
+                'tmYLTickSpacingF', 'tmYLTickStartF', 'tmYLValues', 'tmYMajorGrid',
+                'tmYMajorGridLineColor', 'tmYMajorGridLineDashPattern',
+                'tmYMajorGridThicknessF', 'tmYMinorGrid', 'tmYMinorGridLineColor',
+                'tmYMinorGridLineDashPattern', 'tmYMinorGridThicknessF',
+                'tmYRAutoPrecision', 'tmYRBorderOn', 'tmYRDataBottomF',
+                'tmYRDataTopF', 'tmYRFormat', 'tmYRIrrTensionF',
+                'tmYRIrregularPoints', 'tmYRLabelAngleF', 'tmYRLabelConstantSpacingF',
+                'tmYRLabelDeltaF', 'tmYRLabelDirection', 'tmYRLabelFont',
+                'tmYRLabelFontAspectF', 'tmYRLabelFontColor', 'tmYRLabelFontHeightF',
+                'tmYRLabelFontQuality', 'tmYRLabelFontThicknessF',
+                'tmYRLabelFuncCode', 'tmYRLabelJust', 'tmYRLabelStride', 'tmYRLabels',
+                'tmYRLabelsOn', 'tmYRMajorLengthF', 'tmYRMajorLineColor',
+                'tmYRMajorOutwardLengthF', 'tmYRMajorThicknessF', 'tmYRMaxLabelLenF',
+                'tmYRMaxTicks', 'tmYRMinLabelSpacingF', 'tmYRMinorLengthF',
+                'tmYRMinorLineColor', 'tmYRMinorOn', 'tmYRMinorOutwardLengthF',
+                'tmYRMinorPerMajor', 'tmYRMinorThicknessF', 'tmYRMinorValues',
+                'tmYRMode', 'tmYROn', 'tmYRPrecision', 'tmYRStyle', 'tmYRTickEndF',
+                'tmYRTickSpacingF', 'tmYRTickStartF', 'tmYRValues', 'tmYUseLeft',
+                'trGridType', 'trLineInterpolationOn',
+                'trXAxisType', 'trXCoordPoints', 'trXInterPoints', 'trXLog',
+                'trXMaxF', 'trXMinF', 'trXReverse', 'trXSamples', 'trXTensionF',
+                'trYAxisType', 'trYCoordPoints', 'trYInterPoints', 'trYLog',
+                'trYMaxF', 'trYMinF', 'trYReverse', 'trYSamples', 'trYTensionF',
+                'txAngleF', 'txBackgroundFillColor', 'txConstantSpacingF', 'txDirection',
+                'txFont', 'HLU-Fonts', 'txFontAspectF', 'txFontColor',
+                'txFontHeightF', 'txFontOpacityF', 'txFontQuality',
+                'txFontThicknessF', 'txFuncCode', 'txJust', 'txPerimColor',
+                'txPerimDashLengthF', 'txPerimDashPattern', 'txPerimOn',
+                'txPerimSpaceF', 'txPerimThicknessF', 'txPosXF', 'txPosYF',
+                'txString', 'vcExplicitLabelBarLabelsOn', 'vcFillArrowEdgeColor',
+                'vcFillArrowEdgeThicknessF', 'vcFillArrowFillColor',
+                'vcFillArrowHeadInteriorXF', 'vcFillArrowHeadMinFracXF',
+                'vcFillArrowHeadMinFracYF', 'vcFillArrowHeadXF', 'vcFillArrowHeadYF',
+                'vcFillArrowMinFracWidthF', 'vcFillArrowWidthF', 'vcFillArrowsOn',
+                'vcFillOverEdge', 'vcGlyphOpacityF', 'vcGlyphStyle',
+                'vcLabelBarEndLabelsOn', 'vcLabelFontColor', 'vcLabelFontHeightF',
+                'vcLabelsOn', 'vcLabelsUseVectorColor', 'vcLevelColors',
+                'vcLevelCount', 'vcLevelPalette', 'vcLevelSelectionMode',
+                'vcLevelSpacingF', 'vcLevels', 'vcLineArrowColor',
+                'vcLineArrowHeadMaxSizeF', 'vcLineArrowHeadMinSizeF',
+                'vcLineArrowThicknessF', 'vcMagnitudeFormat',
+                'vcMagnitudeScaleFactorF', 'vcMagnitudeScaleValueF',
+                'vcMagnitudeScalingMode', 'vcMapDirection', 'vcMaxLevelCount',
+                'vcMaxLevelValF', 'vcMaxMagnitudeF', 'vcMinAnnoAngleF',
+                'vcMinAnnoArrowAngleF', 'vcMinAnnoArrowEdgeColor',
+                'vcMinAnnoArrowFillColor', 'vcMinAnnoArrowLineColor',
+                'vcMinAnnoArrowMinOffsetF', 'vcMinAnnoArrowSpaceF',
+                'vcMinAnnoArrowUseVecColor', 'vcMinAnnoBackgroundColor',
+                'vcMinAnnoConstantSpacingF', 'vcMinAnnoExplicitMagnitudeF',
+                'vcMinAnnoFont', 'vcMinAnnoFontAspectF', 'vcMinAnnoFontColor',
+                'vcMinAnnoFontHeightF', 'vcMinAnnoFontQuality',
+                'vcMinAnnoFontThicknessF', 'vcMinAnnoFuncCode', 'vcMinAnnoJust',
+                'vcMinAnnoOn', 'vcMinAnnoOrientation', 'vcMinAnnoOrthogonalPosF',
+                'vcMinAnnoParallelPosF', 'vcMinAnnoPerimColor', 'vcMinAnnoPerimOn',
+                'vcMinAnnoPerimSpaceF', 'vcMinAnnoPerimThicknessF', 'vcMinAnnoSide',
+                'vcMinAnnoString1', 'vcMinAnnoString1On', 'vcMinAnnoString2',
+                'vcMinAnnoString2On', 'vcMinAnnoTextDirection', 'vcMinAnnoZone',
+                'vcMinDistanceF', 'vcMinFracLengthF', 'vcMinLevelValF',
+                'vcMinMagnitudeF', 'vcMonoFillArrowEdgeColor',
+                'vcMonoFillArrowFillColor', 'vcMonoLineArrowColor',
+                'vcMonoWindBarbColor', 'vcNoDataLabelOn', 'vcNoDataLabelString',
+                'vcPositionMode', 'vcRefAnnoAngleF', 'vcRefAnnoArrowAngleF',
+                'vcRefAnnoArrowEdgeColor', 'vcRefAnnoArrowFillColor',
+                'vcRefAnnoArrowLineColor', 'vcRefAnnoArrowMinOffsetF',
+                'vcRefAnnoArrowSpaceF', 'vcRefAnnoArrowUseVecColor',
+                'vcRefAnnoBackgroundColor', 'vcRefAnnoConstantSpacingF',
+                'vcRefAnnoExplicitMagnitudeF', 'vcRefAnnoFont',
+                'vcRefAnnoFontAspectF', 'vcRefAnnoFontColor', 'vcRefAnnoFontHeightF',
+                'vcRefAnnoFontQuality', 'vcRefAnnoFontThicknessF',
+                'vcRefAnnoFuncCode', 'vcRefAnnoJust', 'vcRefAnnoOn',
+                'vcRefAnnoOrientation', 'vcRefAnnoOrthogonalPosF',
+                'vcRefAnnoParallelPosF', 'vcRefAnnoPerimColor', 'vcRefAnnoPerimOn',
+                'vcRefAnnoPerimSpaceF', 'vcRefAnnoPerimThicknessF', 'vcRefAnnoSide',
+                'vcRefAnnoString1', 'vcRefAnnoString1On', 'vcRefAnnoString2',
+                'vcRefAnnoString2On', 'vcRefAnnoTextDirection', 'vcRefAnnoZone',
+                'vcRefLengthF', 'vcRefMagnitudeF', 'vcScalarFieldData',
+                'vcScalarMissingValColor', 'vcScalarValueFormat',
+                'vcScalarValueScaleFactorF', 'vcScalarValueScaleValueF',
+                'vcScalarValueScalingMode', 'vcSpanLevelPalette', 'vcUseRefAnnoRes',
+                'vcUseScalarArray', 'vcVectorDrawOrder', 'vcVectorFieldData',
+                'vcWindBarbCalmCircleSizeF', 'vcWindBarbColor',
+                'vcWindBarbLineThicknessF', 'vcWindBarbScaleFactorF',
+                'vcWindBarbTickAngleF', 'vcWindBarbTickLengthF',
+                'vcWindBarbTickSpacingF', 'vcZeroFLabelAngleF',
+                'vcZeroFLabelBackgroundColor', 'vcZeroFLabelConstantSpacingF',
+                'vcZeroFLabelFont', 'vcZeroFLabelFontAspectF',
+                'vcZeroFLabelFontColor', 'vcZeroFLabelFontHeightF',
+                'vcZeroFLabelFontQuality', 'vcZeroFLabelFontThicknessF',
+                'vcZeroFLabelFuncCode', 'vcZeroFLabelJust', 'vcZeroFLabelOn',
+                'vcZeroFLabelOrthogonalPosF', 'vcZeroFLabelParallelPosF',
+                'vcZeroFLabelPerimColor', 'vcZeroFLabelPerimOn',
+                'vcZeroFLabelPerimSpaceF', 'vcZeroFLabelPerimThicknessF',
+                'vcZeroFLabelSide', 'vcZeroFLabelString', 'vcZeroFLabelTextDirection',
+                'vcZeroFLabelZone', 'vfCopyData', 'vfDataArray',
+                'vfExchangeDimensions', 'vfExchangeUVData', 'vfMagMaxV', 'vfMagMinV',
+                'vfMissingUValueV', 'vfMissingVValueV', 'vfPolarData',
+                'vfSingleMissingValue', 'vfUDataArray', 'vfUMaxV', 'vfUMinV',
+                'vfVDataArray', 'vfVMaxV', 'vfVMinV', 'vfXArray', 'vfXCActualEndF',
+                'vfXCActualStartF', 'vfXCEndIndex', 'vfXCEndSubsetV', 'vfXCEndV',
+                'vfXCStartIndex', 'vfXCStartSubsetV', 'vfXCStartV', 'vfXCStride',
+                'vfYArray', 'vfYCActualEndF', 'vfYCActualStartF', 'vfYCEndIndex',
+                'vfYCEndSubsetV', 'vfYCEndV', 'vfYCStartIndex', 'vfYCStartSubsetV',
+                'vfYCStartV', 'vfYCStride', 'vpAnnoManagerId', 'vpClipOn',
+                'vpHeightF', 'vpKeepAspect', 'vpOn', 'vpUseSegments', 'vpWidthF',
+                'vpXF', 'vpYF', 'wkAntiAlias', 'wkBackgroundColor', 'wkBackgroundOpacityF',
+                'wkColorMapLen', 'wkColorMap', 'wkColorModel', 'wkDashTableLength',
+                'wkDefGraphicStyleId', 'wkDeviceLowerX', 'wkDeviceLowerY',
+                'wkDeviceUpperX', 'wkDeviceUpperY', 'wkFileName', 'wkFillTableLength',
+                'wkForegroundColor', 'wkFormat', 'wkFullBackground', 'wkGksWorkId',
+                'wkHeight', 'wkMarkerTableLength', 'wkMetaName', 'wkOrientation',
+                'wkPDFFileName', 'wkPDFFormat', 'wkPDFResolution', 'wkPSFileName',
+                'wkPSFormat', 'wkPSResolution', 'wkPaperHeightF', 'wkPaperSize',
+                'wkPaperWidthF', 'wkPause', 'wkTopLevelViews', 'wkViews',
+                'wkVisualType', 'wkWidth', 'wkWindowId', 'wkXColorMode', 'wsCurrentSize',
+                'wsMaximumSize', 'wsThresholdSize', 'xyComputeXMax',
+                'xyComputeXMin', 'xyComputeYMax', 'xyComputeYMin', 'xyCoordData',
+                'xyCoordDataSpec', 'xyCurveDrawOrder', 'xyDashPattern',
+                'xyDashPatterns', 'xyExplicitLabels', 'xyExplicitLegendLabels',
+                'xyLabelMode', 'xyLineColor', 'xyLineColors', 'xyLineDashSegLenF',
+                'xyLineLabelConstantSpacingF', 'xyLineLabelFont',
+                'xyLineLabelFontAspectF', 'xyLineLabelFontColor',
+                'xyLineLabelFontColors', 'xyLineLabelFontHeightF',
+                'xyLineLabelFontQuality', 'xyLineLabelFontThicknessF',
+                'xyLineLabelFuncCode', 'xyLineThicknessF', 'xyLineThicknesses',
+                'xyMarkLineMode', 'xyMarkLineModes', 'xyMarker', 'xyMarkerColor',
+                'xyMarkerColors', 'xyMarkerSizeF', 'xyMarkerSizes',
+                'xyMarkerThicknessF', 'xyMarkerThicknesses', 'xyMarkers',
+                'xyMonoDashPattern', 'xyMonoLineColor', 'xyMonoLineLabelFontColor',
+                'xyMonoLineThickness', 'xyMonoMarkLineMode', 'xyMonoMarker',
+                'xyMonoMarkerColor', 'xyMonoMarkerSize', 'xyMonoMarkerThickness',
+                'xyXIrrTensionF', 'xyXIrregularPoints', 'xyXStyle', 'xyYIrrTensionF',
+                'xyYIrregularPoints', 'xyYStyle'), prefix=r'\b'),
+             Name.Builtin),
+
+            # Booleans
+            (r'\.(True|False)\.', Name.Builtin),
+            # Comparing Operators
+            (r'\.(eq|ne|lt|le|gt|ge|not|and|or|xor)\.', Operator.Word),
+        ],
+
+        'strings': [
+            (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+        ],
+
+        'nums': [
+            (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer),
+            (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+            (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+        ],
+    }
diff --git a/lib/pygments/lexers/nimrod.py b/lib/pygments/lexers/nimrod.py
new file mode 100644
index 0000000..365a8dc
--- /dev/null
+++ b/lib/pygments/lexers/nimrod.py
@@ -0,0 +1,199 @@
+"""
+    pygments.lexers.nimrod
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Nim language (formerly known as Nimrod).
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, default, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+__all__ = ['NimrodLexer']
+
+
+class NimrodLexer(RegexLexer):
+    """
+    For Nim source code.
+    """
+
+    name = 'Nimrod'
+    url = 'http://nim-lang.org/'
+    aliases = ['nimrod', 'nim']
+    filenames = ['*.nim', '*.nimrod']
+    mimetypes = ['text/x-nim']
+    version_added = '1.5'
+
+    flags = re.MULTILINE | re.IGNORECASE
+
+    def underscorize(words):
+        newWords = []
+        new = []
+        for word in words:
+            for ch in word:
+                new.append(ch)
+                new.append("_?")
+            newWords.append(''.join(new))
+            new = []
+        return "|".join(newWords)
+
+    keywords = [
+        'addr', 'and', 'as', 'asm', 'bind', 'block', 'break', 'case',
+        'cast', 'concept', 'const', 'continue', 'converter', 'defer', 'discard',
+        'distinct', 'div', 'do', 'elif', 'else', 'end', 'enum', 'except',
+        'export', 'finally', 'for', 'if', 'in', 'yield', 'interface',
+        'is', 'isnot', 'iterator', 'let', 'mixin', 'mod',
+        'not', 'notin', 'object', 'of', 'or', 'out', 'ptr', 'raise',
+        'ref', 'return', 'shl', 'shr', 'static', 'try',
+        'tuple', 'type', 'using', 'when', 'while', 'xor'
+    ]
+
+    keywordsPseudo = [
+        'nil', 'true', 'false'
+    ]
+
+    opWords = [
+        'and', 'or', 'not', 'xor', 'shl', 'shr', 'div', 'mod', 'in',
+        'notin', 'is', 'isnot'
+    ]
+
+    types = [
+        'int', 'int8', 'int16', 'int32', 'int64', 'float', 'float32', 'float64',
+        'bool', 'char', 'range', 'array', 'seq', 'set', 'string'
+    ]
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'##\[', String.Doc, 'doccomment'),
+            (r'##.*$', String.Doc),
+            (r'#\[', Comment.Multiline, 'comment'),
+            (r'#.*$', Comment),
+
+            # Pragmas
+            (r'\{\.', String.Other, 'pragma'),
+
+            # Operators
+            (r'[*=><+\-/@$~&%!?|\\\[\]]', Operator),
+            (r'\.\.|\.|,|\[\.|\.\]|\{\.|\.\}|\(\.|\.\)|\{|\}|\(|\)|:|\^|`|;',
+             Punctuation),
+
+            # Case statement branch
+            (r'(\n\s*)(of)(\s)', bygroups(Text.Whitespace, Keyword,
+                                          Text.Whitespace), 'casebranch'),
+
+            # Strings
+            (r'(?:[\w]+)"', String, 'rdqs'),
+            (r'"""', String.Double, 'tdqs'),
+            ('"', String, 'dqs'),
+
+            # Char
+            ("'", String.Char, 'chars'),
+
+            # Keywords
+            (rf'({underscorize(opWords)})\b', Operator.Word),
+            (r'(proc|func|method|macro|template)(\s)(?![(\[\]])',
+             bygroups(Keyword, Text.Whitespace), 'funcname'),
+            (rf'({underscorize(keywords)})\b', Keyword),
+            (r'({})\b'.format(underscorize(['from', 'import', 'include', 'export'])),
+             Keyword.Namespace),
+            (r'(v_?a_?r)\b', Keyword.Declaration),
+            (rf'({underscorize(types)})\b', Name.Builtin),
+            (rf'({underscorize(keywordsPseudo)})\b', Keyword.Pseudo),
+
+            # Identifiers
+            (r'\b((?![_\d])\w)(((?!_)\w)|(_(?!_)\w))*', Name),
+
+            # Numbers
+            (r'[0-9][0-9_]*(?=([e.]|\'f(32|64)))',
+             Number.Float, ('float-suffix', 'float-number')),
+            (r'0x[a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'),
+            (r'0b[01][01_]*', Number.Bin, 'int-suffix'),
+            (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'),
+            (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'),
+
+            # Whitespace
+            (r'\s+', Text.Whitespace),
+            (r'.+$', Error),
+        ],
+        'chars': [
+            (r'\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape),
+            (r"'", String.Char, '#pop'),
+            (r".", String.Char)
+        ],
+        'strings': [
+            (r'(?|>=|>>|>|<=|<<|<|\+|-|=|/|\*|%|\+=|-=|!|@', Operator),
+            (r'\(|\)|\[|\]|,|\.\.\.|\.\.|\.|::|:', Punctuation),
+            (r'`\{[^`]*`\}', Text),  # Extern blocks won't be Lexed by Nit
+            (r'[\r\n\t ]+', Text),
+        ],
+    }
diff --git a/lib/pygments/lexers/nix.py b/lib/pygments/lexers/nix.py
new file mode 100644
index 0000000..3fa88c6
--- /dev/null
+++ b/lib/pygments/lexers/nix.py
@@ -0,0 +1,144 @@
+"""
+    pygments.lexers.nix
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the NixOS Nix language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Literal
+
+__all__ = ['NixLexer']
+
+
+class NixLexer(RegexLexer):
+    """
+    For the Nix language.
+    """
+
+    name = 'Nix'
+    url = 'http://nixos.org/nix/'
+    aliases = ['nixos', 'nix']
+    filenames = ['*.nix']
+    mimetypes = ['text/x-nix']
+    version_added = '2.0'
+
+    keywords = ['rec', 'with', 'let', 'in', 'inherit', 'assert', 'if',
+                'else', 'then', '...']
+    builtins = ['import', 'abort', 'baseNameOf', 'dirOf', 'isNull', 'builtins',
+                'map', 'removeAttrs', 'throw', 'toString', 'derivation']
+    operators = ['++', '+', '?', '.', '!', '//', '==', '/',
+                 '!=', '&&', '||', '->', '=', '<', '>', '*', '-']
+
+    punctuations = ["(", ")", "[", "]", ";", "{", "}", ":", ",", "@"]
+
+    tokens = {
+        'root': [
+            # comments starting with #
+            (r'#.*$', Comment.Single),
+
+            # multiline comments
+            (r'/\*', Comment.Multiline, 'comment'),
+
+            # whitespace
+            (r'\s+', Text),
+
+            # keywords
+            ('({})'.format('|'.join(re.escape(entry) + '\\b' for entry in keywords)), Keyword),
+
+            # highlight the builtins
+            ('({})'.format('|'.join(re.escape(entry) + '\\b' for entry in builtins)),
+             Name.Builtin),
+
+            (r'\b(true|false|null)\b', Name.Constant),
+
+            # floats
+            (r'-?(\d+\.\d*|\.\d+)([eE][-+]?\d+)?', Number.Float),
+
+            # integers
+            (r'-?[0-9]+', Number.Integer),
+
+            # paths
+            (r'[\w.+-]*(\/[\w.+-]+)+', Literal),
+            (r'~(\/[\w.+-]+)+', Literal),
+            (r'\<[\w.+-]+(\/[\w.+-]+)*\>', Literal),
+
+            # operators
+            ('({})'.format('|'.join(re.escape(entry) for entry in operators)),
+             Operator),
+
+            # word operators
+            (r'\b(or|and)\b', Operator.Word),
+
+            (r'\{', Punctuation, 'block'),
+
+            # punctuations
+            ('({})'.format('|'.join(re.escape(entry) for entry in punctuations)), Punctuation),
+
+            # strings
+            (r'"', String.Double, 'doublequote'),
+            (r"''", String.Multiline, 'multiline'),
+
+            # urls
+            (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[\w%/?:@&=+$,\\.!~*\'-]+', Literal),
+
+            # names of variables
+            (r'[\w-]+(?=\s*=)', String.Symbol),
+            (r'[a-zA-Z_][\w\'-]*', Text),
+
+            (r"\$\{", String.Interpol, 'antiquote'),
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline),
+        ],
+        'multiline': [
+            (r"''(\$|'|\\n|\\r|\\t|\\)", String.Escape),
+            (r"''", String.Multiline, '#pop'),
+            (r'\$\{', String.Interpol, 'antiquote'),
+            (r"[^'\$]+", String.Multiline),
+            (r"\$[^\{']", String.Multiline),
+            (r"'[^']", String.Multiline),
+            (r"\$(?=')", String.Multiline),
+        ],
+        'doublequote': [
+            (r'\\(\\|"|\$|n)', String.Escape),
+            (r'"', String.Double, '#pop'),
+            (r'\$\{', String.Interpol, 'antiquote'),
+            (r'[^"\\\$]+', String.Double),
+            (r'\$[^\{"]', String.Double),
+            (r'\$(?=")', String.Double),
+            (r'\\', String.Double),
+        ],
+        'antiquote': [
+            (r"\}", String.Interpol, '#pop'),
+            # TODO: we should probably escape also here ''${ \${
+            (r"\$\{", String.Interpol, '#push'),
+            include('root'),
+        ],
+        'block': [
+            (r"\}", Punctuation, '#pop'),
+            include('root'),
+        ],
+    }
+
+    def analyse_text(text):
+        rv = 0.0
+        # TODO: let/in
+        if re.search(r'import.+?<[^>]+>', text):
+            rv += 0.4
+        if re.search(r'mkDerivation\s+(\(|\{|rec)', text):
+            rv += 0.4
+        if re.search(r'=\s+mkIf\s+', text):
+            rv += 0.4
+        if re.search(r'\{[a-zA-Z,\s]+\}:', text):
+            rv += 0.1
+        return rv
diff --git a/lib/pygments/lexers/numbair.py b/lib/pygments/lexers/numbair.py
new file mode 100644
index 0000000..435863e
--- /dev/null
+++ b/lib/pygments/lexers/numbair.py
@@ -0,0 +1,63 @@
+"""
+    pygments.lexers.numbair
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for other Numba Intermediate Representation.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Whitespace, Name, String,  Punctuation, Keyword, \
+    Operator, Number
+
+__all__ = ["NumbaIRLexer"]
+
+class NumbaIRLexer(RegexLexer):
+    """
+    Lexer for Numba IR
+    """
+    name = 'Numba_IR'
+    url = "https://numba.readthedocs.io/en/stable/developer/architecture.html#stage-2-generate-the-numba-ir"
+    aliases = ['numba_ir', 'numbair']
+    filenames = ['*.numba_ir']
+    mimetypes = ['text/x-numba_ir', 'text/x-numbair']
+    version_added = '2.19'
+
+    identifier = r'\$[a-zA-Z0-9._]+'
+    fun_or_var = r'([a-zA-Z_]+[a-zA-Z0-9]*)'
+
+    tokens = {
+        'root' : [
+            (r'(label)(\ [0-9]+)(:)$',
+                bygroups(Keyword, Name.Label, Punctuation)),
+
+            (r'=', Operator),
+            include('whitespace'),
+            include('keyword'),
+
+            (identifier, Name.Variable),
+            (fun_or_var + r'(\()',
+                bygroups(Name.Function, Punctuation)),
+            (fun_or_var + r'(\=)',
+                bygroups(Name.Attribute, Punctuation)),
+            (fun_or_var, Name.Constant),
+            (r'[0-9]+', Number),
+
+            # 
+            (r'<[^>\n]*>', String),
+
+            (r'[=<>{}\[\]()*.,!\':]|x\b', Punctuation)
+        ],
+
+        'keyword':[
+            (words((
+                'del', 'jump', 'call', 'branch',
+            ), suffix=' '), Keyword),
+        ],
+
+        'whitespace': [
+            (r'(\n|\s)+', Whitespace),
+        ],
+    }
diff --git a/lib/pygments/lexers/oberon.py b/lib/pygments/lexers/oberon.py
new file mode 100644
index 0000000..61f3c2d
--- /dev/null
+++ b/lib/pygments/lexers/oberon.py
@@ -0,0 +1,120 @@
+"""
+    pygments.lexers.oberon
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Oberon family languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['ComponentPascalLexer']
+
+
+class ComponentPascalLexer(RegexLexer):
+    """
+    For Component Pascal source code.
+    """
+    name = 'Component Pascal'
+    aliases = ['componentpascal', 'cp']
+    filenames = ['*.cp', '*.cps']
+    mimetypes = ['text/x-component-pascal']
+    url = 'https://blackboxframework.org'
+    version_added = '2.1'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+            include('punctuation'),
+            include('numliterals'),
+            include('strings'),
+            include('operators'),
+            include('builtins'),
+            include('identifiers'),
+        ],
+        'whitespace': [
+            (r'\n+', Text),  # blank lines
+            (r'\s+', Text),  # whitespace
+        ],
+        'comments': [
+            (r'\(\*([^$].*?)\*\)', Comment.Multiline),
+            # TODO: nested comments (* (* ... *) ... (* ... *) *) not supported!
+        ],
+        'punctuation': [
+            (r'[()\[\]{},.:;|]', Punctuation),
+        ],
+        'numliterals': [
+            (r'[0-9A-F]+X\b', Number.Hex),                 # char code
+            (r'[0-9A-F]+[HL]\b', Number.Hex),              # hexadecimal number
+            (r'[0-9]+\.[0-9]+E[+-][0-9]+', Number.Float),  # real number
+            (r'[0-9]+\.[0-9]+', Number.Float),             # real number
+            (r'[0-9]+', Number.Integer),                   # decimal whole number
+        ],
+        'strings': [
+            (r"'[^\n']*'", String),  # single quoted string
+            (r'"[^\n"]*"', String),  # double quoted string
+        ],
+        'operators': [
+            # Arithmetic Operators
+            (r'[+-]', Operator),
+            (r'[*/]', Operator),
+            # Relational Operators
+            (r'[=#<>]', Operator),
+            # Dereferencing Operator
+            (r'\^', Operator),
+            # Logical AND Operator
+            (r'&', Operator),
+            # Logical NOT Operator
+            (r'~', Operator),
+            # Assignment Symbol
+            (r':=', Operator),
+            # Range Constructor
+            (r'\.\.', Operator),
+            (r'\$', Operator),
+        ],
+        'identifiers': [
+            (r'([a-zA-Z_$][\w$]*)', Name),
+        ],
+        'builtins': [
+            (words((
+                'ANYPTR', 'ANYREC', 'BOOLEAN', 'BYTE', 'CHAR', 'INTEGER', 'LONGINT',
+                'REAL', 'SET', 'SHORTCHAR', 'SHORTINT', 'SHORTREAL'
+                ), suffix=r'\b'), Keyword.Type),
+            (words((
+                'ABS', 'ABSTRACT', 'ARRAY', 'ASH', 'ASSERT', 'BEGIN', 'BITS', 'BY',
+                'CAP', 'CASE', 'CHR', 'CLOSE', 'CONST', 'DEC', 'DIV', 'DO', 'ELSE',
+                'ELSIF', 'EMPTY', 'END', 'ENTIER', 'EXCL', 'EXIT', 'EXTENSIBLE', 'FOR',
+                'HALT', 'IF', 'IMPORT', 'IN', 'INC', 'INCL', 'IS', 'LEN', 'LIMITED',
+                'LONG', 'LOOP', 'MAX', 'MIN', 'MOD', 'MODULE', 'NEW', 'ODD', 'OF',
+                'OR', 'ORD', 'OUT', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN',
+                'SHORT', 'SHORTCHAR', 'SHORTINT', 'SIZE', 'THEN', 'TYPE', 'TO', 'UNTIL',
+                'VAR', 'WHILE', 'WITH'
+                ), suffix=r'\b'), Keyword.Reserved),
+            (r'(TRUE|FALSE|NIL|INF)\b', Keyword.Constant),
+        ]
+    }
+
+    def analyse_text(text):
+        """The only other lexer using .cp is the C++ one, so we check if for
+        a few common Pascal keywords here. Those are unfortunately quite
+        common across various business languages as well."""
+        result = 0
+        if 'BEGIN' in text:
+            result += 0.01
+        if 'END' in text:
+            result += 0.01
+        if 'PROCEDURE' in text:
+            result += 0.01
+        if 'END' in text:
+            result += 0.01
+
+        return result
diff --git a/lib/pygments/lexers/objective.py b/lib/pygments/lexers/objective.py
new file mode 100644
index 0000000..899c2c4
--- /dev/null
+++ b/lib/pygments/lexers/objective.py
@@ -0,0 +1,513 @@
+"""
+    pygments.lexers.objective
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Objective-C family languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this, words, \
+    inherit, default
+from pygments.token import Text, Keyword, Name, String, Operator, \
+    Number, Punctuation, Literal, Comment, Whitespace
+
+from pygments.lexers.c_cpp import CLexer, CppLexer
+
+__all__ = ['ObjectiveCLexer', 'ObjectiveCppLexer', 'LogosLexer', 'SwiftLexer']
+
+
+def objective(baselexer):
+    """
+    Generate a subclass of baselexer that accepts the Objective-C syntax
+    extensions.
+    """
+
+    # Have to be careful not to accidentally match JavaDoc/Doxygen syntax here,
+    # since that's quite common in ordinary C/C++ files.  It's OK to match
+    # JavaDoc/Doxygen keywords that only apply to Objective-C, mind.
+    #
+    # The upshot of this is that we CANNOT match @class or @interface
+    _oc_keywords = re.compile(r'@(?:end|implementation|protocol)')
+
+    # Matches [ ? identifier  ( identifier ? ] |  identifier? : )
+    # (note the identifier is *optional* when there is a ':'!)
+    _oc_message = re.compile(r'\[\s*[a-zA-Z_]\w*\s+'
+                             r'(?:[a-zA-Z_]\w*\s*\]|'
+                             r'(?:[a-zA-Z_]\w*)?:)')
+
+    class GeneratedObjectiveCVariant(baselexer):
+        """
+        Implements Objective-C syntax on top of an existing C family lexer.
+        """
+
+        tokens = {
+            'statements': [
+                (r'@"', String, 'string'),
+                (r'@(YES|NO)', Number),
+                (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+                (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+                (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+                (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex),
+                (r'@0[0-7]+[Ll]?', Number.Oct),
+                (r'@\d+[Ll]?', Number.Integer),
+                (r'@\(', Literal, 'literal_number'),
+                (r'@\[', Literal, 'literal_array'),
+                (r'@\{', Literal, 'literal_dictionary'),
+                (words((
+                    '@selector', '@private', '@protected', '@public', '@encode',
+                    '@synchronized', '@try', '@throw', '@catch', '@finally',
+                    '@end', '@property', '@synthesize', '__bridge', '__bridge_transfer',
+                    '__autoreleasing', '__block', '__weak', '__strong', 'weak', 'strong',
+                    'copy', 'retain', 'assign', 'unsafe_unretained', 'atomic', 'nonatomic',
+                    'readonly', 'readwrite', 'setter', 'getter', 'typeof', 'in',
+                    'out', 'inout', 'release', 'class', '@dynamic', '@optional',
+                    '@required', '@autoreleasepool', '@import'), suffix=r'\b'),
+                 Keyword),
+                (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL',
+                        'IBOutlet', 'IBAction', 'unichar'), suffix=r'\b'),
+                 Keyword.Type),
+                (r'@(true|false|YES|NO)\n', Name.Builtin),
+                (r'(YES|NO|nil|self|super)\b', Name.Builtin),
+                # Carbon types
+                (r'(Boolean|UInt8|SInt8|UInt16|SInt16|UInt32|SInt32)\b', Keyword.Type),
+                # Carbon built-ins
+                (r'(TRUE|FALSE)\b', Name.Builtin),
+                (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text),
+                 ('#pop', 'oc_classname')),
+                (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text),
+                 ('#pop', 'oc_forward_classname')),
+                # @ can also prefix other expressions like @{...} or @(...)
+                (r'@', Punctuation),
+                inherit,
+            ],
+            'oc_classname': [
+                # interface definition that inherits
+                (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)',
+                 bygroups(Name.Class, Text, Name.Class, Text, Punctuation),
+                 ('#pop', 'oc_ivars')),
+                (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+                 bygroups(Name.Class, Text, Name.Class), '#pop'),
+                # interface definition for a category
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)',
+                 bygroups(Name.Class, Text, Name.Label, Text, Punctuation),
+                 ('#pop', 'oc_ivars')),
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))',
+                 bygroups(Name.Class, Text, Name.Label), '#pop'),
+                # simple interface / implementation
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\{)',
+                 bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')),
+                (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+            ],
+            'oc_forward_classname': [
+                (r'([a-zA-Z$_][\w$]*)(\s*,\s*)',
+                 bygroups(Name.Class, Text), 'oc_forward_classname'),
+                (r'([a-zA-Z$_][\w$]*)(\s*;?)',
+                 bygroups(Name.Class, Text), '#pop')
+            ],
+            'oc_ivars': [
+                include('whitespace'),
+                include('statements'),
+                (';', Punctuation),
+                (r'\{', Punctuation, '#push'),
+                (r'\}', Punctuation, '#pop'),
+            ],
+            'root': [
+                # methods
+                (r'^([-+])(\s*)'                         # method marker
+                 r'(\(.*?\))?(\s*)'                      # return type
+                 r'([a-zA-Z$_][\w$]*:?)',        # begin of method name
+                 bygroups(Punctuation, Text, using(this),
+                          Text, Name.Function),
+                 'method'),
+                inherit,
+            ],
+            'method': [
+                include('whitespace'),
+                # TODO unsure if ellipses are allowed elsewhere, see
+                # discussion in Issue 789
+                (r',', Punctuation),
+                (r'\.\.\.', Punctuation),
+                (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)',
+                 bygroups(using(this), Text, Name.Variable)),
+                (r'[a-zA-Z$_][\w$]*:', Name.Function),
+                (';', Punctuation, '#pop'),
+                (r'\{', Punctuation, 'function'),
+                default('#pop'),
+            ],
+            'literal_number': [
+                (r'\(', Punctuation, 'literal_number_inner'),
+                (r'\)', Literal, '#pop'),
+                include('statement'),
+            ],
+            'literal_number_inner': [
+                (r'\(', Punctuation, '#push'),
+                (r'\)', Punctuation, '#pop'),
+                include('statement'),
+            ],
+            'literal_array': [
+                (r'\[', Punctuation, 'literal_array_inner'),
+                (r'\]', Literal, '#pop'),
+                include('statement'),
+            ],
+            'literal_array_inner': [
+                (r'\[', Punctuation, '#push'),
+                (r'\]', Punctuation, '#pop'),
+                include('statement'),
+            ],
+            'literal_dictionary': [
+                (r'\}', Literal, '#pop'),
+                include('statement'),
+            ],
+        }
+
+        def analyse_text(text):
+            if _oc_keywords.search(text):
+                return 1.0
+            elif '@"' in text:  # strings
+                return 0.8
+            elif re.search('@[0-9]+', text):
+                return 0.7
+            elif _oc_message.search(text):
+                return 0.8
+            return 0
+
+        def get_tokens_unprocessed(self, text, stack=('root',)):
+            from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
+                COCOA_PROTOCOLS, COCOA_PRIMITIVES
+
+            for index, token, value in \
+                    baselexer.get_tokens_unprocessed(self, text, stack):
+                if token is Name or token is Name.Class:
+                    if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
+                       or value in COCOA_PRIMITIVES:
+                        token = Name.Builtin.Pseudo
+
+                yield index, token, value
+
+    return GeneratedObjectiveCVariant
+
+
+class ObjectiveCLexer(objective(CLexer)):
+    """
+    For Objective-C source code with preprocessor directives.
+    """
+
+    name = 'Objective-C'
+    url = 'https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/ProgrammingWithObjectiveC/Introduction/Introduction.html'
+    aliases = ['objective-c', 'objectivec', 'obj-c', 'objc']
+    filenames = ['*.m', '*.h']
+    mimetypes = ['text/x-objective-c']
+    version_added = ''
+    priority = 0.05    # Lower than C
+
+
+class ObjectiveCppLexer(objective(CppLexer)):
+    """
+    For Objective-C++ source code with preprocessor directives.
+    """
+
+    name = 'Objective-C++'
+    aliases = ['objective-c++', 'objectivec++', 'obj-c++', 'objc++']
+    filenames = ['*.mm', '*.hh']
+    mimetypes = ['text/x-objective-c++']
+    version_added = ''
+    priority = 0.05    # Lower than C++
+
+
+class LogosLexer(ObjectiveCppLexer):
+    """
+    For Logos + Objective-C source code with preprocessor directives.
+    """
+
+    name = 'Logos'
+    aliases = ['logos']
+    filenames = ['*.x', '*.xi', '*.xm', '*.xmi']
+    mimetypes = ['text/x-logos']
+    version_added = '1.6'
+    priority = 0.25
+
+    tokens = {
+        'statements': [
+            (r'(%orig|%log)\b', Keyword),
+            (r'(%c)\b(\()(\s*)([a-zA-Z$_][\w$]*)(\s*)(\))',
+             bygroups(Keyword, Punctuation, Text, Name.Class, Text, Punctuation)),
+            (r'(%init)\b(\()',
+             bygroups(Keyword, Punctuation), 'logos_init_directive'),
+            (r'(%init)(?=\s*;)', bygroups(Keyword)),
+            (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)',
+             bygroups(Keyword, Text, Name.Class), '#pop'),
+            (r'(%subclass)(\s+)', bygroups(Keyword, Text),
+             ('#pop', 'logos_classname')),
+            inherit,
+        ],
+        'logos_init_directive': [
+            (r'\s+', Text),
+            (',', Punctuation, ('logos_init_directive', '#pop')),
+            (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)',
+             bygroups(Name.Class, Text, Punctuation, Text, Text)),
+            (r'([a-zA-Z$_][\w$]*)', Name.Class),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'logos_classname': [
+            (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+             bygroups(Name.Class, Text, Name.Class), '#pop'),
+            (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+        ],
+        'root': [
+            (r'(%subclass)(\s+)', bygroups(Keyword, Text),
+             'logos_classname'),
+            (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)',
+             bygroups(Keyword, Text, Name.Class)),
+            (r'(%config)(\s*\(\s*)(\w+)(\s*=)(.*?)(\)\s*)',
+             bygroups(Keyword, Text, Name.Variable, Text, String, Text)),
+            (r'(%ctor)(\s*)(\{)', bygroups(Keyword, Text, Punctuation),
+             'function'),
+            (r'(%new)(\s*)(\()(.*?)(\))',
+             bygroups(Keyword, Text, Keyword, String, Keyword)),
+            (r'(\s*)(%end)(\s*)', bygroups(Text, Keyword, Text)),
+            inherit,
+        ],
+    }
+
+    _logos_keywords = re.compile(r'%(?:hook|ctor|init|c\()')
+
+    def analyse_text(text):
+        if LogosLexer._logos_keywords.search(text):
+            return 1.0
+        return 0
+
+
+class SwiftLexer(RegexLexer):
+    """
+    For Swift source.
+    """
+    name = 'Swift'
+    url = 'https://www.swift.org/'
+    filenames = ['*.swift']
+    aliases = ['swift']
+    mimetypes = ['text/x-swift']
+    version_added = '2.0'
+
+    tokens = {
+        'root': [
+            # Whitespace and Comments
+            (r'\n', Text),
+            (r'\s+', Whitespace),
+            (r'//', Comment.Single, 'comment-single'),
+            (r'/\*', Comment.Multiline, 'comment-multi'),
+            (r'#(if|elseif|else|endif|available)\b', Comment.Preproc, 'preproc'),
+
+            # Keywords
+            include('keywords'),
+
+            # Global Types
+            (words((
+                'Array', 'AutoreleasingUnsafeMutablePointer', 'BidirectionalReverseView',
+                'Bit', 'Bool', 'CFunctionPointer', 'COpaquePointer', 'CVaListPointer',
+                'Character', 'ClosedInterval', 'CollectionOfOne', 'ContiguousArray',
+                'Dictionary', 'DictionaryGenerator', 'DictionaryIndex', 'Double',
+                'EmptyCollection', 'EmptyGenerator', 'EnumerateGenerator',
+                'EnumerateSequence', 'FilterCollectionView',
+                'FilterCollectionViewIndex', 'FilterGenerator', 'FilterSequenceView',
+                'Float', 'Float80', 'FloatingPointClassification', 'GeneratorOf',
+                'GeneratorOfOne', 'GeneratorSequence', 'HalfOpenInterval', 'HeapBuffer',
+                'HeapBufferStorage', 'ImplicitlyUnwrappedOptional', 'IndexingGenerator',
+                'Int', 'Int16', 'Int32', 'Int64', 'Int8', 'LazyBidirectionalCollection',
+                'LazyForwardCollection', 'LazyRandomAccessCollection',
+                'LazySequence', 'MapCollectionView', 'MapSequenceGenerator',
+                'MapSequenceView', 'MirrorDisposition', 'ObjectIdentifier', 'OnHeap',
+                'Optional', 'PermutationGenerator', 'QuickLookObject',
+                'RandomAccessReverseView', 'Range', 'RangeGenerator', 'RawByte', 'Repeat',
+                'ReverseBidirectionalIndex', 'ReverseRandomAccessIndex', 'SequenceOf',
+                'SinkOf', 'Slice', 'StaticString', 'StrideThrough', 'StrideThroughGenerator',
+                'StrideTo', 'StrideToGenerator', 'String', 'UInt', 'UInt16', 'UInt32',
+                'UInt64', 'UInt8', 'UTF16', 'UTF32', 'UTF8', 'UnicodeDecodingResult',
+                'UnicodeScalar', 'Unmanaged', 'UnsafeBufferPointer',
+                'UnsafeBufferPointerGenerator', 'UnsafeMutableBufferPointer',
+                'UnsafeMutablePointer', 'UnsafePointer', 'Zip2', 'ZipGenerator2',
+                # Protocols
+                'AbsoluteValuable', 'AnyObject', 'ArrayLiteralConvertible',
+                'BidirectionalIndexType', 'BitwiseOperationsType',
+                'BooleanLiteralConvertible', 'BooleanType', 'CVarArgType',
+                'CollectionType', 'Comparable', 'DebugPrintable',
+                'DictionaryLiteralConvertible', 'Equatable',
+                'ExtendedGraphemeClusterLiteralConvertible',
+                'ExtensibleCollectionType', 'FloatLiteralConvertible',
+                'FloatingPointType', 'ForwardIndexType', 'GeneratorType', 'Hashable',
+                'IntegerArithmeticType', 'IntegerLiteralConvertible', 'IntegerType',
+                'IntervalType', 'MirrorType', 'MutableCollectionType', 'MutableSliceable',
+                'NilLiteralConvertible', 'OutputStreamType', 'Printable',
+                'RandomAccessIndexType', 'RangeReplaceableCollectionType',
+                'RawOptionSetType', 'RawRepresentable', 'Reflectable', 'SequenceType',
+                'SignedIntegerType', 'SignedNumberType', 'SinkType', 'Sliceable',
+                'Streamable', 'Strideable', 'StringInterpolationConvertible',
+                'StringLiteralConvertible', 'UnicodeCodecType',
+                'UnicodeScalarLiteralConvertible', 'UnsignedIntegerType',
+                '_ArrayBufferType', '_BidirectionalIndexType', '_CocoaStringType',
+                '_CollectionType', '_Comparable', '_ExtensibleCollectionType',
+                '_ForwardIndexType', '_Incrementable', '_IntegerArithmeticType',
+                '_IntegerType', '_ObjectiveCBridgeable', '_RandomAccessIndexType',
+                '_RawOptionSetType', '_SequenceType', '_Sequence_Type',
+                '_SignedIntegerType', '_SignedNumberType', '_Sliceable', '_Strideable',
+                '_SwiftNSArrayRequiredOverridesType', '_SwiftNSArrayType',
+                '_SwiftNSCopyingType', '_SwiftNSDictionaryRequiredOverridesType',
+                '_SwiftNSDictionaryType', '_SwiftNSEnumeratorType',
+                '_SwiftNSFastEnumerationType', '_SwiftNSStringRequiredOverridesType',
+                '_SwiftNSStringType', '_UnsignedIntegerType',
+                # Variables
+                'C_ARGC', 'C_ARGV', 'Process',
+                # Typealiases
+                'Any', 'AnyClass', 'BooleanLiteralType', 'CBool', 'CChar', 'CChar16',
+                'CChar32', 'CDouble', 'CFloat', 'CInt', 'CLong', 'CLongLong', 'CShort',
+                'CSignedChar', 'CUnsignedInt', 'CUnsignedLong', 'CUnsignedShort',
+                'CWideChar', 'ExtendedGraphemeClusterType', 'Float32', 'Float64',
+                'FloatLiteralType', 'IntMax', 'IntegerLiteralType', 'StringLiteralType',
+                'UIntMax', 'UWord', 'UnicodeScalarType', 'Void', 'Word',
+                # Foundation/Cocoa
+                'NSErrorPointer', 'NSObjectProtocol', 'Selector'), suffix=r'\b'),
+             Name.Builtin),
+            # Functions
+            (words((
+                'abs', 'advance', 'alignof', 'alignofValue', 'assert', 'assertionFailure',
+                'contains', 'count', 'countElements', 'debugPrint', 'debugPrintln',
+                'distance', 'dropFirst', 'dropLast', 'dump', 'enumerate', 'equal',
+                'extend', 'fatalError', 'filter', 'find', 'first', 'getVaList', 'indices',
+                'insert', 'isEmpty', 'join', 'last', 'lazy', 'lexicographicalCompare',
+                'map', 'max', 'maxElement', 'min', 'minElement', 'numericCast', 'overlaps',
+                'partition', 'precondition', 'preconditionFailure', 'prefix', 'print',
+                'println', 'reduce', 'reflect', 'removeAll', 'removeAtIndex', 'removeLast',
+                'removeRange', 'reverse', 'sizeof', 'sizeofValue', 'sort', 'sorted',
+                'splice', 'split', 'startsWith', 'stride', 'strideof', 'strideofValue',
+                'suffix', 'swap', 'toDebugString', 'toString', 'transcode',
+                'underestimateCount', 'unsafeAddressOf', 'unsafeBitCast', 'unsafeDowncast',
+                'withExtendedLifetime', 'withUnsafeMutablePointer',
+                'withUnsafeMutablePointers', 'withUnsafePointer', 'withUnsafePointers',
+                'withVaList'), suffix=r'\b'),
+             Name.Builtin.Pseudo),
+
+            # Implicit Block Variables
+            (r'\$\d+', Name.Variable),
+
+            # Binary Literal
+            (r'0b[01_]+', Number.Bin),
+            # Octal Literal
+            (r'0o[0-7_]+', Number.Oct),
+            # Hexadecimal Literal
+            (r'0x[0-9a-fA-F_]+', Number.Hex),
+            # Decimal Literal
+            (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+             r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float),
+            (r'[0-9][0-9_]*', Number.Integer),
+            # String Literal
+            (r'"""', String, 'string-multi'),
+            (r'"', String, 'string'),
+
+            # Operators and Punctuation
+            (r'[(){}\[\].,:;=@#`?]|->|[<&?](?=\w)|(?<=\w)[>!?]', Punctuation),
+            (r'[/=\-+!*%<>&|^?~]+', Operator),
+
+            # Identifier
+            (r'[a-zA-Z_]\w*', Name)
+        ],
+        'keywords': [
+            (words((
+                'as', 'async', 'await', 'break', 'case', 'catch', 'continue', 'default', 'defer',
+                'do', 'else', 'fallthrough', 'for', 'guard', 'if', 'in', 'is',
+                'repeat', 'return', '#selector', 'switch', 'throw', 'try',
+                'where', 'while'), suffix=r'\b'),
+             Keyword),
+            (r'@availability\([^)]+\)', Keyword.Reserved),
+            (words((
+                'associativity', 'convenience', 'dynamic', 'didSet', 'final',
+                'get', 'indirect', 'infix', 'inout', 'lazy', 'left', 'mutating',
+                'none', 'nonmutating', 'optional', 'override', 'postfix',
+                'precedence', 'prefix', 'Protocol', 'required', 'rethrows',
+                'right', 'set', 'throws', 'Type', 'unowned', 'weak', 'willSet',
+                '@availability', '@autoclosure', '@noreturn',
+                '@NSApplicationMain', '@NSCopying', '@NSManaged', '@objc',
+                '@UIApplicationMain', '@IBAction', '@IBDesignable',
+                '@IBInspectable', '@IBOutlet'), suffix=r'\b'),
+             Keyword.Reserved),
+            (r'(as|dynamicType|false|is|nil|self|Self|super|true|__COLUMN__'
+             r'|__FILE__|__FUNCTION__|__LINE__|_'
+             r'|#(?:file|line|column|function))\b', Keyword.Constant),
+            (r'import\b', Keyword.Declaration, 'module'),
+            (r'(class|enum|extension|struct|protocol)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword.Declaration, Whitespace, Name.Class)),
+            (r'(func)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword.Declaration, Whitespace, Name.Function)),
+            (r'(var|let)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration,
+             Whitespace, Name.Variable)),
+            (words((
+                'actor', 'associatedtype', 'class', 'deinit', 'enum', 'extension', 'func', 'import',
+                'init', 'internal', 'let', 'operator', 'private', 'protocol', 'public',
+                'static', 'struct', 'subscript', 'typealias', 'var'), suffix=r'\b'),
+             Keyword.Declaration)
+        ],
+        'comment': [
+            (r':param: [a-zA-Z_]\w*|:returns?:|(FIXME|MARK|TODO):',
+             Comment.Special)
+        ],
+
+        # Nested
+        'comment-single': [
+            (r'\n', Whitespace, '#pop'),
+            include('comment'),
+            (r'[^\n]+', Comment.Single)
+        ],
+        'comment-multi': [
+            include('comment'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]+', Comment.Multiline)
+        ],
+        'module': [
+            (r'\n', Whitespace, '#pop'),
+            (r'[a-zA-Z_]\w*', Name.Class),
+            include('root')
+        ],
+        'preproc': [
+            (r'\n', Whitespace, '#pop'),
+            include('keywords'),
+            (r'[A-Za-z]\w*', Comment.Preproc),
+            include('root')
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            include("string-common"),
+        ],
+        'string-multi': [
+            (r'"""', String, '#pop'),
+            include("string-common"),
+        ],
+        'string-common': [
+            (r'\\\(', String.Interpol, 'string-intp'),
+            (r"""\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
+             r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}""", String.Escape),
+            (r'[^\\"]+', String),
+            (r'\\', String)
+        ],
+        'string-intp': [
+            (r'\(', String.Interpol, '#push'),
+            (r'\)', String.Interpol, '#pop'),
+            include('root')
+        ]
+    }
+
+    def get_tokens_unprocessed(self, text):
+        from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
+            COCOA_PROTOCOLS, COCOA_PRIMITIVES
+
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name or token is Name.Class:
+                if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
+                   or value in COCOA_PRIMITIVES:
+                    token = Name.Builtin.Pseudo
+
+            yield index, token, value
diff --git a/lib/pygments/lexers/ooc.py b/lib/pygments/lexers/ooc.py
new file mode 100644
index 0000000..8a99080
--- /dev/null
+++ b/lib/pygments/lexers/ooc.py
@@ -0,0 +1,84 @@
+"""
+    pygments.lexers.ooc
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Ooc language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['OocLexer']
+
+
+class OocLexer(RegexLexer):
+    """
+    For Ooc source code
+    """
+    name = 'Ooc'
+    url = 'https://ooc-lang.github.io/'
+    aliases = ['ooc']
+    filenames = ['*.ooc']
+    mimetypes = ['text/x-ooc']
+    version_added = '1.2'
+
+    tokens = {
+        'root': [
+            (words((
+                'class', 'interface', 'implement', 'abstract', 'extends', 'from',
+                'this', 'super', 'new', 'const', 'final', 'static', 'import',
+                'use', 'extern', 'inline', 'proto', 'break', 'continue',
+                'fallthrough', 'operator', 'if', 'else', 'for', 'while', 'do',
+                'switch', 'case', 'as', 'in', 'version', 'return', 'true',
+                'false', 'null'), prefix=r'\b', suffix=r'\b'),
+             Keyword),
+            (r'include\b', Keyword, 'include'),
+            (r'(cover)([ \t]+)(from)([ \t]+)(\w+[*@]?)',
+             bygroups(Keyword, Text, Keyword, Text, Name.Class)),
+            (r'(func)((?:[ \t]|\\\n)+)(~[a-z_]\w*)',
+             bygroups(Keyword, Text, Name.Function)),
+            (r'\bfunc\b', Keyword),
+            # Note: %= not listed on https://ooc-lang.github.io/docs/lang/operators/
+            (r'//.*', Comment),
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            (r'(==?|\+=?|-[=>]?|\*=?|/=?|:=|!=?|%=?|\?|>{1,3}=?|<{1,3}=?|\.\.|'
+             r'&&?|\|\|?|\^=?)', Operator),
+            (r'(\.)([ \t]*)([a-z]\w*)', bygroups(Operator, Text,
+                                                 Name.Function)),
+            (r'[A-Z][A-Z0-9_]+', Name.Constant),
+            (r'[A-Z]\w*([@*]|\[[ \t]*\])?', Name.Class),
+
+            (r'([a-z]\w*(?:~[a-z]\w*)?)((?:[ \t]|\\\n)*)(?=\()',
+             bygroups(Name.Function, Text)),
+            (r'[a-z]\w*', Name.Variable),
+
+            # : introduces types
+            (r'[:(){}\[\];,]', Punctuation),
+
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'0c[0-9]+', Number.Oct),
+            (r'0b[01]+', Number.Bin),
+            (r'[0-9_]\.[0-9_]*(?!\.)', Number.Float),
+            (r'[0-9_]+', Number.Decimal),
+
+            (r'"(?:\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\"])*"',
+             String.Double),
+            (r"'(?:\\.|\\[0-9]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
+             String.Char),
+            (r'@', Punctuation),  # pointer dereference
+            (r'\.', Punctuation),  # imports or chain operator
+
+            (r'\\[ \t\n]', Text),
+            (r'[ \t]+', Text),
+        ],
+        'include': [
+            (r'[\w/]+', Name),
+            (r',', Punctuation),
+            (r'[ \t]', Text),
+            (r'[;\n]', Text, '#pop'),
+        ],
+    }
diff --git a/lib/pygments/lexers/openscad.py b/lib/pygments/lexers/openscad.py
new file mode 100644
index 0000000..b06de22
--- /dev/null
+++ b/lib/pygments/lexers/openscad.py
@@ -0,0 +1,96 @@
+"""
+    pygments.lexers.openscad
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the OpenSCAD languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words, include
+from pygments.token import Text, Comment, Punctuation, Operator, Keyword, Name, Number, Whitespace, Literal, String
+
+__all__ = ['OpenScadLexer']
+
+
+class OpenScadLexer(RegexLexer):
+    """For openSCAD code.
+    """
+    name = "OpenSCAD"
+    url = "https://openscad.org/"
+    aliases = ["openscad"]
+    filenames = ["*.scad"]
+    mimetypes = ["application/x-openscad"]
+    version_added = '2.16'
+
+    tokens = {
+        "root": [
+            (r"[^\S\n]+", Whitespace),
+            (r'//', Comment.Single, 'comment-single'),
+            (r'/\*', Comment.Multiline, 'comment-multi'),
+            (r"[{}\[\]\(\),;:]", Punctuation),
+            (r"[*!#%\-+=?/]", Operator),
+            (r"<=|<|==|!=|>=|>|&&|\|\|", Operator),
+            (r"\$(f[asn]|t|vp[rtd]|children)", Operator),
+            (r"(undef|PI)\b", Keyword.Constant),
+            (
+                r"(use|include)((?:\s|\\\\s)+)",
+                bygroups(Keyword.Namespace, Text),
+                "includes",
+            ),
+            (r"(module)(\s*)([^\s\(]+)",
+             bygroups(Keyword.Namespace, Whitespace, Name.Namespace)),
+            (r"(function)(\s*)([^\s\(]+)",
+             bygroups(Keyword.Declaration, Whitespace, Name.Function)),
+            (words(("true", "false"), prefix=r"\b", suffix=r"\b"), Literal),
+            (words((
+                "function", "module", "include", "use", "for",
+                "intersection_for", "if", "else", "return"
+                ), prefix=r"\b", suffix=r"\b"), Keyword
+            ),
+            (words((
+                "circle", "square", "polygon", "text", "sphere", "cube",
+                "cylinder", "polyhedron", "translate", "rotate", "scale",
+                "resize", "mirror", "multmatrix", "color", "offset", "hull",
+                "minkowski", "union", "difference", "intersection", "abs",
+                "sign", "sin", "cos", "tan", "acos", "asin", "atan", "atan2",
+                "floor", "round", "ceil", "ln", "log", "pow", "sqrt", "exp",
+                "rands", "min", "max", "concat", "lookup", "str", "chr",
+                "search", "version", "version_num", "norm", "cross",
+                "parent_module", "echo", "import", "import_dxf",
+                "dxf_linear_extrude", "linear_extrude", "rotate_extrude",
+                "surface", "projection", "render", "dxf_cross",
+                "dxf_dim", "let", "assign", "len"
+                ), prefix=r"\b", suffix=r"\b"),
+                Name.Builtin
+            ),
+            (r"\bchildren\b", Name.Builtin.Pseudo),
+            (r'""".*?"""', String.Double),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"-?\d+(\.\d+)?(e[+-]?\d+)?", Number),
+            (r"\w+", Name),
+        ],
+        "includes": [
+            (
+                r"(<)([^>]*)(>)",
+                bygroups(Punctuation, Comment.PreprocFile, Punctuation),
+            ),
+        ],
+        'comment': [
+            (r':param: [a-zA-Z_]\w*|:returns?:|(FIXME|MARK|TODO):',
+             Comment.Special)
+        ],
+        'comment-single': [
+            (r'\n', Text, '#pop'),
+            include('comment'),
+            (r'[^\n]+', Comment.Single)
+        ],
+        'comment-multi': [
+            include('comment'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+    }
diff --git a/lib/pygments/lexers/other.py b/lib/pygments/lexers/other.py
new file mode 100644
index 0000000..2b7dfb4
--- /dev/null
+++ b/lib/pygments/lexers/other.py
@@ -0,0 +1,41 @@
+"""
+    pygments.lexers.other
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Just export lexer classes previously contained in this module.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.sql import SqlLexer, MySqlLexer, SqliteConsoleLexer
+from pygments.lexers.shell import BashLexer, BashSessionLexer, BatchLexer, \
+    TcshLexer
+from pygments.lexers.robotframework import RobotFrameworkLexer
+from pygments.lexers.testing import GherkinLexer
+from pygments.lexers.esoteric import BrainfuckLexer, BefungeLexer, RedcodeLexer
+from pygments.lexers.prolog import LogtalkLexer
+from pygments.lexers.snobol import SnobolLexer
+from pygments.lexers.rebol import RebolLexer
+from pygments.lexers.configs import KconfigLexer, Cfengine3Lexer
+from pygments.lexers.modeling import ModelicaLexer
+from pygments.lexers.scripting import AppleScriptLexer, MOOCodeLexer, \
+    HybrisLexer
+from pygments.lexers.graphics import PostScriptLexer, GnuplotLexer, \
+    AsymptoteLexer, PovrayLexer
+from pygments.lexers.business import ABAPLexer, OpenEdgeLexer, \
+    GoodDataCLLexer, MaqlLexer
+from pygments.lexers.automation import AutoItLexer, AutohotkeyLexer
+from pygments.lexers.dsls import ProtoBufLexer, BroLexer, PuppetLexer, \
+    MscgenLexer, VGLLexer
+from pygments.lexers.basic import CbmBasicV2Lexer
+from pygments.lexers.pawn import SourcePawnLexer, PawnLexer
+from pygments.lexers.ecl import ECLLexer
+from pygments.lexers.urbi import UrbiscriptLexer
+from pygments.lexers.smalltalk import SmalltalkLexer, NewspeakLexer
+from pygments.lexers.installers import NSISLexer, RPMSpecLexer
+from pygments.lexers.textedit import AwkLexer
+from pygments.lexers.smv import NuSMVLexer
+
+__all__ = []
diff --git a/lib/pygments/lexers/parasail.py b/lib/pygments/lexers/parasail.py
new file mode 100644
index 0000000..150d6a9
--- /dev/null
+++ b/lib/pygments/lexers/parasail.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.parasail
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for ParaSail.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Literal
+
+__all__ = ['ParaSailLexer']
+
+
+class ParaSailLexer(RegexLexer):
+    """
+    For ParaSail source code.
+    """
+
+    name = 'ParaSail'
+    url = 'http://www.parasail-lang.org'
+    aliases = ['parasail']
+    filenames = ['*.psi', '*.psl']
+    mimetypes = ['text/x-parasail']
+    version_added = '2.1'
+
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'[^\S\n]+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'\b(and|or|xor)=', Operator.Word),
+            (r'\b(and(\s+then)?|or(\s+else)?|xor|rem|mod|'
+             r'(is|not)\s+null)\b',
+             Operator.Word),
+            # Keywords
+            (r'\b(abs|abstract|all|block|class|concurrent|const|continue|'
+             r'each|end|exit|extends|exports|forward|func|global|implements|'
+             r'import|in|interface|is|lambda|locked|new|not|null|of|op|'
+             r'optional|private|queued|ref|return|reverse|separate|some|'
+             r'type|until|var|with|'
+             # Control flow
+             r'if|then|else|elsif|case|for|while|loop)\b',
+             Keyword.Reserved),
+            (r'(abstract\s+)?(interface|class|op|func|type)',
+             Keyword.Declaration),
+            # Literals
+            (r'"[^"]*"', String),
+            (r'\\[\'ntrf"0]', String.Escape),
+            (r'#[a-zA-Z]\w*', Literal),       # Enumeration
+            include('numbers'),
+            (r"'[^']'", String.Char),
+            (r'[a-zA-Z]\w*', Name),
+            # Operators and Punctuation
+            (r'(<==|==>|<=>|\*\*=|<\|=|<<=|>>=|==|!=|=\?|<=|>=|'
+             r'\*\*|<<|>>|=>|:=|\+=|-=|\*=|\|=|\||/=|\+|-|\*|/|'
+             r'\.\.|<\.\.|\.\.<|<\.\.<)',
+             Operator),
+            (r'(<|>|\[|\]|\(|\)|\||:|;|,|.|\{|\}|->)',
+             Punctuation),
+            (r'\n+', Text),
+        ],
+        'numbers': [
+            (r'\d[0-9_]*#[0-9a-fA-F][0-9a-fA-F_]*#', Number.Hex),  # any base
+            (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*', Number.Hex),        # C-like hex
+            (r'0[bB][01][01_]*', Number.Bin),                      # C-like bin
+            (r'\d[0-9_]*\.\d[0-9_]*[eE][+-]\d[0-9_]*',             # float exp
+             Number.Float),
+            (r'\d[0-9_]*\.\d[0-9_]*', Number.Float),               # float
+            (r'\d[0-9_]*', Number.Integer),                        # integer
+        ],
+    }
diff --git a/lib/pygments/lexers/parsers.py b/lib/pygments/lexers/parsers.py
new file mode 100644
index 0000000..7a4ed9d
--- /dev/null
+++ b/lib/pygments/lexers/parsers.py
@@ -0,0 +1,798 @@
+"""
+    pygments.lexers.parsers
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for parser generators.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, DelegatingLexer, \
+    include, bygroups, using
+from pygments.token import Punctuation, Other, Text, Comment, Operator, \
+    Keyword, Name, String, Number, Whitespace
+from pygments.lexers.jvm import JavaLexer
+from pygments.lexers.c_cpp import CLexer, CppLexer
+from pygments.lexers.objective import ObjectiveCLexer
+from pygments.lexers.d import DLexer
+from pygments.lexers.dotnet import CSharpLexer
+from pygments.lexers.ruby import RubyLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.perl import PerlLexer
+
+__all__ = ['RagelLexer', 'RagelEmbeddedLexer', 'RagelCLexer', 'RagelDLexer',
+           'RagelCppLexer', 'RagelObjectiveCLexer', 'RagelRubyLexer',
+           'RagelJavaLexer', 'AntlrLexer', 'AntlrPythonLexer',
+           'AntlrPerlLexer', 'AntlrRubyLexer', 'AntlrCppLexer',
+           'AntlrCSharpLexer', 'AntlrObjectiveCLexer',
+           'AntlrJavaLexer', 'AntlrActionScriptLexer',
+           'TreetopLexer', 'EbnfLexer']
+
+
+class RagelLexer(RegexLexer):
+    """A pure `Ragel `_ lexer.  Use this
+    for fragments of Ragel.  For ``.rl`` files, use
+    :class:`RagelEmbeddedLexer` instead (or one of the
+    language-specific subclasses).
+
+    """
+
+    name = 'Ragel'
+    url = 'http://www.colm.net/open-source/ragel/'
+    aliases = ['ragel']
+    filenames = []
+    version_added = '1.1'
+
+    tokens = {
+        'whitespace': [
+            (r'\s+', Whitespace)
+        ],
+        'comments': [
+            (r'\#.*$', Comment),
+        ],
+        'keywords': [
+            (r'(access|action|alphtype)\b', Keyword),
+            (r'(getkey|write|machine|include)\b', Keyword),
+            (r'(any|ascii|extend|alpha|digit|alnum|lower|upper)\b', Keyword),
+            (r'(xdigit|cntrl|graph|print|punct|space|zlen|empty)\b', Keyword)
+        ],
+        'numbers': [
+            (r'0x[0-9A-Fa-f]+', Number.Hex),
+            (r'[+-]?[0-9]+', Number.Integer),
+        ],
+        'literals': [
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+            (r'\[(\\\\|\\[^\\]|[^\\\]])*\]', String),          # square bracket literals
+            (r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', String.Regex),  # regular expressions
+        ],
+        'identifiers': [
+            (r'[a-zA-Z_]\w*', Name.Variable),
+        ],
+        'operators': [
+            (r',', Operator),                           # Join
+            (r'\||&|--?', Operator),                    # Union, Intersection and Subtraction
+            (r'\.|<:|:>>?', Operator),                  # Concatention
+            (r':', Operator),                           # Label
+            (r'->', Operator),                          # Epsilon Transition
+            (r'(>|\$|%|<|@|<>)(/|eof\b)', Operator),    # EOF Actions
+            (r'(>|\$|%|<|@|<>)(!|err\b)', Operator),    # Global Error Actions
+            (r'(>|\$|%|<|@|<>)(\^|lerr\b)', Operator),  # Local Error Actions
+            (r'(>|\$|%|<|@|<>)(~|to\b)', Operator),     # To-State Actions
+            (r'(>|\$|%|<|@|<>)(\*|from\b)', Operator),  # From-State Actions
+            (r'>|@|\$|%', Operator),                    # Transition Actions and Priorities
+            (r'\*|\?|\+|\{[0-9]*,[0-9]*\}', Operator),  # Repetition
+            (r'!|\^', Operator),                        # Negation
+            (r'\(|\)', Operator),                       # Grouping
+        ],
+        'root': [
+            include('literals'),
+            include('whitespace'),
+            include('comments'),
+            include('keywords'),
+            include('numbers'),
+            include('identifiers'),
+            include('operators'),
+            (r'\{', Punctuation, 'host'),
+            (r'=', Operator),
+            (r';', Punctuation),
+        ],
+        'host': [
+            (r'(' + r'|'.join((  # keep host code in largest possible chunks
+                r'[^{}\'"/#]+',  # exclude unsafe characters
+                r'[^\\]\\[{}]',  # allow escaped { or }
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'//.*$\n?',            # single line comment
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+                r'\#.*$\n?',            # ruby comment
+
+                # regular expression: There's no reason for it to start
+                # with a * and this stops confusion with comments.
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',
+
+                # / is safe now that we've handled regex and javadoc comments
+                r'/',
+            )) + r')+', Other),
+
+            (r'\{', Punctuation, '#push'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+    }
+
+
+class RagelEmbeddedLexer(RegexLexer):
+    """
+    A lexer for Ragel embedded in a host language file.
+
+    This will only highlight Ragel statements. If you want host language
+    highlighting then call the language-specific Ragel lexer.
+    """
+
+    name = 'Embedded Ragel'
+    aliases = ['ragel-em']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    tokens = {
+        'root': [
+            (r'(' + r'|'.join((   # keep host code in largest possible chunks
+                r'[^%\'"/#]+',    # exclude unsafe characters
+                r'%(?=[^%]|$)',   # a single % sign is okay, just not 2 of them
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+                r'//.*$\n?',  # single line comment
+                r'\#.*$\n?',  # ruby/ragel comment
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',  # regular expression
+
+                # / is safe now that we've handled regex and javadoc comments
+                r'/',
+            )) + r')+', Other),
+
+            # Single Line FSM.
+            # Please don't put a quoted newline in a single line FSM.
+            # That's just mean. It will break this.
+            (r'(%%)(?![{%])(.*)($|;)(\n?)', bygroups(Punctuation,
+                                                     using(RagelLexer),
+                                                     Punctuation, Text)),
+
+            # Multi Line FSM.
+            (r'(%%%%|%%)\{', Punctuation, 'multi-line-fsm'),
+        ],
+        'multi-line-fsm': [
+            (r'(' + r'|'.join((  # keep ragel code in largest possible chunks.
+                r'(' + r'|'.join((
+                    r'[^}\'"\[/#]',   # exclude unsafe characters
+                    r'\}(?=[^%]|$)',   # } is okay as long as it's not followed by %
+                    r'\}%(?=[^%]|$)',  # ...well, one %'s okay, just not two...
+                    r'[^\\]\\[{}]',   # ...and } is okay if it's escaped
+
+                    # allow / if it's preceded with one of these symbols
+                    # (ragel EOF actions)
+                    r'(>|\$|%|<|@|<>)/',
+
+                    # specifically allow regex followed immediately by *
+                    # so it doesn't get mistaken for a comment
+                    r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/\*',
+
+                    # allow / as long as it's not followed by another / or by a *
+                    r'/(?=[^/*]|$)',
+
+                    # We want to match as many of these as we can in one block.
+                    # Not sure if we need the + sign here,
+                    # does it help performance?
+                )) + r')+',
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r"\[(\\\\|\\[^\\]|[^\]\\])*\]",  # square bracket literal
+                r'/\*(.|\n)*?\*/',          # multi-line javadoc-style comment
+                r'//.*$\n?',                # single line comment
+                r'\#.*$\n?',                # ruby/ragel comment
+            )) + r')+', using(RagelLexer)),
+
+            (r'\}%%', Punctuation, '#pop'),
+        ]
+    }
+
+    def analyse_text(text):
+        return '@LANG: indep' in text
+
+
+class RagelRubyLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a Ruby host file.
+    """
+
+    name = 'Ragel in Ruby Host'
+    aliases = ['ragel-ruby', 'ragel-rb']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(RubyLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: ruby' in text
+
+
+class RagelCLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a C host file.
+    """
+
+    name = 'Ragel in C Host'
+    aliases = ['ragel-c']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: c' in text
+
+
+class RagelDLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a D host file.
+    """
+
+    name = 'Ragel in D Host'
+    aliases = ['ragel-d']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(DLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: d' in text
+
+
+class RagelCppLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a C++ host file.
+    """
+
+    name = 'Ragel in CPP Host'
+    aliases = ['ragel-cpp']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CppLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: c++' in text
+
+
+class RagelObjectiveCLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in an Objective C host file.
+    """
+
+    name = 'Ragel in Objective C Host'
+    aliases = ['ragel-objc']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(ObjectiveCLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: objc' in text
+
+
+class RagelJavaLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a Java host file.
+    """
+
+    name = 'Ragel in Java Host'
+    aliases = ['ragel-java']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(JavaLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: java' in text
+
+
+class AntlrLexer(RegexLexer):
+    """
+    Generic ANTLR Lexer.
+    Should not be called directly, instead
+    use DelegatingLexer for your target language.
+    """
+
+    name = 'ANTLR'
+    aliases = ['antlr']
+    filenames = []
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    _id = r'[A-Za-z]\w*'
+    _TOKEN_REF = r'[A-Z]\w*'
+    _RULE_REF = r'[a-z]\w*'
+    _STRING_LITERAL = r'\'(?:\\\\|\\\'|[^\']*)\''
+    _INT = r'[0-9]+'
+
+    tokens = {
+        'whitespace': [
+            (r'\s+', Whitespace),
+        ],
+        'comments': [
+            (r'//.*$', Comment),
+            (r'/\*(.|\n)*?\*/', Comment),
+        ],
+        'root': [
+            include('whitespace'),
+            include('comments'),
+
+            (r'(lexer|parser|tree)?(\s*)(grammar\b)(\s*)(' + _id + ')(;)',
+             bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Class,
+                      Punctuation)),
+            # optionsSpec
+            (r'options\b', Keyword, 'options'),
+            # tokensSpec
+            (r'tokens\b', Keyword, 'tokens'),
+            # attrScope
+            (r'(scope)(\s*)(' + _id + r')(\s*)(\{)',
+             bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
+                      Punctuation), 'action'),
+            # exception
+            (r'(catch|finally)\b', Keyword, 'exception'),
+            # action
+            (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)',
+             bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
+                      Name.Label, Whitespace, Punctuation), 'action'),
+            # rule
+            (r'((?:protected|private|public|fragment)\b)?(\s*)(' + _id + ')(!)?',
+             bygroups(Keyword, Whitespace, Name.Label, Punctuation),
+             ('rule-alts', 'rule-prelims')),
+        ],
+        'exception': [
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+            include('comments'),
+
+            (r'\[', Punctuation, 'nested-arg-action'),
+            (r'\{', Punctuation, 'action'),
+        ],
+        'rule-prelims': [
+            include('whitespace'),
+            include('comments'),
+
+            (r'returns\b', Keyword),
+            (r'\[', Punctuation, 'nested-arg-action'),
+            (r'\{', Punctuation, 'action'),
+            # throwsSpec
+            (r'(throws)(\s+)(' + _id + ')',
+             bygroups(Keyword, Whitespace, Name.Label)),
+            (r'(,)(\s*)(' + _id + ')',
+             bygroups(Punctuation, Whitespace, Name.Label)),  # Additional throws
+            # optionsSpec
+            (r'options\b', Keyword, 'options'),
+            # ruleScopeSpec - scope followed by target language code or name of action
+            # TODO finish implementing other possibilities for scope
+            # L173 ANTLRv3.g from ANTLR book
+            (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation),
+             'action'),
+            (r'(scope)(\s+)(' + _id + r')(\s*)(;)',
+             bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)),
+            # ruleAction
+            (r'(@' + _id + r')(\s*)(\{)',
+             bygroups(Name.Label, Whitespace, Punctuation), 'action'),
+            # finished prelims, go to rule alts!
+            (r':', Punctuation, '#pop')
+        ],
+        'rule-alts': [
+            include('whitespace'),
+            include('comments'),
+
+            # These might need to go in a separate 'block' state triggered by (
+            (r'options\b', Keyword, 'options'),
+            (r':', Punctuation),
+
+            # literals
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+            (r'<<([^>]|>[^>])>>', String),
+            # identifiers
+            # Tokens start with capital letter.
+            (r'\$?[A-Z_]\w*', Name.Constant),
+            # Rules start with small letter.
+            (r'\$?[a-z_]\w*', Name.Variable),
+            # operators
+            (r'(\+|\||->|=>|=|\(|\)|\.\.|\.|\?|\*|\^|!|\#|~)', Operator),
+            (r',', Punctuation),
+            (r'\[', Punctuation, 'nested-arg-action'),
+            (r'\{', Punctuation, 'action'),
+            (r';', Punctuation, '#pop')
+        ],
+        'tokens': [
+            include('whitespace'),
+            include('comments'),
+            (r'\{', Punctuation),
+            (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL
+             + r')?(\s*)(;)',
+             bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
+                      String, Whitespace, Punctuation)),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'options': [
+            include('whitespace'),
+            include('comments'),
+            (r'\{', Punctuation),
+            (r'(' + _id + r')(\s*)(=)(\s*)(' +
+             '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)',
+             bygroups(Name.Variable, Whitespace, Punctuation, Whitespace,
+                      Text, Whitespace, Punctuation)),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'action': [
+            (r'(' + r'|'.join((    # keep host code in largest possible chunks
+                r'[^${}\'"/\\]+',  # exclude unsafe characters
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'//.*$\n?',            # single line comment
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+
+                # regular expression: There's no reason for it to start
+                # with a * and this stops confusion with comments.
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',
+
+                # backslashes are okay, as long as we are not backslashing a %
+                r'\\(?!%)',
+
+                # Now that we've handled regex and javadoc comments
+                # it's safe to let / through.
+                r'/',
+            )) + r')+', Other),
+            (r'(\\)(%)', bygroups(Punctuation, Other)),
+            (r'(\$[a-zA-Z]+)(\.?)(text|value)?',
+             bygroups(Name.Variable, Punctuation, Name.Property)),
+            (r'\{', Punctuation, '#push'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'nested-arg-action': [
+            (r'(' + r'|'.join((    # keep host code in largest possible chunks.
+                r'[^$\[\]\'"/]+',  # exclude unsafe characters
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'//.*$\n?',            # single line comment
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+
+                # regular expression: There's no reason for it to start
+                # with a * and this stops confusion with comments.
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',
+
+                # Now that we've handled regex and javadoc comments
+                # it's safe to let / through.
+                r'/',
+            )) + r')+', Other),
+
+
+            (r'\[', Punctuation, '#push'),
+            (r'\]', Punctuation, '#pop'),
+            (r'(\$[a-zA-Z]+)(\.?)(text|value)?',
+             bygroups(Name.Variable, Punctuation, Name.Property)),
+            (r'(\\\\|\\\]|\\\[|[^\[\]])+', Other),
+        ]
+    }
+
+    def analyse_text(text):
+        return re.search(r'^\s*grammar\s+[a-zA-Z0-9]+\s*;', text, re.M)
+
+
+# http://www.antlr.org/wiki/display/ANTLR3/Code+Generation+Targets
+
+class AntlrCppLexer(DelegatingLexer):
+    """
+    ANTLR with C++ Target
+    """
+
+    name = 'ANTLR With CPP Target'
+    aliases = ['antlr-cpp']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CppLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*C\s*;', text, re.M)
+
+
+class AntlrObjectiveCLexer(DelegatingLexer):
+    """
+    ANTLR with Objective-C Target
+    """
+
+    name = 'ANTLR With ObjectiveC Target'
+    aliases = ['antlr-objc']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(ObjectiveCLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*ObjC\s*;', text)
+
+
+class AntlrCSharpLexer(DelegatingLexer):
+    """
+    ANTLR with C# Target
+    """
+
+    name = 'ANTLR With C# Target'
+    aliases = ['antlr-csharp', 'antlr-c#']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CSharpLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*CSharp2\s*;', text, re.M)
+
+
+class AntlrPythonLexer(DelegatingLexer):
+    """
+    ANTLR with Python Target
+    """
+
+    name = 'ANTLR With Python Target'
+    aliases = ['antlr-python']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(PythonLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*Python\s*;', text, re.M)
+
+
+class AntlrJavaLexer(DelegatingLexer):
+    """
+    ANTLR with Java Target
+    """
+
+    name = 'ANTLR With Java Target'
+    aliases = ['antlr-java']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(JavaLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        # Antlr language is Java by default
+        return AntlrLexer.analyse_text(text) and 0.9
+
+
+class AntlrRubyLexer(DelegatingLexer):
+    """
+    ANTLR with Ruby Target
+    """
+
+    name = 'ANTLR With Ruby Target'
+    aliases = ['antlr-ruby', 'antlr-rb']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(RubyLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*Ruby\s*;', text, re.M)
+
+
+class AntlrPerlLexer(DelegatingLexer):
+    """
+    ANTLR with Perl Target
+    """
+
+    name = 'ANTLR With Perl Target'
+    aliases = ['antlr-perl']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(PerlLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*Perl5\s*;', text, re.M)
+
+
+class AntlrActionScriptLexer(DelegatingLexer):
+    """
+    ANTLR with ActionScript Target
+    """
+
+    name = 'ANTLR With ActionScript Target'
+    aliases = ['antlr-actionscript', 'antlr-as']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        from pygments.lexers.actionscript import ActionScriptLexer
+        super().__init__(ActionScriptLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*ActionScript\s*;', text, re.M)
+
+
+class TreetopBaseLexer(RegexLexer):
+    """
+    A base lexer for `Treetop `_ grammars.
+    Not for direct use; use :class:`TreetopLexer` instead.
+
+    .. versionadded:: 1.6
+    """
+
+    tokens = {
+        'root': [
+            include('space'),
+            (r'require[ \t]+[^\n\r]+[\n\r]', Other),
+            (r'module\b', Keyword.Namespace, 'module'),
+            (r'grammar\b', Keyword, 'grammar'),
+        ],
+        'module': [
+            include('space'),
+            include('end'),
+            (r'module\b', Keyword, '#push'),
+            (r'grammar\b', Keyword, 'grammar'),
+            (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Namespace),
+        ],
+        'grammar': [
+            include('space'),
+            include('end'),
+            (r'rule\b', Keyword, 'rule'),
+            (r'include\b', Keyword, 'include'),
+            (r'[A-Z]\w*', Name),
+        ],
+        'include': [
+            include('space'),
+            (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Class, '#pop'),
+        ],
+        'rule': [
+            include('space'),
+            include('end'),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+            (r'([A-Za-z_]\w*)(:)', bygroups(Name.Label, Punctuation)),
+            (r'[A-Za-z_]\w*', Name),
+            (r'[()]', Punctuation),
+            (r'[?+*/&!~]', Operator),
+            (r'\[(?:\\.|\[:\^?[a-z]+:\]|[^\\\]])+\]', String.Regex),
+            (r'([0-9]*)(\.\.)([0-9]*)',
+             bygroups(Number.Integer, Operator, Number.Integer)),
+            (r'(<)([^>]+)(>)', bygroups(Punctuation, Name.Class, Punctuation)),
+            (r'\{', Punctuation, 'inline_module'),
+            (r'\.', String.Regex),
+        ],
+        'inline_module': [
+            (r'\{', Other, 'ruby'),
+            (r'\}', Punctuation, '#pop'),
+            (r'[^{}]+', Other),
+        ],
+        'ruby': [
+            (r'\{', Other, '#push'),
+            (r'\}', Other, '#pop'),
+            (r'[^{}]+', Other),
+        ],
+        'space': [
+            (r'[ \t\n\r]+', Whitespace),
+            (r'#[^\n]*', Comment.Single),
+        ],
+        'end': [
+            (r'end\b', Keyword, '#pop'),
+        ],
+    }
+
+
+class TreetopLexer(DelegatingLexer):
+    """
+    A lexer for Treetop grammars.
+    """
+
+    name = 'Treetop'
+    aliases = ['treetop']
+    filenames = ['*.treetop', '*.tt']
+    url = 'https://cjheath.github.io/treetop'
+    version_added = '1.6'
+
+    def __init__(self, **options):
+        super().__init__(RubyLexer, TreetopBaseLexer, **options)
+
+
+class EbnfLexer(RegexLexer):
+    """
+    Lexer for `ISO/IEC 14977 EBNF
+    `_
+    grammars.
+    """
+
+    name = 'EBNF'
+    aliases = ['ebnf']
+    filenames = ['*.ebnf']
+    mimetypes = ['text/x-ebnf']
+    url = 'https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form'
+    version_added = '2.0'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comment_start'),
+            include('identifier'),
+            (r'=', Operator, 'production'),
+        ],
+        'production': [
+            include('whitespace'),
+            include('comment_start'),
+            include('identifier'),
+            (r'"[^"]*"', String.Double),
+            (r"'[^']*'", String.Single),
+            (r'(\?[^?]*\?)', Name.Entity),
+            (r'[\[\]{}(),|]', Punctuation),
+            (r'-', Operator),
+            (r';', Punctuation, '#pop'),
+            (r'\.', Punctuation, '#pop'),
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+        ],
+        'comment_start': [
+            (r'\(\*', Comment.Multiline, 'comment'),
+        ],
+        'comment': [
+            (r'[^*)]', Comment.Multiline),
+            include('comment_start'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            (r'[*)]', Comment.Multiline),
+        ],
+        'identifier': [
+            (r'([a-zA-Z][\w \-]*)', Keyword),
+        ],
+    }
diff --git a/lib/pygments/lexers/pascal.py b/lib/pygments/lexers/pascal.py
new file mode 100644
index 0000000..5f40dcc
--- /dev/null
+++ b/lib/pygments/lexers/pascal.py
@@ -0,0 +1,644 @@
+"""
+    pygments.lexers.pascal
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Pascal family languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.util import get_bool_opt, get_list_opt
+from pygments.token import Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error, Whitespace
+from pygments.scanner import Scanner
+
+# compatibility import
+from pygments.lexers.modula2 import Modula2Lexer # noqa: F401
+
+__all__ = ['DelphiLexer', 'PortugolLexer']
+
+
+class PortugolLexer(Lexer):
+    """For Portugol, a Pascal dialect with keywords in Portuguese."""
+    name = 'Portugol'
+    aliases = ['portugol']
+    filenames = ['*.alg', '*.portugol']
+    mimetypes = []
+    url = "https://www.apoioinformatica.inf.br/produtos/visualg/linguagem"
+    version_added = ''
+
+    def __init__(self, **options):
+        Lexer.__init__(self, **options)
+        self.lexer = DelphiLexer(**options, portugol=True)
+
+    def get_tokens_unprocessed(self, text):
+        return self.lexer.get_tokens_unprocessed(text)
+
+
+class DelphiLexer(Lexer):
+    """
+    For Delphi (Borland Object Pascal),
+    Turbo Pascal and Free Pascal source code.
+
+    Additional options accepted:
+
+    `turbopascal`
+        Highlight Turbo Pascal specific keywords (default: ``True``).
+    `delphi`
+        Highlight Borland Delphi specific keywords (default: ``True``).
+    `freepascal`
+        Highlight Free Pascal specific keywords (default: ``True``).
+    `units`
+        A list of units that should be considered builtin, supported are
+        ``System``, ``SysUtils``, ``Classes`` and ``Math``.
+        Default is to consider all of them builtin.
+    """
+    name = 'Delphi'
+    aliases = ['delphi', 'pas', 'pascal', 'objectpascal']
+    filenames = ['*.pas', '*.dpr']
+    mimetypes = ['text/x-pascal']
+    url = 'https://www.embarcadero.com/products/delphi'
+    version_added = ''
+
+    TURBO_PASCAL_KEYWORDS = (
+        'absolute', 'and', 'array', 'asm', 'begin', 'break', 'case',
+        'const', 'constructor', 'continue', 'destructor', 'div', 'do',
+        'downto', 'else', 'end', 'file', 'for', 'function', 'goto',
+        'if', 'implementation', 'in', 'inherited', 'inline', 'interface',
+        'label', 'mod', 'nil', 'not', 'object', 'of', 'on', 'operator',
+        'or', 'packed', 'procedure', 'program', 'record', 'reintroduce',
+        'repeat', 'self', 'set', 'shl', 'shr', 'string', 'then', 'to',
+        'type', 'unit', 'until', 'uses', 'var', 'while', 'with', 'xor'
+    )
+
+    DELPHI_KEYWORDS = (
+        'as', 'class', 'except', 'exports', 'finalization', 'finally',
+        'initialization', 'is', 'library', 'on', 'property', 'raise',
+        'threadvar', 'try'
+    )
+
+    FREE_PASCAL_KEYWORDS = (
+        'dispose', 'exit', 'false', 'new', 'true'
+    )
+
+    BLOCK_KEYWORDS = {
+        'begin', 'class', 'const', 'constructor', 'destructor', 'end',
+        'finalization', 'function', 'implementation', 'initialization',
+        'label', 'library', 'operator', 'procedure', 'program', 'property',
+        'record', 'threadvar', 'type', 'unit', 'uses', 'var'
+    }
+
+    FUNCTION_MODIFIERS = {
+        'alias', 'cdecl', 'export', 'inline', 'interrupt', 'nostackframe',
+        'pascal', 'register', 'safecall', 'softfloat', 'stdcall',
+        'varargs', 'name', 'dynamic', 'near', 'virtual', 'external',
+        'override', 'assembler'
+    }
+
+    # XXX: those aren't global. but currently we know no way for defining
+    #      them just for the type context.
+    DIRECTIVES = {
+        'absolute', 'abstract', 'assembler', 'cppdecl', 'default', 'far',
+        'far16', 'forward', 'index', 'oldfpccall', 'private', 'protected',
+        'published', 'public'
+    }
+
+    BUILTIN_TYPES = {
+        'ansichar', 'ansistring', 'bool', 'boolean', 'byte', 'bytebool',
+        'cardinal', 'char', 'comp', 'currency', 'double', 'dword',
+        'extended', 'int64', 'integer', 'iunknown', 'longbool', 'longint',
+        'longword', 'pansichar', 'pansistring', 'pbool', 'pboolean',
+        'pbyte', 'pbytearray', 'pcardinal', 'pchar', 'pcomp', 'pcurrency',
+        'pdate', 'pdatetime', 'pdouble', 'pdword', 'pextended', 'phandle',
+        'pint64', 'pinteger', 'plongint', 'plongword', 'pointer',
+        'ppointer', 'pshortint', 'pshortstring', 'psingle', 'psmallint',
+        'pstring', 'pvariant', 'pwidechar', 'pwidestring', 'pword',
+        'pwordarray', 'pwordbool', 'real', 'real48', 'shortint',
+        'shortstring', 'single', 'smallint', 'string', 'tclass', 'tdate',
+        'tdatetime', 'textfile', 'thandle', 'tobject', 'ttime', 'variant',
+        'widechar', 'widestring', 'word', 'wordbool'
+    }
+
+    BUILTIN_UNITS = {
+        'System': (
+            'abs', 'acquireexceptionobject', 'addr', 'ansitoutf8',
+            'append', 'arctan', 'assert', 'assigned', 'assignfile',
+            'beginthread', 'blockread', 'blockwrite', 'break', 'chdir',
+            'chr', 'close', 'closefile', 'comptocurrency', 'comptodouble',
+            'concat', 'continue', 'copy', 'cos', 'dec', 'delete',
+            'dispose', 'doubletocomp', 'endthread', 'enummodules',
+            'enumresourcemodules', 'eof', 'eoln', 'erase', 'exceptaddr',
+            'exceptobject', 'exclude', 'exit', 'exp', 'filepos', 'filesize',
+            'fillchar', 'finalize', 'findclasshinstance', 'findhinstance',
+            'findresourcehinstance', 'flush', 'frac', 'freemem',
+            'get8087cw', 'getdir', 'getlasterror', 'getmem',
+            'getmemorymanager', 'getmodulefilename', 'getvariantmanager',
+            'halt', 'hi', 'high', 'inc', 'include', 'initialize', 'insert',
+            'int', 'ioresult', 'ismemorymanagerset', 'isvariantmanagerset',
+            'length', 'ln', 'lo', 'low', 'mkdir', 'move', 'new', 'odd',
+            'olestrtostring', 'olestrtostrvar', 'ord', 'paramcount',
+            'paramstr', 'pi', 'pos', 'pred', 'ptr', 'pucs4chars', 'random',
+            'randomize', 'read', 'readln', 'reallocmem',
+            'releaseexceptionobject', 'rename', 'reset', 'rewrite', 'rmdir',
+            'round', 'runerror', 'seek', 'seekeof', 'seekeoln',
+            'set8087cw', 'setlength', 'setlinebreakstyle',
+            'setmemorymanager', 'setstring', 'settextbuf',
+            'setvariantmanager', 'sin', 'sizeof', 'slice', 'sqr', 'sqrt',
+            'str', 'stringofchar', 'stringtoolestr', 'stringtowidechar',
+            'succ', 'swap', 'trunc', 'truncate', 'typeinfo',
+            'ucs4stringtowidestring', 'unicodetoutf8', 'uniquestring',
+            'upcase', 'utf8decode', 'utf8encode', 'utf8toansi',
+            'utf8tounicode', 'val', 'vararrayredim', 'varclear',
+            'widecharlentostring', 'widecharlentostrvar',
+            'widechartostring', 'widechartostrvar',
+            'widestringtoucs4string', 'write', 'writeln'
+        ),
+        'SysUtils': (
+            'abort', 'addexitproc', 'addterminateproc', 'adjustlinebreaks',
+            'allocmem', 'ansicomparefilename', 'ansicomparestr',
+            'ansicomparetext', 'ansidequotedstr', 'ansiextractquotedstr',
+            'ansilastchar', 'ansilowercase', 'ansilowercasefilename',
+            'ansipos', 'ansiquotedstr', 'ansisamestr', 'ansisametext',
+            'ansistrcomp', 'ansistricomp', 'ansistrlastchar', 'ansistrlcomp',
+            'ansistrlicomp', 'ansistrlower', 'ansistrpos', 'ansistrrscan',
+            'ansistrscan', 'ansistrupper', 'ansiuppercase',
+            'ansiuppercasefilename', 'appendstr', 'assignstr', 'beep',
+            'booltostr', 'bytetocharindex', 'bytetocharlen', 'bytetype',
+            'callterminateprocs', 'changefileext', 'charlength',
+            'chartobyteindex', 'chartobytelen', 'comparemem', 'comparestr',
+            'comparetext', 'createdir', 'createguid', 'currentyear',
+            'currtostr', 'currtostrf', 'date', 'datetimetofiledate',
+            'datetimetostr', 'datetimetostring', 'datetimetosystemtime',
+            'datetimetotimestamp', 'datetostr', 'dayofweek', 'decodedate',
+            'decodedatefully', 'decodetime', 'deletefile', 'directoryexists',
+            'diskfree', 'disksize', 'disposestr', 'encodedate', 'encodetime',
+            'exceptionerrormessage', 'excludetrailingbackslash',
+            'excludetrailingpathdelimiter', 'expandfilename',
+            'expandfilenamecase', 'expanduncfilename', 'extractfiledir',
+            'extractfiledrive', 'extractfileext', 'extractfilename',
+            'extractfilepath', 'extractrelativepath', 'extractshortpathname',
+            'fileage', 'fileclose', 'filecreate', 'filedatetodatetime',
+            'fileexists', 'filegetattr', 'filegetdate', 'fileisreadonly',
+            'fileopen', 'fileread', 'filesearch', 'fileseek', 'filesetattr',
+            'filesetdate', 'filesetreadonly', 'filewrite', 'finalizepackage',
+            'findclose', 'findcmdlineswitch', 'findfirst', 'findnext',
+            'floattocurr', 'floattodatetime', 'floattodecimal', 'floattostr',
+            'floattostrf', 'floattotext', 'floattotextfmt', 'fmtloadstr',
+            'fmtstr', 'forcedirectories', 'format', 'formatbuf', 'formatcurr',
+            'formatdatetime', 'formatfloat', 'freeandnil', 'getcurrentdir',
+            'getenvironmentvariable', 'getfileversion', 'getformatsettings',
+            'getlocaleformatsettings', 'getmodulename', 'getpackagedescription',
+            'getpackageinfo', 'gettime', 'guidtostring', 'incamonth',
+            'includetrailingbackslash', 'includetrailingpathdelimiter',
+            'incmonth', 'initializepackage', 'interlockeddecrement',
+            'interlockedexchange', 'interlockedexchangeadd',
+            'interlockedincrement', 'inttohex', 'inttostr', 'isdelimiter',
+            'isequalguid', 'isleapyear', 'ispathdelimiter', 'isvalidident',
+            'languages', 'lastdelimiter', 'loadpackage', 'loadstr',
+            'lowercase', 'msecstotimestamp', 'newstr', 'nextcharindex', 'now',
+            'outofmemoryerror', 'quotedstr', 'raiselastoserror',
+            'raiselastwin32error', 'removedir', 'renamefile', 'replacedate',
+            'replacetime', 'safeloadlibrary', 'samefilename', 'sametext',
+            'setcurrentdir', 'showexception', 'sleep', 'stralloc', 'strbufsize',
+            'strbytetype', 'strcat', 'strcharlength', 'strcomp', 'strcopy',
+            'strdispose', 'strecopy', 'strend', 'strfmt', 'stricomp',
+            'stringreplace', 'stringtoguid', 'strlcat', 'strlcomp', 'strlcopy',
+            'strlen', 'strlfmt', 'strlicomp', 'strlower', 'strmove', 'strnew',
+            'strnextchar', 'strpas', 'strpcopy', 'strplcopy', 'strpos',
+            'strrscan', 'strscan', 'strtobool', 'strtobooldef', 'strtocurr',
+            'strtocurrdef', 'strtodate', 'strtodatedef', 'strtodatetime',
+            'strtodatetimedef', 'strtofloat', 'strtofloatdef', 'strtoint',
+            'strtoint64', 'strtoint64def', 'strtointdef', 'strtotime',
+            'strtotimedef', 'strupper', 'supports', 'syserrormessage',
+            'systemtimetodatetime', 'texttofloat', 'time', 'timestamptodatetime',
+            'timestamptomsecs', 'timetostr', 'trim', 'trimleft', 'trimright',
+            'tryencodedate', 'tryencodetime', 'tryfloattocurr', 'tryfloattodatetime',
+            'trystrtobool', 'trystrtocurr', 'trystrtodate', 'trystrtodatetime',
+            'trystrtofloat', 'trystrtoint', 'trystrtoint64', 'trystrtotime',
+            'unloadpackage', 'uppercase', 'widecomparestr', 'widecomparetext',
+            'widefmtstr', 'wideformat', 'wideformatbuf', 'widelowercase',
+            'widesamestr', 'widesametext', 'wideuppercase', 'win32check',
+            'wraptext'
+        ),
+        'Classes': (
+            'activateclassgroup', 'allocatehwnd', 'bintohex', 'checksynchronize',
+            'collectionsequal', 'countgenerations', 'deallocatehwnd', 'equalrect',
+            'extractstrings', 'findclass', 'findglobalcomponent', 'getclass',
+            'groupdescendantswith', 'hextobin', 'identtoint',
+            'initinheritedcomponent', 'inttoident', 'invalidpoint',
+            'isuniqueglobalcomponentname', 'linestart', 'objectbinarytotext',
+            'objectresourcetotext', 'objecttexttobinary', 'objecttexttoresource',
+            'pointsequal', 'readcomponentres', 'readcomponentresex',
+            'readcomponentresfile', 'rect', 'registerclass', 'registerclassalias',
+            'registerclasses', 'registercomponents', 'registerintegerconsts',
+            'registernoicon', 'registernonactivex', 'smallpoint', 'startclassgroup',
+            'teststreamformat', 'unregisterclass', 'unregisterclasses',
+            'unregisterintegerconsts', 'unregistermoduleclasses',
+            'writecomponentresfile'
+        ),
+        'Math': (
+            'arccos', 'arccosh', 'arccot', 'arccoth', 'arccsc', 'arccsch', 'arcsec',
+            'arcsech', 'arcsin', 'arcsinh', 'arctan2', 'arctanh', 'ceil',
+            'comparevalue', 'cosecant', 'cosh', 'cot', 'cotan', 'coth', 'csc',
+            'csch', 'cycletodeg', 'cycletograd', 'cycletorad', 'degtocycle',
+            'degtograd', 'degtorad', 'divmod', 'doubledecliningbalance',
+            'ensurerange', 'floor', 'frexp', 'futurevalue', 'getexceptionmask',
+            'getprecisionmode', 'getroundmode', 'gradtocycle', 'gradtodeg',
+            'gradtorad', 'hypot', 'inrange', 'interestpayment', 'interestrate',
+            'internalrateofreturn', 'intpower', 'isinfinite', 'isnan', 'iszero',
+            'ldexp', 'lnxp1', 'log10', 'log2', 'logn', 'max', 'maxintvalue',
+            'maxvalue', 'mean', 'meanandstddev', 'min', 'minintvalue', 'minvalue',
+            'momentskewkurtosis', 'netpresentvalue', 'norm', 'numberofperiods',
+            'payment', 'periodpayment', 'poly', 'popnstddev', 'popnvariance',
+            'power', 'presentvalue', 'radtocycle', 'radtodeg', 'radtograd',
+            'randg', 'randomrange', 'roundto', 'samevalue', 'sec', 'secant',
+            'sech', 'setexceptionmask', 'setprecisionmode', 'setroundmode',
+            'sign', 'simpleroundto', 'sincos', 'sinh', 'slndepreciation', 'stddev',
+            'sum', 'sumint', 'sumofsquares', 'sumsandsquares', 'syddepreciation',
+            'tan', 'tanh', 'totalvariance', 'variance'
+        )
+    }
+
+    ASM_REGISTERS = {
+        'ah', 'al', 'ax', 'bh', 'bl', 'bp', 'bx', 'ch', 'cl', 'cr0',
+        'cr1', 'cr2', 'cr3', 'cr4', 'cs', 'cx', 'dh', 'di', 'dl', 'dr0',
+        'dr1', 'dr2', 'dr3', 'dr4', 'dr5', 'dr6', 'dr7', 'ds', 'dx',
+        'eax', 'ebp', 'ebx', 'ecx', 'edi', 'edx', 'es', 'esi', 'esp',
+        'fs', 'gs', 'mm0', 'mm1', 'mm2', 'mm3', 'mm4', 'mm5', 'mm6',
+        'mm7', 'si', 'sp', 'ss', 'st0', 'st1', 'st2', 'st3', 'st4', 'st5',
+        'st6', 'st7', 'xmm0', 'xmm1', 'xmm2', 'xmm3', 'xmm4', 'xmm5',
+        'xmm6', 'xmm7'
+    }
+
+    ASM_INSTRUCTIONS = {
+        'aaa', 'aad', 'aam', 'aas', 'adc', 'add', 'and', 'arpl', 'bound',
+        'bsf', 'bsr', 'bswap', 'bt', 'btc', 'btr', 'bts', 'call', 'cbw',
+        'cdq', 'clc', 'cld', 'cli', 'clts', 'cmc', 'cmova', 'cmovae',
+        'cmovb', 'cmovbe', 'cmovc', 'cmovcxz', 'cmove', 'cmovg',
+        'cmovge', 'cmovl', 'cmovle', 'cmovna', 'cmovnae', 'cmovnb',
+        'cmovnbe', 'cmovnc', 'cmovne', 'cmovng', 'cmovnge', 'cmovnl',
+        'cmovnle', 'cmovno', 'cmovnp', 'cmovns', 'cmovnz', 'cmovo',
+        'cmovp', 'cmovpe', 'cmovpo', 'cmovs', 'cmovz', 'cmp', 'cmpsb',
+        'cmpsd', 'cmpsw', 'cmpxchg', 'cmpxchg486', 'cmpxchg8b', 'cpuid',
+        'cwd', 'cwde', 'daa', 'das', 'dec', 'div', 'emms', 'enter', 'hlt',
+        'ibts', 'icebp', 'idiv', 'imul', 'in', 'inc', 'insb', 'insd',
+        'insw', 'int', 'int01', 'int03', 'int1', 'int3', 'into', 'invd',
+        'invlpg', 'iret', 'iretd', 'iretw', 'ja', 'jae', 'jb', 'jbe',
+        'jc', 'jcxz', 'jcxz', 'je', 'jecxz', 'jg', 'jge', 'jl', 'jle',
+        'jmp', 'jna', 'jnae', 'jnb', 'jnbe', 'jnc', 'jne', 'jng', 'jnge',
+        'jnl', 'jnle', 'jno', 'jnp', 'jns', 'jnz', 'jo', 'jp', 'jpe',
+        'jpo', 'js', 'jz', 'lahf', 'lar', 'lcall', 'lds', 'lea', 'leave',
+        'les', 'lfs', 'lgdt', 'lgs', 'lidt', 'ljmp', 'lldt', 'lmsw',
+        'loadall', 'loadall286', 'lock', 'lodsb', 'lodsd', 'lodsw',
+        'loop', 'loope', 'loopne', 'loopnz', 'loopz', 'lsl', 'lss', 'ltr',
+        'mov', 'movd', 'movq', 'movsb', 'movsd', 'movsw', 'movsx',
+        'movzx', 'mul', 'neg', 'nop', 'not', 'or', 'out', 'outsb', 'outsd',
+        'outsw', 'pop', 'popa', 'popad', 'popaw', 'popf', 'popfd', 'popfw',
+        'push', 'pusha', 'pushad', 'pushaw', 'pushf', 'pushfd', 'pushfw',
+        'rcl', 'rcr', 'rdmsr', 'rdpmc', 'rdshr', 'rdtsc', 'rep', 'repe',
+        'repne', 'repnz', 'repz', 'ret', 'retf', 'retn', 'rol', 'ror',
+        'rsdc', 'rsldt', 'rsm', 'sahf', 'sal', 'salc', 'sar', 'sbb',
+        'scasb', 'scasd', 'scasw', 'seta', 'setae', 'setb', 'setbe',
+        'setc', 'setcxz', 'sete', 'setg', 'setge', 'setl', 'setle',
+        'setna', 'setnae', 'setnb', 'setnbe', 'setnc', 'setne', 'setng',
+        'setnge', 'setnl', 'setnle', 'setno', 'setnp', 'setns', 'setnz',
+        'seto', 'setp', 'setpe', 'setpo', 'sets', 'setz', 'sgdt', 'shl',
+        'shld', 'shr', 'shrd', 'sidt', 'sldt', 'smi', 'smint', 'smintold',
+        'smsw', 'stc', 'std', 'sti', 'stosb', 'stosd', 'stosw', 'str',
+        'sub', 'svdc', 'svldt', 'svts', 'syscall', 'sysenter', 'sysexit',
+        'sysret', 'test', 'ud1', 'ud2', 'umov', 'verr', 'verw', 'wait',
+        'wbinvd', 'wrmsr', 'wrshr', 'xadd', 'xbts', 'xchg', 'xlat',
+        'xlatb', 'xor'
+    }
+
+    PORTUGOL_KEYWORDS = (
+        'aleatorio',
+        'algoritmo',
+        'arquivo',
+        'ate',
+        'caso',
+        'cronometro',
+        'debug',
+        'e',
+        'eco',
+        'enquanto',
+        'entao',
+        'escolha',
+        'escreva',
+        'escreval',
+        'faca',
+        'falso',
+        'fimalgoritmo',
+        'fimenquanto',
+        'fimescolha',
+        'fimfuncao',
+        'fimpara',
+        'fimprocedimento',
+        'fimrepita',
+        'fimse',
+        'funcao',
+        'inicio',
+        'int',
+        'interrompa',
+        'leia',
+        'limpatela',
+        'mod',
+        'nao',
+        'ou',
+        'outrocaso',
+        'para',
+        'passo',
+        'pausa',
+        'procedimento',
+        'repita',
+        'retorne',
+        'se',
+        'senao',
+        'timer',
+        'var',
+        'vetor',
+        'verdadeiro',
+        'xou',
+        'div',
+        'mod',
+        'abs',
+        'arccos',
+        'arcsen',
+        'arctan',
+        'cos',
+        'cotan',
+        'Exp',
+        'grauprad',
+        'int',
+        'log',
+        'logn',
+        'pi',
+        'quad',
+        'radpgrau',
+        'raizq',
+        'rand',
+        'randi',
+        'sen',
+        'Tan',
+        'asc',
+        'carac',
+        'caracpnum',
+        'compr',
+        'copia',
+        'maiusc',
+        'minusc',
+        'numpcarac',
+        'pos',
+    )
+
+    PORTUGOL_BUILTIN_TYPES = {
+        'inteiro', 'real', 'caractere', 'logico'
+    }
+
+    def __init__(self, **options):
+        Lexer.__init__(self, **options)
+        self.keywords = set()
+        self.builtins = set()
+        if get_bool_opt(options, 'portugol', False):
+            self.keywords.update(self.PORTUGOL_KEYWORDS)
+            self.builtins.update(self.PORTUGOL_BUILTIN_TYPES)
+            self.is_portugol = True
+        else:
+            self.is_portugol = False
+
+            if get_bool_opt(options, 'turbopascal', True):
+                self.keywords.update(self.TURBO_PASCAL_KEYWORDS)
+            if get_bool_opt(options, 'delphi', True):
+                self.keywords.update(self.DELPHI_KEYWORDS)
+            if get_bool_opt(options, 'freepascal', True):
+                self.keywords.update(self.FREE_PASCAL_KEYWORDS)
+            for unit in get_list_opt(options, 'units', list(self.BUILTIN_UNITS)):
+                self.builtins.update(self.BUILTIN_UNITS[unit])
+
+    def get_tokens_unprocessed(self, text):
+        scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE)
+        stack = ['initial']
+        in_function_block = False
+        in_property_block = False
+        was_dot = False
+        next_token_is_function = False
+        next_token_is_property = False
+        collect_labels = False
+        block_labels = set()
+        brace_balance = [0, 0]
+
+        while not scanner.eos:
+            token = Error
+
+            if stack[-1] == 'initial':
+                if scanner.scan(r'\s+'):
+                    token = Whitespace
+                elif not self.is_portugol and scanner.scan(r'\{.*?\}|\(\*.*?\*\)'):
+                    if scanner.match.startswith('$'):
+                        token = Comment.Preproc
+                    else:
+                        token = Comment.Multiline
+                elif scanner.scan(r'//.*?$'):
+                    token = Comment.Single
+                elif self.is_portugol and scanner.scan(r'(<\-)|(>=)|(<=)|%|<|>|-|\+|\*|\=|(<>)|\/|\.|:|,'):
+                    token = Operator
+                elif not self.is_portugol and scanner.scan(r'[-+*\/=<>:;,.@\^]'):
+                    token = Operator
+                    # stop label highlighting on next ";"
+                    if collect_labels and scanner.match == ';':
+                        collect_labels = False
+                elif scanner.scan(r'[\(\)\[\]]+'):
+                    token = Punctuation
+                    # abort function naming ``foo = Function(...)``
+                    next_token_is_function = False
+                    # if we are in a function block we count the open
+                    # braces because ootherwise it's impossible to
+                    # determine the end of the modifier context
+                    if in_function_block or in_property_block:
+                        if scanner.match == '(':
+                            brace_balance[0] += 1
+                        elif scanner.match == ')':
+                            brace_balance[0] -= 1
+                        elif scanner.match == '[':
+                            brace_balance[1] += 1
+                        elif scanner.match == ']':
+                            brace_balance[1] -= 1
+                elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'):
+                    lowercase_name = scanner.match.lower()
+                    if lowercase_name == 'result':
+                        token = Name.Builtin.Pseudo
+                    elif lowercase_name in self.keywords:
+                        token = Keyword
+                        # if we are in a special block and a
+                        # block ending keyword occurs (and the parenthesis
+                        # is balanced) we end the current block context
+                        if self.is_portugol:
+                            if lowercase_name in ('funcao', 'procedimento'):
+                                in_function_block = True
+                                next_token_is_function = True
+                        else:
+                            if (in_function_block or in_property_block) and \
+                                    lowercase_name in self.BLOCK_KEYWORDS and \
+                                    brace_balance[0] <= 0 and \
+                                    brace_balance[1] <= 0:
+                                in_function_block = False
+                                in_property_block = False
+                                brace_balance = [0, 0]
+                                block_labels = set()
+                            if lowercase_name in ('label', 'goto'):
+                                collect_labels = True
+                            elif lowercase_name == 'asm':
+                                stack.append('asm')
+                            elif lowercase_name == 'property':
+                                in_property_block = True
+                                next_token_is_property = True
+                            elif lowercase_name in ('procedure', 'operator',
+                                                    'function', 'constructor',
+                                                    'destructor'):
+                                in_function_block = True
+                                next_token_is_function = True
+                    # we are in a function block and the current name
+                    # is in the set of registered modifiers. highlight
+                    # it as pseudo keyword
+                    elif not self.is_portugol and in_function_block and \
+                            lowercase_name in self.FUNCTION_MODIFIERS:
+                        token = Keyword.Pseudo
+                    # if we are in a property highlight some more
+                    # modifiers
+                    elif not self.is_portugol and in_property_block and \
+                            lowercase_name in ('read', 'write'):
+                        token = Keyword.Pseudo
+                        next_token_is_function = True
+                    # if the last iteration set next_token_is_function
+                    # to true we now want this name highlighted as
+                    # function. so do that and reset the state
+                    elif next_token_is_function:
+                        # Look if the next token is a dot. If yes it's
+                        # not a function, but a class name and the
+                        # part after the dot a function name
+                        if not self.is_portugol and scanner.test(r'\s*\.\s*'):
+                            token = Name.Class
+                        # it's not a dot, our job is done
+                        else:
+                            token = Name.Function
+                            next_token_is_function = False
+
+                            if self.is_portugol:
+                                block_labels.add(scanner.match.lower())
+
+                    # same for properties
+                    elif not self.is_portugol and next_token_is_property:
+                        token = Name.Property
+                        next_token_is_property = False
+                    # Highlight this token as label and add it
+                    # to the list of known labels
+                    elif not self.is_portugol and collect_labels:
+                        token = Name.Label
+                        block_labels.add(scanner.match.lower())
+                    # name is in list of known labels
+                    elif lowercase_name in block_labels:
+                        token = Name.Label
+                    elif self.is_portugol and lowercase_name in self.PORTUGOL_BUILTIN_TYPES:
+                        token = Keyword.Type
+                    elif not self.is_portugol and lowercase_name in self.BUILTIN_TYPES:
+                        token = Keyword.Type
+                    elif not self.is_portugol and lowercase_name in self.DIRECTIVES:
+                        token = Keyword.Pseudo
+                    # builtins are just builtins if the token
+                    # before isn't a dot
+                    elif not self.is_portugol and not was_dot and lowercase_name in self.builtins:
+                        token = Name.Builtin
+                    else:
+                        token = Name
+                elif self.is_portugol and scanner.scan(r"\""):
+                    token = String
+                    stack.append('string')
+                elif not self.is_portugol and scanner.scan(r"'"):
+                    token = String
+                    stack.append('string')
+                elif not self.is_portugol and scanner.scan(r'\#(\d+|\$[0-9A-Fa-f]+)'):
+                    token = String.Char
+                elif not self.is_portugol and scanner.scan(r'\$[0-9A-Fa-f]+'):
+                    token = Number.Hex
+                elif scanner.scan(r'\d+(?![eE]|\.[^.])'):
+                    token = Number.Integer
+                elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'):
+                    token = Number.Float
+                else:
+                    # if the stack depth is deeper than once, pop
+                    if len(stack) > 1:
+                        stack.pop()
+                    scanner.get_char()
+
+            elif stack[-1] == 'string':
+                if self.is_portugol:
+                    if scanner.scan(r"''"):
+                        token = String.Escape
+                    elif scanner.scan(r"\""):
+                        token = String
+                        stack.pop()
+                    elif scanner.scan(r"[^\"]*"):
+                        token = String
+                    else:
+                        scanner.get_char()
+                        stack.pop()
+                else:
+                    if scanner.scan(r"''"):
+                        token = String.Escape
+                    elif scanner.scan(r"'"):
+                        token = String
+                        stack.pop()
+                    elif scanner.scan(r"[^']*"):
+                        token = String
+                    else:
+                        scanner.get_char()
+                        stack.pop()
+            elif not self.is_portugol and stack[-1] == 'asm':
+                if scanner.scan(r'\s+'):
+                    token = Whitespace
+                elif scanner.scan(r'end'):
+                    token = Keyword
+                    stack.pop()
+                elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'):
+                    if scanner.match.startswith('$'):
+                        token = Comment.Preproc
+                    else:
+                        token = Comment.Multiline
+                elif scanner.scan(r'//.*?$'):
+                    token = Comment.Single
+                elif scanner.scan(r"'"):
+                    token = String
+                    stack.append('string')
+                elif scanner.scan(r'@@[A-Za-z_][A-Za-z_0-9]*'):
+                    token = Name.Label
+                elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'):
+                    lowercase_name = scanner.match.lower()
+                    if lowercase_name in self.ASM_INSTRUCTIONS:
+                        token = Keyword
+                    elif lowercase_name in self.ASM_REGISTERS:
+                        token = Name.Builtin
+                    else:
+                        token = Name
+                elif scanner.scan(r'[-+*\/=<>:;,.@\^]+'):
+                    token = Operator
+                elif scanner.scan(r'[\(\)\[\]]+'):
+                    token = Punctuation
+                elif scanner.scan(r'\$[0-9A-Fa-f]+'):
+                    token = Number.Hex
+                elif scanner.scan(r'\d+(?![eE]|\.[^.])'):
+                    token = Number.Integer
+                elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'):
+                    token = Number.Float
+                else:
+                    scanner.get_char()
+                    stack.pop()
+
+            # save the dot!!!11
+            if not self.is_portugol and scanner.match.strip():
+                was_dot = scanner.match == '.'
+
+            yield scanner.start_pos, token, scanner.match or ''
diff --git a/lib/pygments/lexers/pawn.py b/lib/pygments/lexers/pawn.py
new file mode 100644
index 0000000..99d9c96
--- /dev/null
+++ b/lib/pygments/lexers/pawn.py
@@ -0,0 +1,202 @@
+"""
+    pygments.lexers.pawn
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Pawn languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+from pygments.util import get_bool_opt
+
+__all__ = ['SourcePawnLexer', 'PawnLexer']
+
+
+class SourcePawnLexer(RegexLexer):
+    """
+    For SourcePawn source code with preprocessor directives.
+    """
+    name = 'SourcePawn'
+    aliases = ['sp']
+    filenames = ['*.sp']
+    mimetypes = ['text/x-sourcepawn']
+    url = 'https://github.com/alliedmodders/sourcepawn'
+    version_added = '1.6'
+
+    #: optional Comment or Whitespace
+    _ws = r'(?:\s|//.*?\n|/\*.*?\*/)+'
+    #: only one /* */ style comment
+    _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+
+    tokens = {
+        'root': [
+            # preprocessor directives: without whitespace
+            (r'^#if\s+0', Comment.Preproc, 'if0'),
+            ('^#', Comment.Preproc, 'macro'),
+            # or with whitespace
+            ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'),
+            ('^' + _ws1 + '#', Comment.Preproc, 'macro'),
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'\\\n', Text),  # line continuation
+            (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
+            (r'[{}]', Punctuation),
+            (r'L?"', String, 'string'),
+            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
+            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+            (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+            (r'0[0-7]+[LlUu]*', Number.Oct),
+            (r'\d+[LlUu]*', Number.Integer),
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+            (r'[()\[\],.;]', Punctuation),
+            (r'(case|const|continue|native|'
+             r'default|else|enum|for|if|new|operator|'
+             r'public|return|sizeof|static|decl|struct|switch)\b', Keyword),
+            (r'(bool|Float)\b', Keyword.Type),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+            (r'[^\\"\n]+', String),  # all other characters
+            (r'\\\n', String),       # line continuation
+            (r'\\', String),         # stray backslash
+        ],
+        'macro': [
+            (r'[^/\n]+', Comment.Preproc),
+            (r'/\*(.|\n)*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
+            (r'/', Comment.Preproc),
+            (r'(?<=\\)\n', Comment.Preproc),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'if0': [
+            (r'^\s*#if.*?(?/-]', Operator),
+            (r'[()\[\],.;]', Punctuation),
+            (r'(switch|case|default|const|new|static|char|continue|break|'
+             r'if|else|for|while|do|operator|enum|'
+             r'public|return|sizeof|tagof|state|goto)\b', Keyword),
+            (r'(bool|Float)\b', Keyword.Type),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+            (r'[^\\"\n]+', String),  # all other characters
+            (r'\\\n', String),       # line continuation
+            (r'\\', String),         # stray backslash
+        ],
+        'macro': [
+            (r'[^/\n]+', Comment.Preproc),
+            (r'/\*(.|\n)*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
+            (r'/', Comment.Preproc),
+            (r'(?<=\\)\n', Comment.Preproc),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'if0': [
+            (r'^\s*#if.*?(?<-]', Operator),
+            (r'[a-zA-Z][a-zA-Z0-9_-]*', Name),
+            (r'\?[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable),
+            (r'[0-9]+\.[0-9]+', Number.Float),
+            (r'[0-9]+', Number.Integer),
+        ],
+        'keywords': [
+            (words((
+                ':requirements', ':types', ':constants',
+                ':predicates', ':functions', ':action', ':agent',
+                ':parameters', ':precondition', ':effect',
+                ':durative-action', ':duration', ':condition',
+                ':derived', ':domain', ':objects', ':init',
+                ':goal', ':metric', ':length', ':serial', ':parallel',
+                # the following are requirements
+                ':strips', ':typing', ':negative-preconditions',
+                ':disjunctive-preconditions', ':equality',
+                ':existential-preconditions', ':universal-preconditions',
+                ':conditional-effects', ':fluents', ':numeric-fluents',
+                ':object-fluents', ':adl', ':durative-actions',
+                ':continuous-effects', ':derived-predicates',
+                ':time-intial-literals', ':preferences',
+                ':constraints', ':action-costs', ':multi-agent',
+                ':unfactored-privacy', ':factored-privacy',
+                ':non-deterministic'
+                ), suffix=r'\b'), Keyword)
+        ],
+        'builtins': [
+            (words((
+                'define', 'domain', 'object', 'either', 'and',
+                'forall', 'preference', 'imply', 'or', 'exists',
+                'not', 'when', 'assign', 'scale-up', 'scale-down',
+                'increase', 'decrease', 'at', 'over', 'start',
+                'end', 'all', 'problem', 'always', 'sometime',
+                'within', 'at-most-once', 'sometime-after',
+                'sometime-before', 'always-within', 'hold-during',
+                'hold-after', 'minimize', 'maximize',
+                'total-time', 'is-violated'), suffix=r'\b'),
+                Name.Builtin)
+        ]
+    }
+
diff --git a/lib/pygments/lexers/perl.py b/lib/pygments/lexers/perl.py
new file mode 100644
index 0000000..33f91f5
--- /dev/null
+++ b/lib/pygments/lexers/perl.py
@@ -0,0 +1,733 @@
+"""
+    pygments.lexers.perl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Perl, Raku and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, bygroups, \
+    using, this, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+from pygments.util import shebang_matches
+
+__all__ = ['PerlLexer', 'Perl6Lexer']
+
+
+class PerlLexer(RegexLexer):
+    """
+    For Perl source code.
+    """
+
+    name = 'Perl'
+    url = 'https://www.perl.org'
+    aliases = ['perl', 'pl']
+    filenames = ['*.pl', '*.pm', '*.t', '*.perl']
+    mimetypes = ['text/x-perl', 'application/x-perl']
+    version_added = ''
+
+    flags = re.DOTALL | re.MULTILINE
+    # TODO: give this to a perl guy who knows how to parse perl...
+    tokens = {
+        'balanced-regex': [
+            (r'/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*', String.Regex, '#pop'),
+            (r'!(\\\\|\\[^\\]|[^\\!])*![egimosx]*', String.Regex, '#pop'),
+            (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
+            (r'\{(\\\\|\\[^\\]|[^\\}])*\}[egimosx]*', String.Regex, '#pop'),
+            (r'<(\\\\|\\[^\\]|[^\\>])*>[egimosx]*', String.Regex, '#pop'),
+            (r'\[(\\\\|\\[^\\]|[^\\\]])*\][egimosx]*', String.Regex, '#pop'),
+            (r'\((\\\\|\\[^\\]|[^\\)])*\)[egimosx]*', String.Regex, '#pop'),
+            (r'@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*', String.Regex, '#pop'),
+            (r'%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*', String.Regex, '#pop'),
+            (r'\$(\\\\|\\[^\\]|[^\\$])*\$[egimosx]*', String.Regex, '#pop'),
+        ],
+        'root': [
+            (r'\A\#!.+?$', Comment.Hashbang),
+            (r'\#.*?$', Comment.Single),
+            (r'^=[a-zA-Z0-9]+\s+.*?\n=cut', Comment.Multiline),
+            (words((
+                'case', 'continue', 'do', 'else', 'elsif', 'for', 'foreach',
+                'if', 'last', 'my', 'next', 'our', 'redo', 'reset', 'then',
+                'unless', 'until', 'while', 'print', 'new', 'BEGIN',
+                'CHECK', 'INIT', 'END', 'return'), suffix=r'\b'),
+             Keyword),
+            (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)',
+             bygroups(Keyword, Whitespace, Name, Whitespace, Punctuation, Whitespace), 'format'),
+            (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word),
+            # common delimiters
+            (r's/(\\\\|\\[^\\]|[^\\/])*/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*',
+                String.Regex),
+            (r's!(\\\\|\\!|[^!])*!(\\\\|\\!|[^!])*![egimosx]*', String.Regex),
+            (r's\\(\\\\|[^\\])*\\(\\\\|[^\\])*\\[egimosx]*', String.Regex),
+            (r's@(\\\\|\\[^\\]|[^\\@])*@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*',
+                String.Regex),
+            (r's%(\\\\|\\[^\\]|[^\\%])*%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*',
+                String.Regex),
+            # balanced delimiters
+            (r's\{(\\\\|\\[^\\]|[^\\}])*\}\s*', String.Regex, 'balanced-regex'),
+            (r's<(\\\\|\\[^\\]|[^\\>])*>\s*', String.Regex, 'balanced-regex'),
+            (r's\[(\\\\|\\[^\\]|[^\\\]])*\]\s*', String.Regex,
+                'balanced-regex'),
+            (r's\((\\\\|\\[^\\]|[^\\)])*\)\s*', String.Regex,
+                'balanced-regex'),
+
+            (r'm?/(\\\\|\\[^\\]|[^\\/\n])*/[gcimosx]*', String.Regex),
+            (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),
+            (r'((?<==~)|(?<=\())\s*/(\\\\|\\[^\\]|[^\\/])*/[gcimosx]*',
+                String.Regex),
+            (r'\s+', Whitespace),
+            (words((
+                'abs', 'accept', 'alarm', 'atan2', 'bind', 'binmode', 'bless', 'caller', 'chdir',
+                'chmod', 'chomp', 'chop', 'chown', 'chr', 'chroot', 'close', 'closedir', 'connect',
+                'continue', 'cos', 'crypt', 'dbmclose', 'dbmopen', 'defined', 'delete', 'die',
+                'dump', 'each', 'endgrent', 'endhostent', 'endnetent', 'endprotoent',
+                'endpwent', 'endservent', 'eof', 'eval', 'exec', 'exists', 'exit', 'exp', 'fcntl',
+                'fileno', 'flock', 'fork', 'format', 'formline', 'getc', 'getgrent', 'getgrgid',
+                'getgrnam', 'gethostbyaddr', 'gethostbyname', 'gethostent', 'getlogin',
+                'getnetbyaddr', 'getnetbyname', 'getnetent', 'getpeername', 'getpgrp',
+                'getppid', 'getpriority', 'getprotobyname', 'getprotobynumber',
+                'getprotoent', 'getpwent', 'getpwnam', 'getpwuid', 'getservbyname',
+                'getservbyport', 'getservent', 'getsockname', 'getsockopt', 'glob', 'gmtime',
+                'goto', 'grep', 'hex', 'import', 'index', 'int', 'ioctl', 'join', 'keys', 'kill', 'last',
+                'lc', 'lcfirst', 'length', 'link', 'listen', 'local', 'localtime', 'log', 'lstat',
+                'map', 'mkdir', 'msgctl', 'msgget', 'msgrcv', 'msgsnd', 'my', 'next', 'oct', 'open',
+                'opendir', 'ord', 'our', 'pack', 'pipe', 'pop', 'pos', 'printf',
+                'prototype', 'push', 'quotemeta', 'rand', 'read', 'readdir',
+                'readline', 'readlink', 'readpipe', 'recv', 'redo', 'ref', 'rename',
+                'reverse', 'rewinddir', 'rindex', 'rmdir', 'scalar', 'seek', 'seekdir',
+                'select', 'semctl', 'semget', 'semop', 'send', 'setgrent', 'sethostent', 'setnetent',
+                'setpgrp', 'setpriority', 'setprotoent', 'setpwent', 'setservent',
+                'setsockopt', 'shift', 'shmctl', 'shmget', 'shmread', 'shmwrite', 'shutdown',
+                'sin', 'sleep', 'socket', 'socketpair', 'sort', 'splice', 'split', 'sprintf', 'sqrt',
+                'srand', 'stat', 'study', 'substr', 'symlink', 'syscall', 'sysopen', 'sysread',
+                'sysseek', 'system', 'syswrite', 'tell', 'telldir', 'tie', 'tied', 'time', 'times', 'tr',
+                'truncate', 'uc', 'ucfirst', 'umask', 'undef', 'unlink', 'unpack', 'unshift', 'untie',
+                'utime', 'values', 'vec', 'wait', 'waitpid', 'wantarray', 'warn', 'write'), suffix=r'\b'),
+             Name.Builtin),
+            (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo),
+            (r'(<<)([\'"]?)([a-zA-Z_]\w*)(\2;?\n.*?\n)(\3)(\n)',
+             bygroups(String, String, String.Delimiter, String, String.Delimiter, Whitespace)),
+            (r'__END__', Comment.Preproc, 'end-part'),
+            (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global),
+            (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global),
+            (r'[$@%#]+', Name.Variable, 'varname'),
+            (r'0_?[0-7]+(_[0-7]+)*', Number.Oct),
+            (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex),
+            (r'0b[01]+(_[01]+)*', Number.Bin),
+            (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?',
+             Number.Float),
+            (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float),
+            (r'\d+(_\d+)*', Number.Integer),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            (r'`(\\\\|\\[^\\]|[^`\\])*`', String.Backtick),
+            (r'<([^\s>]+)>', String.Regex),
+            (r'(q|qq|qw|qr|qx)\{', String.Other, 'cb-string'),
+            (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'),
+            (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'),
+            (r'(q|qq|qw|qr|qx)\<', String.Other, 'lt-string'),
+            (r'(q|qq|qw|qr|qx)([\W_])(.|\n)*?\2', String.Other),
+            (r'(package)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Whitespace, Name.Namespace)),
+            (r'(use|require|no)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Whitespace, Name.Namespace)),
+            (r'(sub)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (words((
+                'no', 'package', 'require', 'use'), suffix=r'\b'),
+             Keyword),
+            (r'(\[\]|\*\*|::|<<|>>|>=|<=>|<=|={3}|!=|=~|'
+             r'!~|&&?|\|\||\.{1,3})', Operator),
+            (r'[-+/*%=<>&^|!\\~]=?', Operator),
+            (r'[()\[\]:;,<>/?{}]', Punctuation),  # yes, there's no shortage
+                                                  # of punctuation in Perl!
+            (r'(?=\w)', Name, 'name'),
+        ],
+        'format': [
+            (r'\.\n', String.Interpol, '#pop'),
+            (r'[^\n]*\n', String.Interpol),
+        ],
+        'varname': [
+            (r'\s+', Whitespace),
+            (r'\{', Punctuation, '#pop'),    # hash syntax?
+            (r'\)|,', Punctuation, '#pop'),  # argument specifier
+            (r'\w+::', Name.Namespace),
+            (r'[\w:]+', Name.Variable, '#pop'),
+        ],
+        'name': [
+            (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*(::)?(?=\s*->)', Name.Namespace, '#pop'),
+            (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*::', Name.Namespace, '#pop'),
+            (r'[\w:]+', Name, '#pop'),
+            (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'),
+            (r'(?=\W)', Text, '#pop'),
+        ],
+        'funcname': [
+            (r'[a-zA-Z_]\w*[!?]?', Name.Function),
+            (r'\s+', Whitespace),
+            # argument declaration
+            (r'(\([$@%]*\))(\s*)', bygroups(Punctuation, Whitespace)),
+            (r';', Punctuation, '#pop'),
+            (r'.*?\{', Punctuation, '#pop'),
+        ],
+        'cb-string': [
+            (r'\\[{}\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\{', String.Other, 'cb-string'),
+            (r'\}', String.Other, '#pop'),
+            (r'[^{}\\]+', String.Other)
+        ],
+        'rb-string': [
+            (r'\\[()\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\(', String.Other, 'rb-string'),
+            (r'\)', String.Other, '#pop'),
+            (r'[^()]+', String.Other)
+        ],
+        'sb-string': [
+            (r'\\[\[\]\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\[', String.Other, 'sb-string'),
+            (r'\]', String.Other, '#pop'),
+            (r'[^\[\]]+', String.Other)
+        ],
+        'lt-string': [
+            (r'\\[<>\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\<', String.Other, 'lt-string'),
+            (r'\>', String.Other, '#pop'),
+            (r'[^<>]+', String.Other)
+        ],
+        'end-part': [
+            (r'.+', Comment.Preproc, '#pop')
+        ]
+    }
+
+    def analyse_text(text):
+        if shebang_matches(text, r'perl'):
+            return True
+
+        result = 0
+
+        if re.search(r'(?:my|our)\s+[$@%(]', text):
+            result += 0.9
+
+        if ':=' in text:
+            # := is not valid Perl, but it appears in unicon, so we should
+            # become less confident if we think we found Perl with :=
+            result /= 2
+
+        return result
+
+
+class Perl6Lexer(ExtendedRegexLexer):
+    """
+    For Raku (a.k.a. Perl 6) source code.
+    """
+
+    name = 'Perl6'
+    url = 'https://www.raku.org'
+    aliases = ['perl6', 'pl6', 'raku']
+    filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6',
+                 '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod',
+                 '*.rakutest', '*.rakudoc']
+    mimetypes = ['text/x-perl6', 'application/x-perl6']
+    version_added = '2.0'
+    flags = re.MULTILINE | re.DOTALL
+
+    PERL6_IDENTIFIER_RANGE = r"['\w:-]"
+
+    PERL6_KEYWORDS = (
+        #Phasers
+        'BEGIN','CATCH','CHECK','CLOSE','CONTROL','DOC','END','ENTER','FIRST',
+        'INIT','KEEP','LAST','LEAVE','NEXT','POST','PRE','QUIT','UNDO',
+        #Keywords
+        'anon','augment','but','class','constant','default','does','else',
+        'elsif','enum','for','gather','given','grammar','has','if','import',
+        'is','let','loop','made','make','method','module','multi','my','need',
+        'orwith','our','proceed','proto','repeat','require','return',
+        'return-rw','returns','role','rule','state','sub','submethod','subset',
+        'succeed','supersede','token','try','unit','unless','until','use',
+        'when','while','with','without',
+        #Traits
+        'export','native','repr','required','rw','symbol',
+    )
+
+    PERL6_BUILTINS = (
+        'ACCEPTS','abs','abs2rel','absolute','accept','accessed','acos',
+        'acosec','acosech','acosh','acotan','acotanh','acquire','act','action',
+        'actions','add','add_attribute','add_enum_value','add_fallback',
+        'add_method','add_parent','add_private_method','add_role','add_trustee',
+        'adverb','after','all','allocate','allof','allowed','alternative-names',
+        'annotations','antipair','antipairs','any','anyof','app_lifetime',
+        'append','arch','archname','args','arity','Array','asec','asech','asin',
+        'asinh','ASSIGN-KEY','ASSIGN-POS','assuming','ast','at','atan','atan2',
+        'atanh','AT-KEY','atomic-assign','atomic-dec-fetch','atomic-fetch',
+        'atomic-fetch-add','atomic-fetch-dec','atomic-fetch-inc',
+        'atomic-fetch-sub','atomic-inc-fetch','AT-POS','attributes','auth',
+        'await','backtrace','Bag','BagHash','bail-out','base','basename',
+        'base-repeating','batch','BIND-KEY','BIND-POS','bind-stderr',
+        'bind-stdin','bind-stdout','bind-udp','bits','bless','block','Bool',
+        'bool-only','bounds','break','Bridge','broken','BUILD','build-date',
+        'bytes','cache','callframe','calling-package','CALL-ME','callsame',
+        'callwith','can','cancel','candidates','cando','can-ok','canonpath',
+        'caps','caption','Capture','cas','catdir','categorize','categorize-list',
+        'catfile','catpath','cause','ceiling','cglobal','changed','Channel',
+        'chars','chdir','child','child-name','child-typename','chmod','chomp',
+        'chop','chr','chrs','chunks','cis','classify','classify-list','cleanup',
+        'clone','close','closed','close-stdin','cmp-ok','code','codes','collate',
+        'column','comb','combinations','command','comment','compiler','Complex',
+        'compose','compose_type','composer','condition','config',
+        'configure_destroy','configure_type_checking','conj','connect',
+        'constraints','construct','contains','contents','copy','cos','cosec',
+        'cosech','cosh','cotan','cotanh','count','count-only','cpu-cores',
+        'cpu-usage','CREATE','create_type','cross','cue','curdir','curupdir','d',
+        'Date','DateTime','day','daycount','day-of-month','day-of-week',
+        'day-of-year','days-in-month','declaration','decode','decoder','deepmap',
+        'default','defined','DEFINITE','delayed','DELETE-KEY','DELETE-POS',
+        'denominator','desc','DESTROY','destroyers','devnull','diag',
+        'did-you-mean','die','dies-ok','dir','dirname','dir-sep','DISTROnames',
+        'do','does','does-ok','done','done-testing','duckmap','dynamic','e',
+        'eager','earlier','elems','emit','enclosing','encode','encoder',
+        'encoding','end','ends-with','enum_from_value','enum_value_list',
+        'enum_values','enums','eof','EVAL','eval-dies-ok','EVALFILE',
+        'eval-lives-ok','exception','excludes-max','excludes-min','EXISTS-KEY',
+        'EXISTS-POS','exit','exitcode','exp','expected','explicitly-manage',
+        'expmod','extension','f','fail','fails-like','fc','feature','file',
+        'filename','find_method','find_method_qualified','finish','first','flat',
+        'flatmap','flip','floor','flunk','flush','fmt','format','formatter',
+        'freeze','from','from-list','from-loop','from-posix','full',
+        'full-barrier','get','get_value','getc','gist','got','grab','grabpairs',
+        'grep','handle','handled','handles','hardware','has_accessor','Hash',
+        'head','headers','hh-mm-ss','hidden','hides','hour','how','hyper','id',
+        'illegal','im','in','indent','index','indices','indir','infinite',
+        'infix','infix:<+>','infix:<->','install_method_cache','Instant',
+        'instead','Int','int-bounds','interval','in-timezone','invalid-str',
+        'invert','invocant','IO','IO::Notification.watch-path','is_trusted',
+        'is_type','isa','is-absolute','isa-ok','is-approx','is-deeply',
+        'is-hidden','is-initial-thread','is-int','is-lazy','is-leap-year',
+        'isNaN','isnt','is-prime','is-relative','is-routine','is-setting',
+        'is-win','item','iterator','join','keep','kept','KERNELnames','key',
+        'keyof','keys','kill','kv','kxxv','l','lang','last','lastcall','later',
+        'lazy','lc','leading','level','like','line','lines','link','List',
+        'listen','live','lives-ok','local','lock','log','log10','lookup','lsb',
+        'made','MAIN','make','Map','match','max','maxpairs','merge','message',
+        'method','method_table','methods','migrate','min','minmax','minpairs',
+        'minute','misplaced','Mix','MixHash','mkdir','mode','modified','month',
+        'move','mro','msb','multi','multiness','my','name','named','named_names',
+        'narrow','nativecast','native-descriptor','nativesizeof','new','new_type',
+        'new-from-daycount','new-from-pairs','next','nextcallee','next-handle',
+        'nextsame','nextwith','NFC','NFD','NFKC','NFKD','nl-in','nl-out',
+        'nodemap','nok','none','norm','not','note','now','nude','Num',
+        'numerator','Numeric','of','offset','offset-in-hours','offset-in-minutes',
+        'ok','old','on-close','one','on-switch','open','opened','operation',
+        'optional','ord','ords','orig','os-error','osname','out-buffer','pack',
+        'package','package-kind','package-name','packages','pair','pairs',
+        'pairup','parameter','params','parent','parent-name','parents','parse',
+        'parse-base','parsefile','parse-names','parts','pass','path','path-sep',
+        'payload','peer-host','peer-port','periods','perl','permutations','phaser',
+        'pick','pickpairs','pid','placeholder','plan','plus','polar','poll',
+        'polymod','pop','pos','positional','posix','postfix','postmatch',
+        'precomp-ext','precomp-target','pred','prefix','prematch','prepend',
+        'print','printf','print-nl','print-to','private','private_method_table',
+        'proc','produce','Promise','prompt','protect','pull-one','push',
+        'push-all','push-at-least','push-exactly','push-until-lazy','put',
+        'qualifier-type','quit','r','race','radix','rand','range','Rat','raw',
+        're','read','readchars','readonly','ready','Real','reallocate','reals',
+        'reason','rebless','receive','recv','redispatcher','redo','reduce',
+        'rel2abs','relative','release','rename','repeated','replacement',
+        'report','reserved','resolve','restore','result','resume','rethrow',
+        'reverse','right','rindex','rmdir','role','roles_to_compose','rolish',
+        'roll','rootdir','roots','rotate','rotor','round','roundrobin',
+        'routine-type','run','rwx','s','samecase','samemark','samewith','say',
+        'schedule-on','scheduler','scope','sec','sech','second','seek','self',
+        'send','Set','set_hidden','set_name','set_package','set_rw','set_value',
+        'SetHash','set-instruments','setup_finalization','shape','share','shell',
+        'shift','sibling','sigil','sign','signal','signals','signature','sin',
+        'sinh','sink','sink-all','skip','skip-at-least','skip-at-least-pull-one',
+        'skip-one','skip-rest','sleep','sleep-timer','sleep-until','Slip','slurp',
+        'slurp-rest','slurpy','snap','snapper','so','socket-host','socket-port',
+        'sort','source','source-package','spawn','SPEC','splice','split',
+        'splitdir','splitpath','sprintf','spurt','sqrt','squish','srand','stable',
+        'start','started','starts-with','status','stderr','stdout','Str',
+        'sub_signature','subbuf','subbuf-rw','subname','subparse','subst',
+        'subst-mutate','substr','substr-eq','substr-rw','subtest','succ','sum',
+        'Supply','symlink','t','tail','take','take-rw','tan','tanh','tap',
+        'target','target-name','tc','tclc','tell','then','throttle','throw',
+        'throws-like','timezone','tmpdir','to','today','todo','toggle','to-posix',
+        'total','trailing','trans','tree','trim','trim-leading','trim-trailing',
+        'truncate','truncated-to','trusts','try_acquire','trying','twigil','type',
+        'type_captures','typename','uc','udp','uncaught_handler','unimatch',
+        'uniname','uninames','uniparse','uniprop','uniprops','unique','unival',
+        'univals','unlike','unlink','unlock','unpack','unpolar','unshift',
+        'unwrap','updir','USAGE','use-ok','utc','val','value','values','VAR',
+        'variable','verbose-config','version','VMnames','volume','vow','w','wait',
+        'warn','watch','watch-path','week','weekday-of-month','week-number',
+        'week-year','WHAT','when','WHERE','WHEREFORE','WHICH','WHO',
+        'whole-second','WHY','wordcase','words','workaround','wrap','write',
+        'write-to','x','yada','year','yield','yyyy-mm-dd','z','zip','zip-latest',
+
+    )
+
+    PERL6_BUILTIN_CLASSES = (
+        #Booleans
+        'False','True',
+        #Classes
+        'Any','Array','Associative','AST','atomicint','Attribute','Backtrace',
+        'Backtrace::Frame','Bag','Baggy','BagHash','Blob','Block','Bool','Buf',
+        'Callable','CallFrame','Cancellation','Capture','CArray','Channel','Code',
+        'compiler','Complex','ComplexStr','Cool','CurrentThreadScheduler',
+        'Cursor','Date','Dateish','DateTime','Distro','Duration','Encoding',
+        'Exception','Failure','FatRat','Grammar','Hash','HyperWhatever','Instant',
+        'Int','int16','int32','int64','int8','IntStr','IO','IO::ArgFiles',
+        'IO::CatHandle','IO::Handle','IO::Notification','IO::Path',
+        'IO::Path::Cygwin','IO::Path::QNX','IO::Path::Unix','IO::Path::Win32',
+        'IO::Pipe','IO::Socket','IO::Socket::Async','IO::Socket::INET','IO::Spec',
+        'IO::Spec::Cygwin','IO::Spec::QNX','IO::Spec::Unix','IO::Spec::Win32',
+        'IO::Special','Iterable','Iterator','Junction','Kernel','Label','List',
+        'Lock','Lock::Async','long','longlong','Macro','Map','Match',
+        'Metamodel::AttributeContainer','Metamodel::C3MRO','Metamodel::ClassHOW',
+        'Metamodel::EnumHOW','Metamodel::Finalization','Metamodel::MethodContainer',
+        'Metamodel::MROBasedMethodDispatch','Metamodel::MultipleInheritance',
+        'Metamodel::Naming','Metamodel::Primitives','Metamodel::PrivateMethodContainer',
+        'Metamodel::RoleContainer','Metamodel::Trusting','Method','Mix','MixHash',
+        'Mixy','Mu','NFC','NFD','NFKC','NFKD','Nil','Num','num32','num64',
+        'Numeric','NumStr','ObjAt','Order','Pair','Parameter','Perl','Pod::Block',
+        'Pod::Block::Code','Pod::Block::Comment','Pod::Block::Declarator',
+        'Pod::Block::Named','Pod::Block::Para','Pod::Block::Table','Pod::Heading',
+        'Pod::Item','Pointer','Positional','PositionalBindFailover','Proc',
+        'Proc::Async','Promise','Proxy','PseudoStash','QuantHash','Range','Rat',
+        'Rational','RatStr','Real','Regex','Routine','Scalar','Scheduler',
+        'Semaphore','Seq','Set','SetHash','Setty','Signature','size_t','Slip',
+        'Stash','Str','StrDistance','Stringy','Sub','Submethod','Supplier',
+        'Supplier::Preserving','Supply','Systemic','Tap','Telemetry',
+        'Telemetry::Instrument::Thread','Telemetry::Instrument::Usage',
+        'Telemetry::Period','Telemetry::Sampler','Thread','ThreadPoolScheduler',
+        'UInt','uint16','uint32','uint64','uint8','Uni','utf8','Variable',
+        'Version','VM','Whatever','WhateverCode','WrapHandle'
+    )
+
+    PERL6_OPERATORS = (
+        'X', 'Z', 'after', 'also', 'and', 'andthen', 'before', 'cmp', 'div',
+        'eq', 'eqv', 'extra', 'ff', 'fff', 'ge', 'gt', 'le', 'leg', 'lt', 'm',
+        'mm', 'mod', 'ne', 'or', 'orelse', 'rx', 's', 'tr', 'x', 'xor', 'xx',
+        '++', '--', '**', '!', '+', '-', '~', '?', '|', '||', '+^', '~^', '?^',
+        '^', '*', '/', '%', '%%', '+&', '+<', '+>', '~&', '~<', '~>', '?&',
+        'gcd', 'lcm', '+', '-', '+|', '+^', '~|', '~^', '?|', '?^',
+        '~', '&', '^', 'but', 'does', '<=>', '..', '..^', '^..', '^..^',
+        '!=', '==', '<', '<=', '>', '>=', '~~', '===', '!eqv',
+        '&&', '||', '^^', '//', 'min', 'max', '??', '!!', 'ff', 'fff', 'so',
+        'not', '<==', '==>', '<<==', '==>>','unicmp',
+    )
+
+    # Perl 6 has a *lot* of possible bracketing characters
+    # this list was lifted from STD.pm6 (https://github.com/perl6/std)
+    PERL6_BRACKETS = {
+        '\u0028': '\u0029', '\u003c': '\u003e', '\u005b': '\u005d',
+        '\u007b': '\u007d', '\u00ab': '\u00bb', '\u0f3a': '\u0f3b',
+        '\u0f3c': '\u0f3d', '\u169b': '\u169c', '\u2018': '\u2019',
+        '\u201a': '\u2019', '\u201b': '\u2019', '\u201c': '\u201d',
+        '\u201e': '\u201d', '\u201f': '\u201d', '\u2039': '\u203a',
+        '\u2045': '\u2046', '\u207d': '\u207e', '\u208d': '\u208e',
+        '\u2208': '\u220b', '\u2209': '\u220c', '\u220a': '\u220d',
+        '\u2215': '\u29f5', '\u223c': '\u223d', '\u2243': '\u22cd',
+        '\u2252': '\u2253', '\u2254': '\u2255', '\u2264': '\u2265',
+        '\u2266': '\u2267', '\u2268': '\u2269', '\u226a': '\u226b',
+        '\u226e': '\u226f', '\u2270': '\u2271', '\u2272': '\u2273',
+        '\u2274': '\u2275', '\u2276': '\u2277', '\u2278': '\u2279',
+        '\u227a': '\u227b', '\u227c': '\u227d', '\u227e': '\u227f',
+        '\u2280': '\u2281', '\u2282': '\u2283', '\u2284': '\u2285',
+        '\u2286': '\u2287', '\u2288': '\u2289', '\u228a': '\u228b',
+        '\u228f': '\u2290', '\u2291': '\u2292', '\u2298': '\u29b8',
+        '\u22a2': '\u22a3', '\u22a6': '\u2ade', '\u22a8': '\u2ae4',
+        '\u22a9': '\u2ae3', '\u22ab': '\u2ae5', '\u22b0': '\u22b1',
+        '\u22b2': '\u22b3', '\u22b4': '\u22b5', '\u22b6': '\u22b7',
+        '\u22c9': '\u22ca', '\u22cb': '\u22cc', '\u22d0': '\u22d1',
+        '\u22d6': '\u22d7', '\u22d8': '\u22d9', '\u22da': '\u22db',
+        '\u22dc': '\u22dd', '\u22de': '\u22df', '\u22e0': '\u22e1',
+        '\u22e2': '\u22e3', '\u22e4': '\u22e5', '\u22e6': '\u22e7',
+        '\u22e8': '\u22e9', '\u22ea': '\u22eb', '\u22ec': '\u22ed',
+        '\u22f0': '\u22f1', '\u22f2': '\u22fa', '\u22f3': '\u22fb',
+        '\u22f4': '\u22fc', '\u22f6': '\u22fd', '\u22f7': '\u22fe',
+        '\u2308': '\u2309', '\u230a': '\u230b', '\u2329': '\u232a',
+        '\u23b4': '\u23b5', '\u2768': '\u2769', '\u276a': '\u276b',
+        '\u276c': '\u276d', '\u276e': '\u276f', '\u2770': '\u2771',
+        '\u2772': '\u2773', '\u2774': '\u2775', '\u27c3': '\u27c4',
+        '\u27c5': '\u27c6', '\u27d5': '\u27d6', '\u27dd': '\u27de',
+        '\u27e2': '\u27e3', '\u27e4': '\u27e5', '\u27e6': '\u27e7',
+        '\u27e8': '\u27e9', '\u27ea': '\u27eb', '\u2983': '\u2984',
+        '\u2985': '\u2986', '\u2987': '\u2988', '\u2989': '\u298a',
+        '\u298b': '\u298c', '\u298d': '\u298e', '\u298f': '\u2990',
+        '\u2991': '\u2992', '\u2993': '\u2994', '\u2995': '\u2996',
+        '\u2997': '\u2998', '\u29c0': '\u29c1', '\u29c4': '\u29c5',
+        '\u29cf': '\u29d0', '\u29d1': '\u29d2', '\u29d4': '\u29d5',
+        '\u29d8': '\u29d9', '\u29da': '\u29db', '\u29f8': '\u29f9',
+        '\u29fc': '\u29fd', '\u2a2b': '\u2a2c', '\u2a2d': '\u2a2e',
+        '\u2a34': '\u2a35', '\u2a3c': '\u2a3d', '\u2a64': '\u2a65',
+        '\u2a79': '\u2a7a', '\u2a7d': '\u2a7e', '\u2a7f': '\u2a80',
+        '\u2a81': '\u2a82', '\u2a83': '\u2a84', '\u2a8b': '\u2a8c',
+        '\u2a91': '\u2a92', '\u2a93': '\u2a94', '\u2a95': '\u2a96',
+        '\u2a97': '\u2a98', '\u2a99': '\u2a9a', '\u2a9b': '\u2a9c',
+        '\u2aa1': '\u2aa2', '\u2aa6': '\u2aa7', '\u2aa8': '\u2aa9',
+        '\u2aaa': '\u2aab', '\u2aac': '\u2aad', '\u2aaf': '\u2ab0',
+        '\u2ab3': '\u2ab4', '\u2abb': '\u2abc', '\u2abd': '\u2abe',
+        '\u2abf': '\u2ac0', '\u2ac1': '\u2ac2', '\u2ac3': '\u2ac4',
+        '\u2ac5': '\u2ac6', '\u2acd': '\u2ace', '\u2acf': '\u2ad0',
+        '\u2ad1': '\u2ad2', '\u2ad3': '\u2ad4', '\u2ad5': '\u2ad6',
+        '\u2aec': '\u2aed', '\u2af7': '\u2af8', '\u2af9': '\u2afa',
+        '\u2e02': '\u2e03', '\u2e04': '\u2e05', '\u2e09': '\u2e0a',
+        '\u2e0c': '\u2e0d', '\u2e1c': '\u2e1d', '\u2e20': '\u2e21',
+        '\u3008': '\u3009', '\u300a': '\u300b', '\u300c': '\u300d',
+        '\u300e': '\u300f', '\u3010': '\u3011', '\u3014': '\u3015',
+        '\u3016': '\u3017', '\u3018': '\u3019', '\u301a': '\u301b',
+        '\u301d': '\u301e', '\ufd3e': '\ufd3f', '\ufe17': '\ufe18',
+        '\ufe35': '\ufe36', '\ufe37': '\ufe38', '\ufe39': '\ufe3a',
+        '\ufe3b': '\ufe3c', '\ufe3d': '\ufe3e', '\ufe3f': '\ufe40',
+        '\ufe41': '\ufe42', '\ufe43': '\ufe44', '\ufe47': '\ufe48',
+        '\ufe59': '\ufe5a', '\ufe5b': '\ufe5c', '\ufe5d': '\ufe5e',
+        '\uff08': '\uff09', '\uff1c': '\uff1e', '\uff3b': '\uff3d',
+        '\uff5b': '\uff5d', '\uff5f': '\uff60', '\uff62': '\uff63',
+    }
+
+    def _build_word_match(words, boundary_regex_fragment=None, prefix='', suffix=''):
+        if boundary_regex_fragment is None:
+            return r'\b(' + prefix + r'|'.join(re.escape(x) for x in words) + \
+                suffix + r')\b'
+        else:
+            return r'(? 0:
+                    next_open_pos = text.find(opening_chars, search_pos + n_chars)
+                    next_close_pos = text.find(closing_chars, search_pos + n_chars)
+
+                    if next_close_pos == -1:
+                        next_close_pos = len(text)
+                        nesting_level = 0
+                    elif next_open_pos != -1 and next_open_pos < next_close_pos:
+                        nesting_level += 1
+                        search_pos = next_open_pos
+                    else:  # next_close_pos < next_open_pos
+                        nesting_level -= 1
+                        search_pos = next_close_pos
+
+                end_pos = next_close_pos
+
+            if end_pos < 0:     # if we didn't find a closer, just highlight the
+                                # rest of the text in this class
+                end_pos = len(text)
+
+            if adverbs is not None and re.search(r':to\b', adverbs):
+                heredoc_terminator = text[match.start('delimiter') + n_chars:end_pos]
+                end_heredoc = re.search(r'^\s*' + re.escape(heredoc_terminator) +
+                                        r'\s*$', text[end_pos:], re.MULTILINE)
+
+                if end_heredoc:
+                    end_pos += end_heredoc.end()
+                else:
+                    end_pos = len(text)
+
+            yield match.start(), token_class, text[match.start():end_pos + n_chars]
+            context.pos = end_pos + n_chars
+
+        return callback
+
+    def opening_brace_callback(lexer, match, context):
+        stack = context.stack
+
+        yield match.start(), Text, context.text[match.start():match.end()]
+        context.pos = match.end()
+
+        # if we encounter an opening brace and we're one level
+        # below a token state, it means we need to increment
+        # the nesting level for braces so we know later when
+        # we should return to the token rules.
+        if len(stack) > 2 and stack[-2] == 'token':
+            context.perl6_token_nesting_level += 1
+
+    def closing_brace_callback(lexer, match, context):
+        stack = context.stack
+
+        yield match.start(), Text, context.text[match.start():match.end()]
+        context.pos = match.end()
+
+        # if we encounter a free closing brace and we're one level
+        # below a token state, it means we need to check the nesting
+        # level to see if we need to return to the token state.
+        if len(stack) > 2 and stack[-2] == 'token':
+            context.perl6_token_nesting_level -= 1
+            if context.perl6_token_nesting_level == 0:
+                stack.pop()
+
+    def embedded_perl6_callback(lexer, match, context):
+        context.perl6_token_nesting_level = 1
+        yield match.start(), Text, context.text[match.start():match.end()]
+        context.pos = match.end()
+        context.stack.append('root')
+
+    # If you're modifying these rules, be careful if you need to process '{' or '}'
+    # characters. We have special logic for processing these characters (due to the fact
+    # that you can nest Perl 6 code in regex blocks), so if you need to process one of
+    # them, make sure you also process the corresponding one!
+    tokens = {
+        'common': [
+            (r'#[`|=](?P(?P[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)',
+             brackets_callback(Comment.Multiline)),
+            (r'#[^\n]*$', Comment.Single),
+            (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline),
+            (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline),
+            (r'^=.*?\n\s*?\n', Comment.Multiline),
+            (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)',
+             bygroups(Keyword, Name), 'token-sym-brackets'),
+            (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?',
+             bygroups(Keyword, Name), 'pre-token'),
+            # deal with a special case in the Perl 6 grammar (role q { ... })
+            (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Whitespace, Name, Whitespace)),
+            (_build_word_match(PERL6_KEYWORDS, PERL6_IDENTIFIER_RANGE), Keyword),
+            (_build_word_match(PERL6_BUILTIN_CLASSES, PERL6_IDENTIFIER_RANGE, suffix='(?::[UD])?'),
+             Name.Builtin),
+            (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin),
+            # copied from PerlLexer
+            (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*',
+             Name.Variable),
+            (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global),
+            (r'::\?\w+', Name.Variable.Global),
+            (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*',
+             Name.Variable.Global),
+            (r'\$(?:<.*?>)+', Name.Variable),
+            (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P:[\w\s:]+)?\s*(?P(?P[^0-9a-zA-Z:\s])'
+             r'(?P=first_char)*)', brackets_callback(String)),
+            # copied from PerlLexer
+            (r'0_?[0-7]+(_[0-7]+)*', Number.Oct),
+            (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex),
+            (r'0b[01]+(_[01]+)*', Number.Bin),
+            (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?',
+             Number.Float),
+            (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float),
+            (r'\d+(_\d+)*', Number.Integer),
+            (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex),
+            (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex),
+            (r'm\w+(?=\()', Name),
+            (r'(?:m|ms|rx)\s*(?P:[\w\s:]+)?\s*(?P(?P[^\w:\s])'
+             r'(?P=first_char)*)', brackets_callback(String.Regex)),
+            (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/',
+             String.Regex),
+            (r'<[^\s=].*?\S>', String),
+            (_build_word_match(PERL6_OPERATORS), Operator),
+            (r'\w' + PERL6_IDENTIFIER_RANGE + '*', Name),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+        ],
+        'root': [
+            include('common'),
+            (r'\{', opening_brace_callback),
+            (r'\}', closing_brace_callback),
+            (r'.+?', Text),
+        ],
+        'pre-token': [
+            include('common'),
+            (r'\{', Text, ('#pop', 'token')),
+            (r'.+?', Text),
+        ],
+        'token-sym-brackets': [
+            (r'(?P(?P[' + ''.join(PERL6_BRACKETS) + '])(?P=first_char)*)',
+             brackets_callback(Name), ('#pop', 'pre-token')),
+            default(('#pop', 'pre-token')),
+        ],
+        'token': [
+            (r'\}', Text, '#pop'),
+            (r'(?<=:)(?:my|our|state|constant|temp|let).*?;', using(this)),
+            # make sure that quotes in character classes aren't treated as strings
+            (r'<(?:[-!?+.]\s*)?\[.*?\]>', String.Regex),
+            # make sure that '#' characters in quotes aren't treated as comments
+            (r"(?my|our)\s+)?(?:module|class|role|enum|grammar)', line)
+            if class_decl:
+                if saw_perl_decl or class_decl.group('scope') is not None:
+                    return True
+                rating = 0.05
+                continue
+            break
+
+        if ':=' in text:
+            # Same logic as above for PerlLexer
+            rating /= 2
+
+        return rating
+
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.encoding = options.get('encoding', 'utf-8')
diff --git a/lib/pygments/lexers/phix.py b/lib/pygments/lexers/phix.py
new file mode 100644
index 0000000..f0b0377
--- /dev/null
+++ b/lib/pygments/lexers/phix.py
@@ -0,0 +1,363 @@
+"""
+    pygments.lexers.phix
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Phix.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Whitespace
+
+__all__ = ['PhixLexer']
+
+
+class PhixLexer(RegexLexer):
+    """
+    Pygments Lexer for Phix files (.exw).
+    See http://phix.x10.mx
+    """
+
+    name = 'Phix'
+    url = 'http://phix.x10.mx'
+    aliases = ['phix']
+    filenames = ['*.exw']
+    mimetypes = ['text/x-phix']
+    version_added = '2.14'
+
+    flags = re.MULTILINE    # nb: **NOT** re.DOTALL! (totally spanners comment handling)
+
+    preproc = (
+        'ifdef', 'elsifdef', 'elsedef'
+    )
+    # Note these lists are auto-generated by pwa/p2js.exw, when pwa\src\p2js_keywords.e (etc)
+    #     change, though of course subsequent copy/commit/pull requests are all manual steps.
+    types = (
+        'string', 'nullable_string', 'atom_string', 'atom', 'bool', 'boolean',
+        'cdCanvan', 'cdCanvas', 'complex', 'CURLcode', 'dictionary', 'int',
+        'integer', 'Ihandle', 'Ihandles', 'Ihandln', 'mpfr', 'mpq', 'mpz',
+        'mpz_or_string', 'number', 'rid_string', 'seq', 'sequence', 'timedate',
+        'object'
+    )
+    keywords = (
+        'abstract', 'class', 'continue', 'export', 'extends', 'nullable',
+        'private', 'public', 'static', 'struct', 'trace',
+        'and', 'break', 'by', 'case', 'catch', 'const', 'constant', 'debug',
+        'default', 'do', 'else', 'elsif', 'end', 'enum', 'exit', 'fallthru',
+        'fallthrough', 'for', 'forward', 'function', 'global', 'if', 'in',
+        'include', 'js', 'javascript', 'javascript_semantics', 'let', 'not',
+        'or', 'procedure', 'profile', 'profile_time', 'return', 'safe_mode',
+        'switch', 'then', 'to', 'try', 'type', 'type_check', 'until', 'warning',
+        'while', 'with', 'without', 'xor'
+    )
+    routines = (
+        'abort', 'abs', 'adjust_timedate', 'and_bits', 'and_bitsu', 'apply',
+        'append', 'arccos', 'arcsin', 'arctan', 'assert', 'atan2',
+        'atom_to_float32', 'atom_to_float64', 'bankers_rounding', 'beep',
+        'begins', 'binary_search', 'bits_to_int', 'bk_color', 'bytes_to_int',
+        'call_func', 'call_proc', 'cdCanvasActivate', 'cdCanvasArc',
+        'cdCanvasBegin', 'cdCanvasBox', 'cdCanvasChord', 'cdCanvasCircle',
+        'cdCanvasClear', 'cdCanvasEnd', 'cdCanvasFlush', 'cdCanvasFont',
+        'cdCanvasGetImageRGB', 'cdCanvasGetSize', 'cdCanvasGetTextAlignment',
+        'cdCanvasGetTextSize', 'cdCanvasLine', 'cdCanvasMark',
+        'cdCanvasMarkSize', 'cdCanvasMultiLineVectorText', 'cdCanvasPixel',
+        'cdCanvasRect', 'cdCanvasRoundedBox', 'cdCanvasRoundedRect',
+        'cdCanvasSector', 'cdCanvasSetAttribute', 'cdCanvasSetBackground',
+        'cdCanvasSetFillMode', 'cdCanvasSetForeground',
+        'cdCanvasSetInteriorStyle', 'cdCanvasSetLineStyle',
+        'cdCanvasSetLineWidth', 'cdCanvasSetTextAlignment', 'cdCanvasText',
+        'cdCanvasSetTextOrientation', 'cdCanvasGetTextOrientation',
+        'cdCanvasVectorText', 'cdCanvasVectorTextDirection',
+        'cdCanvasVectorTextSize', 'cdCanvasVertex', 'cdCreateCanvas',
+        'cdDecodeAlpha', 'cdDecodeColor', 'cdDecodeColorAlpha', 'cdEncodeAlpha',
+        'cdEncodeColor', 'cdEncodeColorAlpha', 'cdKillCanvas', 'cdVersion',
+        'cdVersionDate', 'ceil', 'change_timezone', 'choose', 'clear_screen',
+        'columnize', 'command_line', 'compare', 'complex_abs', 'complex_add',
+        'complex_arg', 'complex_conjugate', 'complex_cos', 'complex_cosh',
+        'complex_div', 'complex_exp', 'complex_imag', 'complex_inv',
+        'complex_log', 'complex_mul', 'complex_neg', 'complex_new',
+        'complex_norm', 'complex_power', 'complex_rho', 'complex_real',
+        'complex_round', 'complex_sin', 'complex_sinh', 'complex_sprint',
+        'complex_sqrt', 'complex_sub', 'complex_theta', 'concat', 'cos',
+        'crash', 'custom_sort', 'date', 'day_of_week', 'day_of_year',
+        'days_in_month', 'decode_base64', 'decode_flags', 'deep_copy', 'deld',
+        'deserialize', 'destroy_dict', 'destroy_queue', 'destroy_stack',
+        'dict_name', 'dict_size', 'elapsed', 'elapsed_short', 'encode_base64',
+        'equal', 'even', 'exp', 'extract', 'factorial', 'factors',
+        'file_size_k', 'find', 'find_all', 'find_any', 'find_replace', 'filter',
+        'flatten', 'float32_to_atom', 'float64_to_atom', 'floor',
+        'format_timedate', 'free_console', 'from_polar', 'gcd', 'get_file_base',
+        'get_file_extension', 'get_file_name', 'get_file_name_and_path',
+        'get_file_path', 'get_file_path_and_name', 'get_maxprime', 'get_prime',
+        'get_primes', 'get_primes_le', 'get_proper_dir', 'get_proper_path',
+        'get_rand', 'get_routine_info', 'get_test_abort', 'get_test_logfile',
+        'get_test_pause', 'get_test_verbosity', 'get_tzid', 'getd', 'getdd',
+        'getd_all_keys', 'getd_by_index', 'getd_index', 'getd_partial_key',
+        'glAttachShader', 'glBindBuffer', 'glBindTexture', 'glBufferData',
+        'glCanvasSpecialText', 'glClear', 'glClearColor', 'glColor',
+        'glCompileShader', 'glCreateBuffer', 'glCreateProgram',
+        'glCreateShader', 'glCreateTexture', 'glDeleteProgram',
+        'glDeleteShader', 'glDrawArrays', 'glEnable',
+        'glEnableVertexAttribArray', 'glFloat32Array', 'glInt32Array',
+        'glFlush', 'glGetAttribLocation', 'glGetError', 'glGetProgramInfoLog',
+        'glGetProgramParameter', 'glGetShaderInfoLog', 'glGetShaderParameter',
+        'glGetUniformLocation', 'glLinkProgram', 'glLoadIdentity',
+        'glMatrixMode', 'glOrtho', 'glRotatef', 'glShadeModel',
+        'glShaderSource', 'glSimpleA7texcoords', 'glTexImage2Dc',
+        'glTexParameteri', 'glTranslate', 'glUniform1f', 'glUniform1i',
+        'glUniformMatrix4fv', 'glUseProgram', 'glVertex',
+        'glVertexAttribPointer', 'glViewport', 'head', 'hsv_to_rgb', 'iff',
+        'iif', 'include_file', 'incl0de_file', 'insert', 'instance',
+        'int_to_bits', 'int_to_bytes', 'is_dict', 'is_integer', 's_leap_year',
+        'is_prime', 'is_prime2', 'islower', 'isupper', 'Icallback',
+        'iup_isdouble', 'iup_isprint', 'iup_XkeyBase', 'IupAppend', 'IupAlarm',
+        'IupBackgroundBox', 'IupButton', 'IupCalendar', 'IupCanvas',
+        'IupClipboard', 'IupClose', 'IupCloseOnEscape', 'IupControlsOpen',
+        'IupDatePick', 'IupDestroy', 'IupDialog', 'IupDrawArc', 'IupDrawBegin',
+        'IupDrawEnd', 'IupDrawGetSize', 'IupDrawGetTextSize', 'IupDrawLine',
+        'IupDrawRectangle', 'IupDrawText', 'IupExpander', 'IupFill',
+        'IupFlatLabel', 'IupFlatList', 'IupFlatTree', 'IupFlush', 'IupFrame',
+        'IupGetAttribute', 'IupGetAttributeId', 'IupGetAttributePtr',
+        'IupGetBrother', 'IupGetChild', 'IupGetChildCount', 'IupGetClassName',
+        'IupGetDialog', 'IupGetDialogChild', 'IupGetDouble', 'IupGetFocus',
+        'IupGetGlobal', 'IupGetGlobalInt', 'IupGetGlobalIntInt', 'IupGetInt',
+        'IupGetInt2', 'IupGetIntId', 'IupGetIntInt', 'IupGetParent',
+        'IupGLCanvas', 'IupGLCanvasOpen', 'IupGLMakeCurrent', 'IupGraph',
+        'IupHbox', 'IupHide', 'IupImage', 'IupImageRGBA', 'IupItem',
+        'iupKeyCodeToName', 'IupLabel', 'IupLink', 'IupList', 'IupMap',
+        'IupMenu', 'IupMenuItem', 'IupMessage', 'IupMessageDlg', 'IupMultiBox',
+        'IupMultiLine', 'IupNextField', 'IupNormaliser', 'IupOpen',
+        'IupPlayInput', 'IupPopup', 'IupPreviousField', 'IupProgressBar',
+        'IupRadio', 'IupRecordInput', 'IupRedraw', 'IupRefresh',
+        'IupRefreshChildren', 'IupSeparator', 'IupSetAttribute',
+        'IupSetAttributes', 'IupSetAttributeHandle', 'IupSetAttributeId',
+        'IupSetAttributePtr', 'IupSetCallback', 'IupSetCallbacks',
+        'IupSetDouble', 'IupSetFocus', 'IupSetGlobal', 'IupSetGlobalInt',
+        'IupSetGlobalFunction', 'IupSetHandle', 'IupSetInt',
+        'IupSetStrAttribute', 'IupSetStrGlobal', 'IupShow', 'IupShowXY',
+        'IupSplit', 'IupStoreAttribute', 'IupSubmenu', 'IupTable',
+        'IupTableClearSelected', 'IupTableClick_cb', 'IupTableGetSelected',
+        'IupTableResize_cb', 'IupTableSetData', 'IupTabs', 'IupText',
+        'IupTimer', 'IupToggle', 'IupTreeAddNodes', 'IupTreeView', 'IupUpdate',
+        'IupValuator', 'IupVbox', 'join', 'join_by', 'join_path', 'k_perm',
+        'largest', 'lcm', 'length', 'log', 'log10', 'log2', 'lower',
+        'm4_crossProduct', 'm4_inverse', 'm4_lookAt', 'm4_multiply',
+        'm4_normalize', 'm4_perspective', 'm4_subtractVectors', 'm4_xRotate',
+        'm4_yRotate', 'machine_bits', 'machine_word', 'match', 'match_all',
+        'match_replace', 'max', 'maxsq', 'min', 'minsq', 'mod', 'mpfr_add',
+        'mpfr_ceil', 'mpfr_cmp', 'mpfr_cmp_si', 'mpfr_const_pi', 'mpfr_div',
+        'mpfr_div_si', 'mpfr_div_z', 'mpfr_floor', 'mpfr_free', 'mpfr_get_d',
+        'mpfr_get_default_precision', 'mpfr_get_default_rounding_mode',
+        'mpfr_get_fixed', 'mpfr_get_precision', 'mpfr_get_si', 'mpfr_init',
+        'mpfr_inits', 'mpfr_init_set', 'mpfr_init_set_q', 'mpfr_init_set_z',
+        'mpfr_mul', 'mpfr_mul_si', 'mpfr_pow_si', 'mpfr_set', 'mpfr_set_d',
+        'mpfr_set_default_precision', 'mpfr_set_default_rounding_mode',
+        'mpfr_set_precision', 'mpfr_set_q', 'mpfr_set_si', 'mpfr_set_str',
+        'mpfr_set_z', 'mpfr_si_div', 'mpfr_si_sub', 'mpfr_sqrt', 'mpfr_sub',
+        'mpfr_sub_si', 'mpq_abs', 'mpq_add', 'mpq_add_si', 'mpq_canonicalize',
+        'mpq_cmp', 'mpq_cmp_si', 'mpq_div', 'mpq_div_2exp', 'mpq_free',
+        'mpq_get_den', 'mpq_get_num', 'mpq_get_str', 'mpq_init', 'mpq_init_set',
+        'mpq_init_set_si', 'mpq_init_set_str', 'mpq_init_set_z', 'mpq_inits',
+        'mpq_inv', 'mpq_mul', 'mpq_neg', 'mpq_set', 'mpq_set_si', 'mpq_set_str',
+        'mpq_set_z', 'mpq_sub', 'mpz_abs', 'mpz_add', 'mpz_addmul',
+        'mpz_addmul_ui', 'mpz_addmul_si', 'mpz_add_si', 'mpz_add_ui', 'mpz_and',
+        'mpz_bin_uiui', 'mpz_cdiv_q', 'mpz_cmp', 'mpz_cmp_si', 'mpz_divexact',
+        'mpz_divexact_ui', 'mpz_divisible_p', 'mpz_divisible_ui_p', 'mpz_even',
+        'mpz_fac_ui', 'mpz_factorstring', 'mpz_fdiv_q', 'mpz_fdiv_q_2exp',
+        'mpz_fdiv_q_ui', 'mpz_fdiv_qr', 'mpz_fdiv_r', 'mpz_fdiv_ui',
+        'mpz_fib_ui', 'mpz_fib2_ui', 'mpz_fits_atom', 'mpz_fits_integer',
+        'mpz_free', 'mpz_gcd', 'mpz_gcd_ui', 'mpz_get_atom', 'mpz_get_integer',
+        'mpz_get_short_str', 'mpz_get_str', 'mpz_init', 'mpz_init_set',
+        'mpz_inits', 'mpz_invert', 'mpz_lcm', 'mpz_lcm_ui', 'mpz_max',
+        'mpz_min', 'mpz_mod', 'mpz_mod_ui', 'mpz_mul', 'mpz_mul_2exp',
+        'mpz_mul_d', 'mpz_mul_si', 'mpz_neg', 'mpz_nthroot', 'mpz_odd',
+        'mpz_pollard_rho', 'mpz_pow_ui', 'mpz_powm', 'mpz_powm_ui', 'mpz_prime',
+        'mpz_prime_factors', 'mpz_prime_mr', 'mpz_rand', 'mpz_rand_ui',
+        'mpz_re_compose', 'mpz_remove', 'mpz_scan0', 'mpz_scan1', 'mpz_set',
+        'mpz_set_d', 'mpz_set_si', 'mpz_set_str', 'mpz_set_v', 'mpz_sign',
+        'mpz_sizeinbase', 'mpz_sqrt', 'mpz_sub', 'mpz_sub_si', 'mpz_sub_ui',
+        'mpz_si_sub', 'mpz_tdiv_q_2exp', 'mpz_tdiv_r_2exp', 'mpz_tstbit',
+        'mpz_ui_pow_ui', 'mpz_xor', 'named_dict', 'new_dict', 'new_queue',
+        'new_stack', 'not_bits', 'not_bitsu', 'odd', 'or_all', 'or_allu',
+        'or_bits', 'or_bitsu', 'ord', 'ordinal', 'ordinant',
+        'override_timezone', 'pad', 'pad_head', 'pad_tail', 'parse_date_string',
+        'papply', 'peep', 'peepn', 'peep_dict', 'permute', 'permutes',
+        'platform', 'pop', 'popn', 'pop_dict', 'power', 'pp', 'ppEx', 'ppExf',
+        'ppf', 'ppOpt', 'pq_add', 'pq_destroy', 'pq_empty', 'pq_new', 'pq_peek',
+        'pq_pop', 'pq_pop_data', 'pq_size', 'prepend', 'prime_factors',
+        'printf', 'product', 'proper', 'push', 'pushn', 'putd', 'puts',
+        'queue_empty', 'queue_size', 'rand', 'rand_range', 'reinstate',
+        'remainder', 'remove', 'remove_all', 'repeat', 'repeatch', 'replace',
+        'requires', 'reverse', 'rfind', 'rgb', 'rmatch', 'rmdr', 'rnd', 'round',
+        'routine_id', 'scanf', 'serialize', 'series', 'set_rand',
+        'set_test_abort', 'set_test_logfile', 'set_test_module',
+        'set_test_pause', 'set_test_verbosity', 'set_timedate_formats',
+        'set_timezone', 'setd', 'setd_default', 'shorten', 'sha256',
+        'shift_bits', 'shuffle', 'sign', 'sin', 'smallest', 'sort',
+        'sort_columns', 'speak', 'splice', 'split', 'split_any', 'split_by',
+        'sprint', 'sprintf', 'sq_abs', 'sq_add', 'sq_and', 'sq_and_bits',
+        'sq_arccos', 'sq_arcsin', 'sq_arctan', 'sq_atom', 'sq_ceil', 'sq_cmp',
+        'sq_cos', 'sq_div', 'sq_even', 'sq_eq', 'sq_floor', 'sq_floor_div',
+        'sq_ge', 'sq_gt', 'sq_int', 'sq_le', 'sq_log', 'sq_log10', 'sq_log2',
+        'sq_lt', 'sq_max', 'sq_min', 'sq_mod', 'sq_mul', 'sq_ne', 'sq_not',
+        'sq_not_bits', 'sq_odd', 'sq_or', 'sq_or_bits', 'sq_power', 'sq_rand',
+        'sq_remainder', 'sq_rmdr', 'sq_rnd', 'sq_round', 'sq_seq', 'sq_sign',
+        'sq_sin', 'sq_sqrt', 'sq_str', 'sq_sub', 'sq_tan', 'sq_trunc',
+        'sq_uminus', 'sq_xor', 'sq_xor_bits', 'sqrt', 'square_free',
+        'stack_empty', 'stack_size', 'substitute', 'substitute_all', 'sum',
+        'tail', 'tan', 'test_equal', 'test_fail', 'test_false',
+        'test_not_equal', 'test_pass', 'test_summary', 'test_true',
+        'text_color', 'throw', 'time', 'timedate_diff', 'timedelta',
+        'to_integer', 'to_number', 'to_rgb', 'to_string', 'traverse_dict',
+        'traverse_dict_partial_key', 'trim', 'trim_head', 'trim_tail', 'trunc',
+        'tagset', 'tagstart', 'typeof', 'unique', 'unix_dict', 'upper',
+        'utf8_to_utf32', 'utf32_to_utf8', 'version', 'vlookup', 'vslice',
+        'wglGetProcAddress', 'wildcard_file', 'wildcard_match', 'with_rho',
+        'with_theta', 'xml_new_doc', 'xml_new_element', 'xml_set_attribute',
+        'xml_sprint', 'xor_bits', 'xor_bitsu',
+        'accept', 'allocate', 'allocate_string', 'allow_break', 'ARM',
+        'atom_to_float80', 'c_func', 'c_proc', 'call_back', 'chdir',
+        'check_break', 'clearDib', 'close', 'closesocket', 'console',
+        'copy_file', 'create', 'create_directory', 'create_thread',
+        'curl_easy_cleanup', 'curl_easy_get_file', 'curl_easy_init',
+        'curl_easy_perform', 'curl_easy_perform_ex', 'curl_easy_setopt',
+        'curl_easy_strerror', 'curl_global_cleanup', 'curl_global_init',
+        'curl_slist_append', 'curl_slist_free_all', 'current_dir', 'cursor',
+        'define_c_func', 'define_c_proc', 'delete', 'delete_cs', 'delete_file',
+        'dir', 'DLL', 'drawDib', 'drawShadedPolygonToDib', 'ELF32', 'ELF64',
+        'enter_cs', 'eval', 'exit_thread', 'free', 'file_exists', 'final',
+        'float80_to_atom', 'format', 'get_bytes', 'get_file_date',
+        'get_file_size', 'get_file_type', 'get_interpreter', 'get_key',
+        'get_socket_error', 'get_text', 'get_thread_exitcode', 'get_thread_id',
+        'getc', 'getenv', 'gets', 'getsockaddr', 'glBegin', 'glCallList',
+        'glFrustum', 'glGenLists', 'glGetString', 'glLight', 'glMaterial',
+        'glNewList', 'glNormal', 'glPopMatrix', 'glPushMatrix', 'glRotate',
+        'glEnd', 'glEndList', 'glTexImage2D', 'goto', 'GUI', 'icons', 'ilASM',
+        'include_files', 'include_paths', 'init_cs', 'ip_to_string',
+        'IupConfig', 'IupConfigDialogClosed', 'IupConfigDialogShow',
+        'IupConfigGetVariableInt', 'IupConfigLoad', 'IupConfigSave',
+        'IupConfigSetVariableInt', 'IupExitLoop', 'IupFileDlg', 'IupFileList',
+        'IupGLSwapBuffers', 'IupHelp', 'IupLoopStep', 'IupMainLoop',
+        'IupNormalizer', 'IupPlot', 'IupPlotAdd', 'IupPlotBegin', 'IupPlotEnd',
+        'IupPlotInsert', 'IupSaveImage', 'IupTreeGetUserId', 'IupUser',
+        'IupVersion', 'IupVersionDate', 'IupVersionNumber', 'IupVersionShow',
+        'killDib', 'leave_cs', 'listen', 'manifest', 'mem_copy', 'mem_set',
+        'mpfr_gamma', 'mpfr_printf', 'mpfr_sprintf', 'mpz_export', 'mpz_import',
+        'namespace', 'new', 'newDib', 'open', 'open_dll', 'PE32', 'PE64',
+        'peek', 'peek_string', 'peek1s', 'peek1u', 'peek2s', 'peek2u', 'peek4s',
+        'peek4u', 'peek8s', 'peek8u', 'peekNS', 'peekns', 'peeknu', 'poke',
+        'poke2', 'poke4', 'poke8', 'pokeN', 'poke_string', 'poke_wstring',
+        'position', 'progress', 'prompt_number', 'prompt_string', 'read_file',
+        'read_lines', 'recv', 'resume_thread', 'seek', 'select', 'send',
+        'setHandler', 'shutdown', 'sleep', 'SO', 'sockaddr_in', 'socket',
+        'split_path', 'suspend_thread', 'system', 'system_exec', 'system_open',
+        'system_wait', 'task_clock_start', 'task_clock_stop', 'task_create',
+        'task_delay', 'task_list', 'task_schedule', 'task_self', 'task_status',
+        'task_suspend', 'task_yield', 'thread_safe_string', 'try_cs',
+        'utf8_to_utf16', 'utf16_to_utf8', 'utf16_to_utf32', 'utf32_to_utf16',
+        'video_config', 'WSACleanup', 'wait_thread', 'walk_dir', 'where',
+        'write_lines', 'wait_key'
+    )
+    constants = (
+        'ANY_QUEUE', 'ASCENDING', 'BLACK', 'BLOCK_CURSOR', 'BLUE',
+        'BRIGHT_CYAN', 'BRIGHT_BLUE', 'BRIGHT_GREEN', 'BRIGHT_MAGENTA',
+        'BRIGHT_RED', 'BRIGHT_WHITE', 'BROWN', 'C_DWORD', 'C_INT', 'C_POINTER',
+        'C_USHORT', 'C_WORD', 'CD_AMBER', 'CD_BLACK', 'CD_BLUE', 'CD_BOLD',
+        'CD_BOLD_ITALIC', 'CD_BOX', 'CD_CENTER', 'CD_CIRCLE', 'CD_CLOSED_LINES',
+        'CD_CONTINUOUS', 'CD_CUSTOM', 'CD_CYAN', 'CD_DARK_BLUE', 'CD_DARK_CYAN',
+        'CD_DARK_GRAY', 'CD_DARK_GREY', 'CD_DARK_GREEN', 'CD_DARK_MAGENTA',
+        'CD_DARK_RED', 'CD_DARK_YELLOW', 'CD_DASH_DOT', 'CD_DASH_DOT_DOT',
+        'CD_DASHED', 'CD_DBUFFER', 'CD_DEG2RAD', 'CD_DIAMOND', 'CD_DOTTED',
+        'CD_EAST', 'CD_EVENODD', 'CD_FILL', 'CD_GL', 'CD_GRAY', 'CD_GREY',
+        'CD_GREEN', 'CD_HATCH', 'CD_HOLLOW', 'CD_HOLLOW_BOX',
+        'CD_HOLLOW_CIRCLE', 'CD_HOLLOW_DIAMOND', 'CD_INDIGO', 'CD_ITALIC',
+        'CD_IUP', 'CD_IUPDBUFFER', 'CD_LIGHT_BLUE', 'CD_LIGHT_GRAY',
+        'CD_LIGHT_GREY', 'CD_LIGHT_GREEN', 'CD_LIGHT_PARCHMENT', 'CD_MAGENTA',
+        'CD_NAVY', 'CD_NORTH', 'CD_NORTH_EAST', 'CD_NORTH_WEST', 'CD_OLIVE',
+        'CD_OPEN_LINES', 'CD_ORANGE', 'CD_PARCHMENT', 'CD_PATTERN',
+        'CD_PRINTER', 'CD_PURPLE', 'CD_PLAIN', 'CD_PLUS', 'CD_QUERY',
+        'CD_RAD2DEG', 'CD_RED', 'CD_SILVER', 'CD_SOLID', 'CD_SOUTH_EAST',
+        'CD_SOUTH_WEST', 'CD_STAR', 'CD_STIPPLE', 'CD_STRIKEOUT',
+        'CD_UNDERLINE', 'CD_WEST', 'CD_WHITE', 'CD_WINDING', 'CD_VIOLET',
+        'CD_X', 'CD_YELLOW', 'CURLE_OK', 'CURLOPT_MAIL_FROM',
+        'CURLOPT_MAIL_RCPT', 'CURLOPT_PASSWORD', 'CURLOPT_READDATA',
+        'CURLOPT_READFUNCTION', 'CURLOPT_SSL_VERIFYPEER',
+        'CURLOPT_SSL_VERIFYHOST', 'CURLOPT_UPLOAD', 'CURLOPT_URL',
+        'CURLOPT_USE_SSL', 'CURLOPT_USERNAME', 'CURLOPT_VERBOSE',
+        'CURLOPT_WRITEFUNCTION', 'CURLUSESSL_ALL', 'CYAN', 'D_NAME',
+        'D_ATTRIBUTES', 'D_SIZE', 'D_YEAR', 'D_MONTH', 'D_DAY', 'D_HOUR',
+        'D_MINUTE', 'D_SECOND', 'D_CREATION', 'D_LASTACCESS', 'D_MODIFICATION',
+        'DT_YEAR', 'DT_MONTH', 'DT_DAY', 'DT_HOUR', 'DT_MINUTE', 'DT_SECOND',
+        'DT_DOW', 'DT_MSEC', 'DT_DOY', 'DT_GMT', 'EULER', 'E_CODE', 'E_ADDR',
+        'E_LINE', 'E_RTN', 'E_NAME', 'E_FILE', 'E_PATH', 'E_USER', 'false',
+        'False', 'FALSE', 'FIFO_QUEUE', 'FILETYPE_DIRECTORY', 'FILETYPE_FILE',
+        'GET_EOF', 'GET_FAIL', 'GET_IGNORE', 'GET_SUCCESS',
+        'GL_AMBIENT_AND_DIFFUSE', 'GL_ARRAY_BUFFER', 'GL_CLAMP',
+        'GL_CLAMP_TO_BORDER', 'GL_CLAMP_TO_EDGE', 'GL_COLOR_BUFFER_BIT',
+        'GL_COMPILE', 'GL_COMPILE_STATUS', 'GL_CULL_FACE',
+        'GL_DEPTH_BUFFER_BIT', 'GL_DEPTH_TEST', 'GL_EXTENSIONS', 'GL_FLAT',
+        'GL_FLOAT', 'GL_FRAGMENT_SHADER', 'GL_FRONT', 'GL_LIGHT0',
+        'GL_LIGHTING', 'GL_LINEAR', 'GL_LINK_STATUS', 'GL_MODELVIEW',
+        'GL_NEAREST', 'GL_NO_ERROR', 'GL_NORMALIZE', 'GL_POSITION',
+        'GL_PROJECTION', 'GL_QUAD_STRIP', 'GL_QUADS', 'GL_RENDERER',
+        'GL_REPEAT', 'GL_RGB', 'GL_RGBA', 'GL_SMOOTH', 'GL_STATIC_DRAW',
+        'GL_TEXTURE_2D', 'GL_TEXTURE_MAG_FILTER', 'GL_TEXTURE_MIN_FILTER',
+        'GL_TEXTURE_WRAP_S', 'GL_TEXTURE_WRAP_T', 'GL_TRIANGLES',
+        'GL_UNSIGNED_BYTE', 'GL_VENDOR', 'GL_VERSION', 'GL_VERTEX_SHADER',
+        'GRAY', 'GREEN', 'GT_LF_STRIPPED', 'GT_WHOLE_FILE', 'INVLN10',
+        'IUP_CLOSE', 'IUP_CONTINUE', 'IUP_DEFAULT', 'IUP_BLACK', 'IUP_BLUE',
+        'IUP_BUTTON1', 'IUP_BUTTON3', 'IUP_CENTER', 'IUP_CYAN', 'IUP_DARK_BLUE',
+        'IUP_DARK_CYAN', 'IUP_DARK_GRAY', 'IUP_DARK_GREY', 'IUP_DARK_GREEN',
+        'IUP_DARK_MAGENTA', 'IUP_DARK_RED', 'IUP_GRAY', 'IUP_GREY', 'IUP_GREEN',
+        'IUP_IGNORE', 'IUP_INDIGO', 'IUP_MAGENTA', 'IUP_MASK_INT',
+        'IUP_MASK_UINT', 'IUP_MOUSEPOS', 'IUP_NAVY', 'IUP_OLIVE', 'IUP_RECTEXT',
+        'IUP_RED', 'IUP_LIGHT_BLUE', 'IUP_LIGHT_GRAY', 'IUP_LIGHT_GREY',
+        'IUP_LIGHT_GREEN', 'IUP_ORANGE', 'IUP_PARCHMENT', 'IUP_PURPLE',
+        'IUP_SILVER', 'IUP_TEAL', 'IUP_VIOLET', 'IUP_WHITE', 'IUP_YELLOW',
+        'K_BS', 'K_cA', 'K_cC', 'K_cD', 'K_cF5', 'K_cK', 'K_cM', 'K_cN', 'K_cO',
+        'K_cP', 'K_cR', 'K_cS', 'K_cT', 'K_cW', 'K_CR', 'K_DEL', 'K_DOWN',
+        'K_END', 'K_ESC', 'K_F1', 'K_F2', 'K_F3', 'K_F4', 'K_F5', 'K_F6',
+        'K_F7', 'K_F8', 'K_F9', 'K_F10', 'K_F11', 'K_F12', 'K_HOME', 'K_INS',
+        'K_LEFT', 'K_MIDDLE', 'K_PGDN', 'K_PGUP', 'K_RIGHT', 'K_SP', 'K_TAB',
+        'K_UP', 'K_h', 'K_i', 'K_j', 'K_p', 'K_r', 'K_s', 'JS', 'LIFO_QUEUE',
+        'LINUX', 'MAX_HEAP', 'MAGENTA', 'MIN_HEAP', 'Nan', 'NO_CURSOR', 'null',
+        'NULL', 'PI', 'pp_Ascii', 'pp_Brkt', 'pp_Date', 'pp_File', 'pp_FltFmt',
+        'pp_Indent', 'pp_IntCh', 'pp_IntFmt', 'pp_Maxlen', 'pp_Nest',
+        'pp_Pause', 'pp_Q22', 'pp_StrFmt', 'RED', 'SEEK_OK', 'SLASH',
+        'TEST_ABORT', 'TEST_CRASH', 'TEST_PAUSE', 'TEST_PAUSE_FAIL',
+        'TEST_QUIET', 'TEST_SHOW_ALL', 'TEST_SHOW_FAILED', 'TEST_SUMMARY',
+        'true', 'True', 'TRUE', 'VC_SCRNLINES', 'WHITE', 'WINDOWS', 'YELLOW'
+    )
+
+    tokens = {
+        'root': [
+            (r"\s+", Whitespace),
+            (r'/\*|--/\*|#\[', Comment.Multiline, 'comment'),
+            (r'(?://|--|#!).*$', Comment.Single),
+#Alt:
+#           (r'//.*$|--.*$|#!.*$', Comment.Single),
+            (r'"([^"\\]|\\.)*"', String.Other),
+            (r'\'[^\']*\'', String.Other),
+            (r'`[^`]*`', String.Other),
+
+            (words(types, prefix=r'\b', suffix=r'\b'), Name.Function),
+            (words(routines, prefix=r'\b', suffix=r'\b'), Name.Function),
+            (words(preproc, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+            (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+            (words(constants, prefix=r'\b', suffix=r'\b'), Name.Constant),
+            # Aside: Phix only supports/uses the ascii/non-unicode tilde
+            (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|\.(){},?:\[\]$\\;#]', Operator),
+            (r'[\w-]+', Text)
+        ],
+        'comment': [
+            (r'[^*/#]+', Comment.Multiline),
+            (r'/\*|#\[', Comment.Multiline, '#push'),
+            (r'\*/|#\]', Comment.Multiline, '#pop'),
+            (r'[*/#]', Comment.Multiline)
+        ]
+    }
diff --git a/lib/pygments/lexers/php.py b/lib/pygments/lexers/php.py
new file mode 100644
index 0000000..82d4aeb
--- /dev/null
+++ b/lib/pygments/lexers/php.py
@@ -0,0 +1,334 @@
+"""
+    pygments.lexers.php
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for PHP and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, bygroups, default, \
+    using, this, words, do_insertions, line_re
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Other, Generic
+from pygments.util import get_bool_opt, get_list_opt, shebang_matches
+
+__all__ = ['ZephirLexer', 'PsyshConsoleLexer', 'PhpLexer']
+
+
+class ZephirLexer(RegexLexer):
+    """
+    For Zephir language source code.
+
+    Zephir is a compiled high level language aimed
+    to the creation of C-extensions for PHP.
+    """
+
+    name = 'Zephir'
+    url = 'http://zephir-lang.com/'
+    aliases = ['zephir']
+    filenames = ['*.zep']
+    version_added = '2.0'
+
+    zephir_keywords = ['fetch', 'echo', 'isset', 'empty']
+    zephir_type = ['bit', 'bits', 'string']
+
+    flags = re.DOTALL | re.MULTILINE
+
+    tokens = {
+        'commentsandwhitespace': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
+        ],
+        'slashstartsregex': [
+            include('commentsandwhitespace'),
+            (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+             r'([gim]+\b|\B)', String.Regex, '#pop'),
+            (r'/', Operator, '#pop'),
+            default('#pop')
+        ],
+        'badregex': [
+            (r'\n', Text, '#pop')
+        ],
+        'root': [
+            (r'^(?=\s|/)', Text, 'slashstartsregex'),
+            include('commentsandwhitespace'),
+            (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+             r'(<<|>>>?|==?|!=?|->|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+            (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+            (r'[})\].]', Punctuation),
+            (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|'
+             r'require|inline|throw|try|catch|finally|new|delete|typeof|instanceof|void|'
+             r'namespace|use|extends|this|fetch|isset|unset|echo|fetch|likely|unlikely|'
+             r'empty)\b', Keyword, 'slashstartsregex'),
+            (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+            (r'(abstract|boolean|bool|char|class|const|double|enum|export|extends|final|'
+             r'native|goto|implements|import|int|string|interface|long|ulong|char|uchar|'
+             r'float|unsigned|private|protected|public|short|static|self|throws|reverse|'
+             r'transient|volatile|readonly)\b', Keyword.Reserved),
+            (r'(true|false|null|undefined)\b', Keyword.Constant),
+            (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|'
+             r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|'
+             r'window)\b', Name.Builtin),
+            (r'[$a-zA-Z_][\w\\]*', Name.Other),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[0-9]+', Number.Integer),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+        ]
+    }
+
+
+class PsyshConsoleLexer(Lexer):
+    """
+    For PsySH console output, such as:
+
+    .. sourcecode:: psysh
+
+        >>> $greeting = function($name): string {
+        ...     return "Hello, {$name}";
+        ... };
+        => Closure($name): string {#2371 …3}
+        >>> $greeting('World')
+        => "Hello, World"
+    """
+    name = 'PsySH console session for PHP'
+    url = 'https://psysh.org/'
+    aliases = ['psysh']
+    version_added = '2.7'
+
+    def __init__(self, **options):
+        options['startinline'] = True
+        Lexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        phplexer = PhpLexer(**self.options)
+        curcode = ''
+        insertions = []
+        for match in line_re.finditer(text):
+            line = match.group()
+            if line.startswith('>>> ') or line.startswith('... '):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:4])]))
+                curcode += line[4:]
+            elif line.rstrip() == '...':
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, '...')]))
+                curcode += line[3:]
+            else:
+                if curcode:
+                    yield from do_insertions(
+                        insertions, phplexer.get_tokens_unprocessed(curcode))
+                    curcode = ''
+                    insertions = []
+                yield match.start(), Generic.Output, line
+        if curcode:
+            yield from do_insertions(insertions,
+                                     phplexer.get_tokens_unprocessed(curcode))
+
+
+class PhpLexer(RegexLexer):
+    """
+    For PHP source code.
+    For PHP embedded in HTML, use the `HtmlPhpLexer`.
+
+    Additional options accepted:
+
+    `startinline`
+        If given and ``True`` the lexer starts highlighting with
+        php code (i.e.: no starting ``>> from pygments.lexers._php_builtins import MODULES
+            >>> MODULES.keys()
+            ['PHP Options/Info', 'Zip', 'dba', ...]
+
+        In fact the names of those modules match the module names from
+        the php documentation.
+    """
+
+    name = 'PHP'
+    url = 'https://www.php.net/'
+    aliases = ['php', 'php3', 'php4', 'php5']
+    filenames = ['*.php', '*.php[345]', '*.inc']
+    mimetypes = ['text/x-php']
+    version_added = ''
+
+    # Note that a backslash is included, PHP uses a backslash as a namespace
+    # separator.
+    _ident_inner = r'(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*'
+    # But not inside strings.
+    _ident_nons = r'(?:[_a-z]|[^\x00-\x7f])(?:\w|[^\x00-\x7f])*'
+
+    flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'<\?(php)?', Comment.Preproc, 'php'),
+            (r'[^<]+', Other),
+            (r'<', Other)
+        ],
+        'php': [
+            (r'\?>', Comment.Preproc, '#pop'),
+            (r'(<<<)([\'"]?)(' + _ident_nons + r')(\2\n.*?\n\s*)(\3)(;?)(\n)',
+             bygroups(String, String, String.Delimiter, String, String.Delimiter,
+                      Punctuation, Text)),
+            (r'\s+', Text),
+            (r'#\[', Punctuation, 'attribute'),
+            (r'#.*?\n', Comment.Single),
+            (r'//.*?\n', Comment.Single),
+            # put the empty comment here, it is otherwise seen as
+            # the start of a docstring
+            (r'/\*\*/', Comment.Multiline),
+            (r'/\*\*.*?\*/', String.Doc),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'(->|::)(\s*)(' + _ident_nons + ')',
+             bygroups(Operator, Text, Name.Attribute)),
+            (r'[~!%^&*+=|:.<>/@-]+', Operator),
+            (r'\?', Operator),  # don't add to the charclass above!
+            (r'[\[\]{}();,]+', Punctuation),
+            (r'(new)(\s+)(class)\b', bygroups(Keyword, Text, Keyword)),
+            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+            (r'(function)(\s*)(?=\()', bygroups(Keyword, Text)),
+            (r'(function)(\s+)(&?)(\s*)',
+             bygroups(Keyword, Text, Operator, Text), 'functionname'),
+            (r'(const)(\s+)(' + _ident_inner + ')',
+             bygroups(Keyword, Text, Name.Constant)),
+            (r'(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|'
+             r'eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|'
+             r'FALSE|print|for|require|continue|foreach|require_once|'
+             r'declare|return|default|static|do|switch|die|stdClass|'
+             r'echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|'
+             r'virtual|endfor|include_once|while|endforeach|global|'
+             r'endif|list|endswitch|new|endwhile|not|'
+             r'array|E_ALL|NULL|final|php_user_filter|interface|'
+             r'implements|public|private|protected|abstract|clone|try|'
+             r'catch|throw|this|use|namespace|trait|yield|'
+             r'finally|match)\b', Keyword),
+            (r'(true|false|null)\b', Keyword.Constant),
+            include('magicconstants'),
+            (r'\$\{', Name.Variable, 'variablevariable'),
+            (r'\$+' + _ident_inner, Name.Variable),
+            (_ident_inner, Name.Other),
+            (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float),
+            (r'\d+e[+-]?[0-9]+', Number.Float),
+            (r'0[0-7]+', Number.Oct),
+            (r'0x[a-f0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'0b[01]+', Number.Bin),
+            (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single),
+            (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
+            (r'"', String.Double, 'string'),
+        ],
+        'variablevariable': [
+            (r'\}', Name.Variable, '#pop'),
+            include('php')
+        ],
+        'magicfuncs': [
+            # source: http://php.net/manual/en/language.oop5.magic.php
+            (words((
+                '__construct', '__destruct', '__call', '__callStatic', '__get', '__set',
+                '__isset', '__unset', '__sleep', '__wakeup', '__toString', '__invoke',
+                '__set_state', '__clone', '__debugInfo',), suffix=r'\b'),
+             Name.Function.Magic),
+        ],
+        'magicconstants': [
+            # source: http://php.net/manual/en/language.constants.predefined.php
+            (words((
+                '__LINE__', '__FILE__', '__DIR__', '__FUNCTION__', '__CLASS__',
+                '__TRAIT__', '__METHOD__', '__NAMESPACE__',),
+                suffix=r'\b'),
+             Name.Constant),
+        ],
+        'classname': [
+            (_ident_inner, Name.Class, '#pop')
+        ],
+        'functionname': [
+            include('magicfuncs'),
+            (_ident_inner, Name.Function, '#pop'),
+            default('#pop')
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'[^{$"\\]+', String.Double),
+            (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape),
+            (r'\$' + _ident_nons + r'(\[\S+?\]|->' + _ident_nons + ')?',
+             String.Interpol),
+            (r'(\{\$\{)(.*?)(\}\})',
+             bygroups(String.Interpol, using(this, _startinline=True),
+                      String.Interpol)),
+            (r'(\{)(\$.*?)(\})',
+             bygroups(String.Interpol, using(this, _startinline=True),
+                      String.Interpol)),
+            (r'(\$\{)(\S+)(\})',
+             bygroups(String.Interpol, Name.Variable, String.Interpol)),
+            (r'[${\\]', String.Double)
+        ],
+        'attribute': [
+            (r'\]', Punctuation, '#pop'),
+            (r'\(', Punctuation, 'attributeparams'),
+            (_ident_inner, Name.Decorator),
+            include('php')
+        ],
+        'attributeparams': [
+            (r'\)', Punctuation, '#pop'),
+            include('php')
+        ],
+    }
+
+    def __init__(self, **options):
+        self.funcnamehighlighting = get_bool_opt(
+            options, 'funcnamehighlighting', True)
+        self.disabledmodules = get_list_opt(
+            options, 'disabledmodules', ['unknown'])
+        self.startinline = get_bool_opt(options, 'startinline', False)
+
+        # private option argument for the lexer itself
+        if '_startinline' in options:
+            self.startinline = options.pop('_startinline')
+
+        # collect activated functions in a set
+        self._functions = set()
+        if self.funcnamehighlighting:
+            from pygments.lexers._php_builtins import MODULES
+            for key, value in MODULES.items():
+                if key not in self.disabledmodules:
+                    self._functions.update(value)
+        RegexLexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        stack = ['root']
+        if self.startinline:
+            stack.append('php')
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text, stack):
+            if token is Name.Other:
+                if value in self._functions:
+                    yield index, Name.Builtin, value
+                    continue
+            yield index, token, value
+
+    def analyse_text(text):
+        if shebang_matches(text, r'php'):
+            return True
+        rv = 0.0
+        if re.search(r'<\?(?!xml)', text):
+            rv += 0.3
+        return rv
diff --git a/lib/pygments/lexers/pointless.py b/lib/pygments/lexers/pointless.py
new file mode 100644
index 0000000..adedb75
--- /dev/null
+++ b/lib/pygments/lexers/pointless.py
@@ -0,0 +1,70 @@
+"""
+    pygments.lexers.pointless
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Pointless.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+
+__all__ = ['PointlessLexer']
+
+
+class PointlessLexer(RegexLexer):
+    """
+    For Pointless source code.
+    """
+
+    name = 'Pointless'
+    url = 'https://ptls.dev'
+    aliases = ['pointless']
+    filenames = ['*.ptls']
+    version_added = '2.7'
+
+    ops = words([
+        "+", "-", "*", "/", "**", "%", "+=", "-=", "*=",
+        "/=", "**=", "%=", "|>", "=", "==", "!=", "<", ">",
+        "<=", ">=", "=>", "$", "++",
+    ])
+
+    keywords = words([
+        "if", "then", "else", "where", "with", "cond",
+        "case", "and", "or", "not", "in", "as", "for",
+        "requires", "throw", "try", "catch", "when",
+        "yield", "upval",
+    ], suffix=r'\b')
+
+    tokens = {
+        'root': [
+            (r'[ \n\r]+', Text),
+            (r'--.*$', Comment.Single),
+            (r'"""', String, 'multiString'),
+            (r'"', String, 'string'),
+            (r'[\[\](){}:;,.]', Punctuation),
+            (ops, Operator),
+            (keywords, Keyword),
+            (r'\d+|\d*\.\d+', Number),
+            (r'(true|false)\b', Name.Builtin),
+            (r'[A-Z][a-zA-Z0-9]*\b', String.Symbol),
+            (r'output\b', Name.Variable.Magic),
+            (r'(export|import)\b', Keyword.Namespace),
+            (r'[a-z][a-zA-Z0-9]*\b', Name.Variable)
+        ],
+        'multiString': [
+            (r'\\.', String.Escape),
+            (r'"""', String, '#pop'),
+            (r'"', String),
+            (r'[^\\"]+', String),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'"', String, '#pop'),
+            (r'\n', Error),
+            (r'[^\\"]+', String),
+        ],
+    }
diff --git a/lib/pygments/lexers/pony.py b/lib/pygments/lexers/pony.py
new file mode 100644
index 0000000..055423a
--- /dev/null
+++ b/lib/pygments/lexers/pony.py
@@ -0,0 +1,93 @@
+"""
+    pygments.lexers.pony
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Pony and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['PonyLexer']
+
+
+class PonyLexer(RegexLexer):
+    """
+    For Pony source code.
+    """
+
+    name = 'Pony'
+    aliases = ['pony']
+    filenames = ['*.pony']
+    url = 'https://www.ponylang.io'
+    version_added = '2.4'
+
+    _caps = r'(iso|trn|ref|val|box|tag)'
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'//.*\n', Comment.Single),
+            (r'/\*', Comment.Multiline, 'nested_comment'),
+            (r'"""(?:.|\n)*?"""', String.Doc),
+            (r'"', String, 'string'),
+            (r'\'.*\'', String.Char),
+            (r'=>|[]{}:().~;,|&!^?[]', Punctuation),
+            (words((
+                'addressof', 'and', 'as', 'consume', 'digestof', 'is', 'isnt',
+                'not', 'or'),
+                suffix=r'\b'),
+             Operator.Word),
+            (r'!=|==|<<|>>|[-+/*%=<>]', Operator),
+            (words((
+                'box', 'break', 'compile_error', 'compile_intrinsic',
+                'continue', 'do', 'else', 'elseif', 'embed', 'end', 'error',
+                'for', 'if', 'ifdef', 'in', 'iso', 'lambda', 'let', 'match',
+                'object', 'recover', 'ref', 'repeat', 'return', 'tag', 'then',
+                'this', 'trn', 'try', 'until', 'use', 'var', 'val', 'where',
+                'while', 'with', '#any', '#read', '#send', '#share'),
+                suffix=r'\b'),
+             Keyword),
+            (r'(actor|class|struct|primitive|interface|trait|type)((?:\s)+)',
+             bygroups(Keyword, Text), 'typename'),
+            (r'(new|fun|be)((?:\s)+)', bygroups(Keyword, Text), 'methodname'),
+            (words((
+                'I8', 'U8', 'I16', 'U16', 'I32', 'U32', 'I64', 'U64', 'I128',
+                'U128', 'ILong', 'ULong', 'ISize', 'USize', 'F32', 'F64',
+                'Bool', 'Pointer', 'None', 'Any', 'Array', 'String',
+                'Iterator'),
+                suffix=r'\b'),
+             Name.Builtin.Type),
+            (r'_?[A-Z]\w*', Name.Type),
+            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'(true|false)\b', Name.Builtin),
+            (r'_\d*', Name),
+            (r'_?[a-z][\w\']*', Name)
+        ],
+        'typename': [
+            (_caps + r'?((?:\s)*)(_?[A-Z]\w*)',
+             bygroups(Keyword, Text, Name.Class), '#pop')
+        ],
+        'methodname': [
+            (_caps + r'?((?:\s)*)(_?[a-z]\w*)',
+             bygroups(Keyword, Text, Name.Function), '#pop')
+        ],
+        'nested_comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\"', String),
+            (r'[^\\"]+', String)
+        ]
+    }
diff --git a/lib/pygments/lexers/praat.py b/lib/pygments/lexers/praat.py
new file mode 100644
index 0000000..054f5b6
--- /dev/null
+++ b/lib/pygments/lexers/praat.py
@@ -0,0 +1,303 @@
+"""
+    pygments.lexers.praat
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Praat
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, bygroups, include
+from pygments.token import Name, Text, Comment, Keyword, String, Punctuation, \
+    Number, Operator, Whitespace
+
+__all__ = ['PraatLexer']
+
+
+class PraatLexer(RegexLexer):
+    """
+    For Praat scripts.
+    """
+
+    name = 'Praat'
+    url = 'http://www.praat.org'
+    aliases = ['praat']
+    filenames = ['*.praat', '*.proc', '*.psc']
+    version_added = '2.1'
+
+    keywords = (
+        'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to',
+        'endfor', 'endproc', 'while', 'endwhile', 'repeat', 'until', 'select', 'plus',
+        'minus', 'demo', 'assert', 'stopwatch', 'nocheck', 'nowarn', 'noprogress',
+        'editor', 'endeditor', 'clearinfo',
+    )
+
+    functions_string = (
+        'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile',
+        'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine',
+        'extractWord', 'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace',
+        'replace_regex', 'right', 'selected', 'string', 'unicodeToBackslashTrigraphs',
+    )
+
+    functions_numeric = (
+        'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos',
+        'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz',
+        'beginPause', 'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2',
+        'binomialP', 'binomialQ', 'boolean', 'ceiling', 'chiSquareP', 'chiSquareQ',
+        'choice', 'comment', 'cos', 'cosh', 'createDirectory', 'deleteFile',
+        'demoClicked', 'demoClickedIn', 'demoCommandKeyPressed',
+        'demoExtraControlKeyPressed', 'demoInput', 'demoKeyPressed',
+        'demoOptionKeyPressed', 'demoShiftKeyPressed', 'demoShow', 'demoWaitForInput',
+        'demoWindowTitle', 'demoX', 'demoY', 'differenceLimensToPhon', 'do', 'editor',
+        'endPause', 'endSendPraat', 'endsWith', 'erb', 'erbToHertz', 'erf', 'erfc',
+        'exitScript', 'exp', 'extractNumber', 'fileReadable', 'fisherP', 'fisherQ',
+        'floor', 'gaussP', 'gaussQ', 'hertzToBark', 'hertzToErb', 'hertzToMel',
+        'hertzToSemitones', 'imax', 'imin', 'incompleteBeta', 'incompleteGammaP', 'index',
+        'index_regex', 'integer', 'invBinomialP', 'invBinomialQ', 'invChiSquareQ', 'invFisherQ',
+        'invGaussQ', 'invSigmoid', 'invStudentQ', 'length', 'ln', 'lnBeta', 'lnGamma',
+        'log10', 'log2', 'max', 'melToHertz', 'min', 'minusObject', 'natural', 'number',
+        'numberOfColumns', 'numberOfRows', 'numberOfSelected', 'objectsAreIdentical',
+        'option', 'optionMenu', 'pauseScript', 'phonToDifferenceLimens', 'plusObject',
+        'positive', 'randomBinomial', 'randomGauss', 'randomInteger', 'randomPoisson',
+        'randomUniform', 'real', 'readFile', 'removeObject', 'rindex', 'rindex_regex',
+        'round', 'runScript', 'runSystem', 'runSystem_nocheck', 'selectObject',
+        'selected', 'semitonesToHertz', 'sentence', 'sentencetext', 'sigmoid', 'sin', 'sinc',
+        'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP',
+        'studentQ', 'tan', 'tanh', 'text', 'variableExists', 'word', 'writeFile', 'writeFileLine',
+        'writeInfo', 'writeInfoLine',
+    )
+
+    functions_array = (
+        'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero',
+    )
+
+    objects = (
+        'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword',
+        'Autosegment', 'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories',
+        'Cepstrogram', 'Cepstrum', 'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable',
+        'Cochleagram', 'Collection', 'ComplexSpectrogram', 'Configuration', 'Confusion',
+        'ContingencyTable', 'Corpus', 'Correlation', 'Covariance',
+        'CrossCorrelationTable', 'CrossCorrelationTables', 'DTW', 'DataModeler',
+        'Diagonalizer', 'Discriminant', 'Dissimilarity', 'Distance', 'Distributions',
+        'DurationTier', 'EEG', 'ERP', 'ERPTier', 'EditCostsTable', 'EditDistanceTable',
+        'Eigen', 'Excitation', 'Excitations', 'ExperimentMFC', 'FFNet', 'FeatureWeights',
+        'FileInMemory', 'FilesInMemory', 'Formant', 'FormantFilter', 'FormantGrid',
+        'FormantModeler', 'FormantPoint', 'FormantTier', 'GaussianMixture', 'HMM',
+        'HMM_Observation', 'HMM_ObservationSequence', 'HMM_State', 'HMM_StateSequence',
+        'Harmonicity', 'ISpline', 'Index', 'Intensity', 'IntensityTier', 'IntervalTier',
+        'KNN', 'KlattGrid', 'KlattTable', 'LFCC', 'LPC', 'Label', 'LegendreSeries',
+        'LinearRegression', 'LogisticRegression', 'LongSound', 'Ltas', 'MFCC', 'MSpline',
+        'ManPages', 'Manipulation', 'Matrix', 'MelFilter', 'MelSpectrogram',
+        'MixingMatrix', 'Movie', 'Network', 'Object', 'OTGrammar', 'OTHistory', 'OTMulti',
+        'PCA', 'PairDistribution', 'ParamCurve', 'Pattern', 'Permutation', 'Photo',
+        'Pitch', 'PitchModeler', 'PitchTier', 'PointProcess', 'Polygon', 'Polynomial',
+        'PowerCepstrogram', 'PowerCepstrum', 'Procrustes', 'RealPoint', 'RealTier',
+        'ResultsMFC', 'Roots', 'SPINET', 'SSCP', 'SVD', 'Salience', 'ScalarProduct',
+        'Similarity', 'SimpleString', 'SortedSetOfString', 'Sound', 'Speaker',
+        'Spectrogram', 'Spectrum', 'SpectrumTier', 'SpeechSynthesizer', 'SpellingChecker',
+        'Strings', 'StringsIndex', 'Table', 'TableOfReal', 'TextGrid', 'TextInterval',
+        'TextPoint', 'TextTier', 'Tier', 'Transition', 'VocalTract', 'VocalTractTier',
+        'Weight', 'WordList',
+    )
+
+    variables_numeric = (
+        'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined',
+    )
+
+    variables_string = (
+        'praatVersion', 'tab', 'shellDirectory', 'homeDirectory',
+        'preferencesDirectory', 'newline', 'temporaryDirectory',
+        'defaultDirectory',
+    )
+
+    object_attributes = (
+        'ncol', 'nrow', 'xmin', 'ymin', 'xmax', 'ymax', 'nx', 'ny', 'dx', 'dy',
+    )
+
+    tokens = {
+        'root': [
+            (r'(\s+)(#.*?$)',  bygroups(Whitespace, Comment.Single)),
+            (r'^#.*?$',        Comment.Single),
+            (r';[^\n]*',       Comment.Single),
+            (r'\s+',           Whitespace),
+
+            (r'\bprocedure\b', Keyword,       'procedure_definition'),
+            (r'\bcall\b',      Keyword,       'procedure_call'),
+            (r'@',             Name.Function, 'procedure_call'),
+
+            include('function_call'),
+
+            (words(keywords, suffix=r'\b'), Keyword),
+
+            (r'(\bform\b)(\s+)([^\n]+)',
+             bygroups(Keyword, Whitespace, String), 'old_form'),
+
+            (r'(print(?:line|tab)?|echo|exit|asserterror|pause|send(?:praat|socket)|'
+             r'include|execute|system(?:_nocheck)?)(\s+)',
+             bygroups(Keyword, Whitespace), 'string_unquoted'),
+
+            (r'(goto|label)(\s+)(\w+)', bygroups(Keyword, Whitespace, Name.Label)),
+
+            include('variable_name'),
+            include('number'),
+
+            (r'"', String, 'string'),
+
+            (words((objects), suffix=r'(?=\s+\S+\n)'), Name.Class, 'string_unquoted'),
+
+            (r'\b[A-Z]', Keyword, 'command'),
+            (r'(\.{3}|[)(,])', Punctuation),
+        ],
+        'command': [
+            (r'( ?[\w()-]+ ?)', Keyword),
+
+            include('string_interpolated'),
+
+            (r'\.{3}', Keyword, ('#pop', 'old_arguments')),
+            (r':', Keyword, ('#pop', 'comma_list')),
+            (r'\s', Whitespace, '#pop'),
+        ],
+        'procedure_call': [
+            (r'\s+', Whitespace),
+            (r'([\w.]+)(?:(:)|(?:(\s*)(\()))',
+             bygroups(Name.Function, Punctuation,
+                      Text.Whitespace, Punctuation), '#pop'),
+            (r'([\w.]+)', Name.Function, ('#pop', 'old_arguments')),
+        ],
+        'procedure_definition': [
+            (r'\s', Whitespace),
+            (r'([\w.]+)(\s*?[(:])',
+             bygroups(Name.Function, Whitespace), '#pop'),
+            (r'([\w.]+)([^\n]*)',
+             bygroups(Name.Function, Text), '#pop'),
+        ],
+        'function_call': [
+            (words(functions_string, suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'),
+            (words(functions_array, suffix=r'#(?=\s*[:(])'),   Name.Function, 'function'),
+            (words(functions_numeric, suffix=r'(?=\s*[:(])'),  Name.Function, 'function'),
+        ],
+        'function': [
+            (r'\s+',   Whitespace),
+            (r':',     Punctuation, ('#pop', 'comma_list')),
+            (r'\s*\(', Punctuation, ('#pop', 'comma_list')),
+        ],
+        'comma_list': [
+            (r'(\s*\n\s*)(\.{3})', bygroups(Whitespace, Punctuation)),
+
+            (r'(\s*)(?:([)\]])|(\n))', bygroups(
+                Whitespace, Punctuation, Whitespace), '#pop'),
+
+            (r'\s+', Whitespace),
+            (r'"',   String, 'string'),
+            (r'\b(if|then|else|fi|endif)\b', Keyword),
+
+            include('function_call'),
+            include('variable_name'),
+            include('operator'),
+            include('number'),
+
+            (r'[()]', Text),
+            (r',', Punctuation),
+        ],
+        'old_arguments': [
+            (r'\n', Whitespace, '#pop'),
+
+            include('variable_name'),
+            include('operator'),
+            include('number'),
+
+            (r'"', String, 'string'),
+            (r'[^\n]', Text),
+        ],
+        'number': [
+            (r'\n', Whitespace, '#pop'),
+            (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number),
+        ],
+        'object_reference': [
+            include('string_interpolated'),
+            (r'([a-z][a-zA-Z0-9_]*|\d+)', Name.Builtin),
+
+            (words(object_attributes, prefix=r'\.'), Name.Builtin, '#pop'),
+
+            (r'\$', Name.Builtin),
+            (r'\[', Text, '#pop'),
+        ],
+        'variable_name': [
+            include('operator'),
+            include('number'),
+
+            (words(variables_string,  suffix=r'\$'), Name.Variable.Global),
+            (words(variables_numeric,
+             suffix=r'(?=[^a-zA-Z0-9_."\'$#\[:(]|\s|^|$)'),
+             Name.Variable.Global),
+
+            (words(objects, prefix=r'\b', suffix=r"(_)"),
+             bygroups(Name.Builtin, Name.Builtin),
+             'object_reference'),
+
+            (r'\.?_?[a-z][\w.]*(\$|#)?', Text),
+            (r'[\[\]]', Punctuation, 'comma_list'),
+
+            include('string_interpolated'),
+        ],
+        'operator': [
+            (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)',       Operator),
+            (r'(?', Punctuation),
+            (r'"(?:\\x[0-9a-fA-F]+\\|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|'
+             r'\\[0-7]+\\|\\["\\abcefnrstv]|[^\\"])*"', String.Double),
+            (r"'(?:''|[^'])*'", String.Atom),  # quoted atom
+            # Needs to not be followed by an atom.
+            # (r'=(?=\s|[a-zA-Z\[])', Operator),
+            (r'is\b', Operator),
+            (r'(<|>|=<|>=|==|=:=|=|/|//|\*|\+|-)(?=\s|[a-zA-Z0-9\[])',
+             Operator),
+            (r'(mod|div|not)\b', Operator),
+            (r'_', Keyword),  # The don't-care variable
+            (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)),
+            (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+             r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+             r'(\s*)(:-|-->)',
+             bygroups(Name.Function, Text, Operator)),  # function defn
+            (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+             r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+             r'(\s*)(\()',
+             bygroups(Name.Function, Text, Punctuation)),
+            (r'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+             r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*',
+             String.Atom),  # atom, characters
+            # This one includes !
+            (r'[#&*+\-./:<=>?@\\^~\u00a1-\u00bf\u2010-\u303f]+',
+             String.Atom),  # atom, graphics
+            (r'[A-Z_]\w*', Name.Variable),
+            (r'\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text),
+        ],
+        'nested-comment': [
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'[*/]', Comment.Multiline),
+        ],
+    }
+
+    def analyse_text(text):
+        """Competes with IDL and Visual Prolog on *.pro"""
+        if ':-' in text:
+            # Visual Prolog also uses :-
+            return 0.5
+        else:
+            return 0
+
+
+class LogtalkLexer(RegexLexer):
+    """
+    For Logtalk source code.
+    """
+
+    name = 'Logtalk'
+    url = 'http://logtalk.org/'
+    aliases = ['logtalk']
+    filenames = ['*.lgt', '*.logtalk']
+    mimetypes = ['text/x-logtalk']
+    version_added = '0.10'
+
+    tokens = {
+        'root': [
+            # Directives
+            (r'^\s*:-\s', Punctuation, 'directive'),
+            # Comments
+            (r'%.*?\n', Comment),
+            (r'/\*(.|\n)*?\*/', Comment),
+            # Whitespace
+            (r'\n', Text),
+            (r'\s+', Text),
+            # Numbers
+            (r"0'[\\]?.", Number),
+            (r'0b[01]+', Number.Bin),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+            # Variables
+            (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable),
+            # Event handlers
+            (r'(after|before)(?=[(])', Keyword),
+            # Message forwarding handler
+            (r'forward(?=[(])', Keyword),
+            # Execution-context methods
+            (r'(context|parameter|this|se(lf|nder))(?=[(])', Keyword),
+            # Reflection
+            (r'(current_predicate|predicate_property)(?=[(])', Keyword),
+            # DCGs and term expansion
+            (r'(expand_(goal|term)|(goal|term)_expansion|phrase)(?=[(])', Keyword),
+            # Entity
+            (r'(abolish|c(reate|urrent))_(object|protocol|category)(?=[(])', Keyword),
+            (r'(object|protocol|category)_property(?=[(])', Keyword),
+            # Entity relations
+            (r'co(mplements_object|nforms_to_protocol)(?=[(])', Keyword),
+            (r'extends_(object|protocol|category)(?=[(])', Keyword),
+            (r'imp(lements_protocol|orts_category)(?=[(])', Keyword),
+            (r'(instantiat|specializ)es_class(?=[(])', Keyword),
+            # Events
+            (r'(current_event|(abolish|define)_events)(?=[(])', Keyword),
+            # Flags
+            (r'(create|current|set)_logtalk_flag(?=[(])', Keyword),
+            # Compiling, loading, and library paths
+            (r'logtalk_(compile|l(ibrary_path|oad|oad_context)|make(_target_action)?)(?=[(])', Keyword),
+            (r'\blogtalk_make\b', Keyword),
+            # Database
+            (r'(clause|retract(all)?)(?=[(])', Keyword),
+            (r'a(bolish|ssert(a|z))(?=[(])', Keyword),
+            # Control constructs
+            (r'(ca(ll|tch)|throw)(?=[(])', Keyword),
+            (r'(fa(il|lse)|true|(instantiation|system)_error)\b', Keyword),
+            (r'(uninstantiation|type|domain|existence|permission|representation|evaluation|resource|syntax)_error(?=[(])', Keyword),
+            # All solutions
+            (r'((bag|set)of|f(ind|or)all)(?=[(])', Keyword),
+            # Multi-threading predicates
+            (r'threaded(_(ca(ll|ncel)|once|ignore|exit|peek|wait|notify))?(?=[(])', Keyword),
+            # Engine predicates
+            (r'threaded_engine(_(create|destroy|self|next|next_reified|yield|post|fetch))?(?=[(])', Keyword),
+            # Term unification
+            (r'(subsumes_term|unify_with_occurs_check)(?=[(])', Keyword),
+            # Term creation and decomposition
+            (r'(functor|arg|copy_term|numbervars|term_variables)(?=[(])', Keyword),
+            # Evaluable functors
+            (r'(div|rem|m(ax|in|od)|abs|sign)(?=[(])', Keyword),
+            (r'float(_(integer|fractional)_part)?(?=[(])', Keyword),
+            (r'(floor|t(an|runcate)|round|ceiling)(?=[(])', Keyword),
+            # Other arithmetic functors
+            (r'(cos|a(cos|sin|tan|tan2)|exp|log|s(in|qrt)|xor)(?=[(])', Keyword),
+            # Term testing
+            (r'(var|atom(ic)?|integer|float|c(allable|ompound)|n(onvar|umber)|ground|acyclic_term)(?=[(])', Keyword),
+            # Term comparison
+            (r'compare(?=[(])', Keyword),
+            # Stream selection and control
+            (r'(curren|se)t_(in|out)put(?=[(])', Keyword),
+            (r'(open|close)(?=[(])', Keyword),
+            (r'flush_output(?=[(])', Keyword),
+            (r'(at_end_of_stream|flush_output)\b', Keyword),
+            (r'(stream_property|at_end_of_stream|set_stream_position)(?=[(])', Keyword),
+            # Character and byte input/output
+            (r'(nl|(get|peek|put)_(byte|c(har|ode)))(?=[(])', Keyword),
+            (r'\bnl\b', Keyword),
+            # Term input/output
+            (r'read(_term)?(?=[(])', Keyword),
+            (r'write(q|_(canonical|term))?(?=[(])', Keyword),
+            (r'(current_)?op(?=[(])', Keyword),
+            (r'(current_)?char_conversion(?=[(])', Keyword),
+            # Atomic term processing
+            (r'atom_(length|c(hars|o(ncat|des)))(?=[(])', Keyword),
+            (r'(char_code|sub_atom)(?=[(])', Keyword),
+            (r'number_c(har|ode)s(?=[(])', Keyword),
+            # Implementation defined hooks functions
+            (r'(se|curren)t_prolog_flag(?=[(])', Keyword),
+            (r'\bhalt\b', Keyword),
+            (r'halt(?=[(])', Keyword),
+            # Message sending operators
+            (r'(::|:|\^\^)', Operator),
+            # External call
+            (r'[{}]', Keyword),
+            # Logic and control
+            (r'(ignore|once)(?=[(])', Keyword),
+            (r'\brepeat\b', Keyword),
+            # Sorting
+            (r'(key)?sort(?=[(])', Keyword),
+            # Bitwise functors
+            (r'(>>|<<|/\\|\\\\|\\)', Operator),
+            # Predicate aliases
+            (r'\bas\b', Operator),
+            # Arithmetic evaluation
+            (r'\bis\b', Keyword),
+            # Arithmetic comparison
+            (r'(=:=|=\\=|<|=<|>=|>)', Operator),
+            # Term creation and decomposition
+            (r'=\.\.', Operator),
+            # Term unification
+            (r'(=|\\=)', Operator),
+            # Term comparison
+            (r'(==|\\==|@=<|@<|@>=|@>)', Operator),
+            # Evaluable functors
+            (r'(//|[-+*/])', Operator),
+            (r'\b(e|pi|div|mod|rem)\b', Operator),
+            # Other arithmetic functors
+            (r'\b\*\*\b', Operator),
+            # DCG rules
+            (r'-->', Operator),
+            # Control constructs
+            (r'([!;]|->)', Operator),
+            # Logic and control
+            (r'\\+', Operator),
+            # Mode operators
+            (r'[?@]', Operator),
+            # Existential quantifier
+            (r'\^', Operator),
+            # Punctuation
+            (r'[()\[\],.|]', Text),
+            # Atoms
+            (r"[a-z][a-zA-Z0-9_]*", Text),
+            (r"'", String, 'quoted_atom'),
+            # Double-quoted terms
+            (r'"', String, 'double_quoted_term'),
+        ],
+
+        'quoted_atom': [
+            (r"''", String),
+            (r"'", String, '#pop'),
+            (r'\\([\\abfnrtv"\']|(x[a-fA-F0-9]+|[0-7]+)\\)', String.Escape),
+            (r"[^\\'\n]+", String),
+            (r'\\', String),
+        ],
+
+        'double_quoted_term': [
+            (r'""', String),
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|(x[a-fA-F0-9]+|[0-7]+)\\)', String.Escape),
+            (r'[^\\"\n]+', String),
+            (r'\\', String),
+        ],
+
+        'directive': [
+            # Conditional compilation directives
+            (r'(el)?if(?=[(])', Keyword, 'root'),
+            (r'(e(lse|ndif))(?=[.])', Keyword, 'root'),
+            # Entity directives
+            (r'(category|object|protocol)(?=[(])', Keyword, 'entityrelations'),
+            (r'(end_(category|object|protocol))(?=[.])', Keyword, 'root'),
+            # Predicate scope directives
+            (r'(public|protected|private)(?=[(])', Keyword, 'root'),
+            # Other directives
+            (r'e(n(coding|sure_loaded)|xport)(?=[(])', Keyword, 'root'),
+            (r'in(clude|itialization|fo)(?=[(])', Keyword, 'root'),
+            (r'(built_in|dynamic|synchronized|threaded)(?=[.])', Keyword, 'root'),
+            (r'(alias|d(ynamic|iscontiguous)|m(eta_(non_terminal|predicate)|ode|ultifile)|s(et_(logtalk|prolog)_flag|ynchronized))(?=[(])', Keyword, 'root'),
+            (r'op(?=[(])', Keyword, 'root'),
+            (r'(c(alls|oinductive)|module|reexport|use(s|_module))(?=[(])', Keyword, 'root'),
+            (r'[a-z][a-zA-Z0-9_]*(?=[(])', Text, 'root'),
+            (r'[a-z][a-zA-Z0-9_]*(?=[.])', Text, 'root'),
+        ],
+
+        'entityrelations': [
+            (r'(complements|extends|i(nstantiates|mp(lements|orts))|specializes)(?=[(])', Keyword),
+            # Numbers
+            (r"0'[\\]?.", Number),
+            (r'0b[01]+', Number.Bin),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+            # Variables
+            (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable),
+            # Atoms
+            (r"[a-z][a-zA-Z0-9_]*", Text),
+            (r"'", String, 'quoted_atom'),
+            # Double-quoted terms
+            (r'"', String, 'double_quoted_term'),
+            # End of entity-opening directive
+            (r'([)]\.)', Text, 'root'),
+            # Scope operator
+            (r'(::)', Operator),
+            # Punctuation
+            (r'[()\[\],.|]', Text),
+            # Comments
+            (r'%.*?\n', Comment),
+            (r'/\*(.|\n)*?\*/', Comment),
+            # Whitespace
+            (r'\n', Text),
+            (r'\s+', Text),
+        ]
+    }
+
+    def analyse_text(text):
+        if ':- object(' in text:
+            return 1.0
+        elif ':- protocol(' in text:
+            return 1.0
+        elif ':- category(' in text:
+            return 1.0
+        elif re.search(r'^:-\s[a-z]', text, re.M):
+            return 0.9
+        else:
+            return 0.0
diff --git a/lib/pygments/lexers/promql.py b/lib/pygments/lexers/promql.py
new file mode 100644
index 0000000..cad3c25
--- /dev/null
+++ b/lib/pygments/lexers/promql.py
@@ -0,0 +1,176 @@
+"""
+    pygments.lexers.promql
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Prometheus Query Language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, default, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Whitespace
+
+__all__ = ["PromQLLexer"]
+
+
+class PromQLLexer(RegexLexer):
+    """
+    For PromQL queries.
+
+    For details about the grammar see:
+    https://github.com/prometheus/prometheus/tree/master/promql/parser
+
+    .. versionadded: 2.7
+    """
+
+    name = "PromQL"
+    url = 'https://prometheus.io/docs/prometheus/latest/querying/basics/'
+    aliases = ["promql"]
+    filenames = ["*.promql"]
+    version_added = ''
+
+    base_keywords = (
+        words(
+            (
+                "bool",
+                "by",
+                "group_left",
+                "group_right",
+                "ignoring",
+                "offset",
+                "on",
+                "without",
+            ),
+            suffix=r"\b",
+        ),
+        Keyword,
+    )
+
+    aggregator_keywords = (
+        words(
+            (
+                "sum",
+                "min",
+                "max",
+                "avg",
+                "group",
+                "stddev",
+                "stdvar",
+                "count",
+                "count_values",
+                "bottomk",
+                "topk",
+                "quantile",
+            ),
+            suffix=r"\b",
+        ),
+        Keyword,
+    )
+
+    function_keywords = (
+        words(
+            (
+                "abs",
+                "absent",
+                "absent_over_time",
+                "avg_over_time",
+                "ceil",
+                "changes",
+                "clamp_max",
+                "clamp_min",
+                "count_over_time",
+                "day_of_month",
+                "day_of_week",
+                "days_in_month",
+                "delta",
+                "deriv",
+                "exp",
+                "floor",
+                "histogram_quantile",
+                "holt_winters",
+                "hour",
+                "idelta",
+                "increase",
+                "irate",
+                "label_join",
+                "label_replace",
+                "ln",
+                "log10",
+                "log2",
+                "max_over_time",
+                "min_over_time",
+                "minute",
+                "month",
+                "predict_linear",
+                "quantile_over_time",
+                "rate",
+                "resets",
+                "round",
+                "scalar",
+                "sort",
+                "sort_desc",
+                "sqrt",
+                "stddev_over_time",
+                "stdvar_over_time",
+                "sum_over_time",
+                "time",
+                "timestamp",
+                "vector",
+                "year",
+            ),
+            suffix=r"\b",
+        ),
+        Keyword.Reserved,
+    )
+
+    tokens = {
+        "root": [
+            (r"\n", Whitespace),
+            (r"\s+", Whitespace),
+            (r",", Punctuation),
+            # Keywords
+            base_keywords,
+            aggregator_keywords,
+            function_keywords,
+            # Offsets
+            (r"[1-9][0-9]*[smhdwy]", String),
+            # Numbers
+            (r"-?[0-9]+\.[0-9]+", Number.Float),
+            (r"-?[0-9]+", Number.Integer),
+            # Comments
+            (r"#.*?$", Comment.Single),
+            # Operators
+            (r"(\+|\-|\*|\/|\%|\^)", Operator),
+            (r"==|!=|>=|<=|<|>", Operator),
+            (r"and|or|unless", Operator.Word),
+            # Metrics
+            (r"[_a-zA-Z][a-zA-Z0-9_]+", Name.Variable),
+            # Params
+            (r'(["\'])(.*?)(["\'])', bygroups(Punctuation, String, Punctuation)),
+            # Other states
+            (r"\(", Operator, "function"),
+            (r"\)", Operator),
+            (r"\{", Punctuation, "labels"),
+            (r"\[", Punctuation, "range"),
+        ],
+        "labels": [
+            (r"\}", Punctuation, "#pop"),
+            (r"\n", Whitespace),
+            (r"\s+", Whitespace),
+            (r",", Punctuation),
+            (r'([_a-zA-Z][a-zA-Z0-9_]*?)(\s*?)(=~|!=|=|!~)(\s*?)("|\')(.*?)("|\')',
+             bygroups(Name.Label, Whitespace, Operator, Whitespace,
+                      Punctuation, String, Punctuation)),
+        ],
+        "range": [
+            (r"\]", Punctuation, "#pop"),
+            (r"[1-9][0-9]*[smhdwy]", String),
+        ],
+        "function": [
+            (r"\)", Operator, "#pop"),
+            (r"\(", Operator, "#push"),
+            default("#pop"),
+        ],
+    }
diff --git a/lib/pygments/lexers/prql.py b/lib/pygments/lexers/prql.py
new file mode 100644
index 0000000..ee95d2d
--- /dev/null
+++ b/lib/pygments/lexers/prql.py
@@ -0,0 +1,251 @@
+"""
+    pygments.lexers.prql
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the PRQL query language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, combined, words, include, bygroups
+from pygments.token import Comment, Literal, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text, Whitespace
+
+__all__ = ['PrqlLexer']
+
+
+class PrqlLexer(RegexLexer):
+    """
+    For PRQL source code.
+
+    grammar: https://github.com/PRQL/prql/tree/main/grammars
+    """
+
+    name = 'PRQL'
+    url = 'https://prql-lang.org/'
+    aliases = ['prql']
+    filenames = ['*.prql']
+    mimetypes = ['application/prql', 'application/x-prql']
+    version_added = '2.17'
+
+    builtinTypes = words((
+        "bool",
+        "int",
+        "int8", "int16", "int32", "int64", "int128",
+        "float",
+        "text",
+        "set"), suffix=r'\b')
+
+    def innerstring_rules(ttype):
+        return [
+            # the new style '{}'.format(...) string formatting
+            (r'\{'
+             r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?'  # field name
+             r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
+             r'\}', String.Interpol),
+
+            (r'[^\\\'"%{\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            (r'%|(\{{1,2})', ttype)
+        ]
+
+    def fstring_rules(ttype):
+        return [
+            (r'\}', String.Interpol),
+            (r'\{', String.Interpol, 'expr-inside-fstring'),
+            (r'[^\\\'"{}\n]+', ttype),
+            (r'[\'"\\]', ttype),
+        ]
+
+    tokens = {
+        'root': [
+
+            # Comments
+            (r'#!.*', String.Doc),
+            (r'#.*', Comment.Single),
+
+            # Whitespace
+            (r'\s+', Whitespace),
+
+            # Modules
+            (r'^(\s*)(module)(\s*)',
+             bygroups(Whitespace, Keyword.Namespace, Whitespace),
+             'imports'),
+
+            (builtinTypes, Keyword.Type),
+
+            # Main
+            (r'^prql ', Keyword.Reserved),
+
+            ('let', Keyword.Declaration),
+
+            include('keywords'),
+            include('expr'),
+
+            # Transforms
+            (r'^[A-Za-z_][a-zA-Z0-9_]*', Keyword),
+        ],
+        'expr': [
+            # non-raw f-strings
+            ('(f)(""")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'tdqf')),
+            ("(f)(''')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'tsqf')),
+            ('(f)(")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'dqf')),
+            ("(f)(')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'sqf')),
+
+            # non-raw s-strings
+            ('(s)(""")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'tdqf')),
+            ("(s)(''')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'tsqf')),
+            ('(s)(")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'dqf')),
+            ("(s)(')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'sqf')),
+
+            # raw strings
+            ('(?i)(r)(""")',
+             bygroups(String.Affix, String.Double), 'tdqs'),
+            ("(?i)(r)(''')",
+             bygroups(String.Affix, String.Single), 'tsqs'),
+            ('(?i)(r)(")',
+             bygroups(String.Affix, String.Double), 'dqs'),
+            ("(?i)(r)(')",
+             bygroups(String.Affix, String.Single), 'sqs'),
+
+            # non-raw strings
+            ('"""', String.Double, combined('stringescape', 'tdqs')),
+            ("'''", String.Single, combined('stringescape', 'tsqs')),
+            ('"', String.Double, combined('stringescape', 'dqs')),
+            ("'", String.Single, combined('stringescape', 'sqs')),
+
+            # Time and dates
+            (r'@\d{4}-\d{2}-\d{2}T\d{2}(:\d{2})?(:\d{2})?(\.\d{1,6})?(Z|[+-]\d{1,2}(:\d{1,2})?)?', Literal.Date),
+            (r'@\d{4}-\d{2}-\d{2}', Literal.Date),
+            (r'@\d{2}(:\d{2})?(:\d{2})?(\.\d{1,6})?(Z|[+-]\d{1,2}(:\d{1,2})?)?', Literal.Date),
+
+            (r'[^\S\n]+', Text),
+            include('numbers'),
+            (r'->|=>|==|!=|>=|<=|~=|&&|\|\||\?\?|\/\/', Operator),
+            (r'[-~+/*%=<>&^|.@]', Operator),
+            (r'[]{}:(),;[]', Punctuation),
+            include('functions'),
+
+            # Variable Names
+            (r'[A-Za-z_][a-zA-Z0-9_]*', Name.Variable),
+        ],
+        'numbers': [
+            (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
+             r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
+            (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
+            (r'0[oO](?:_?[0-7])+', Number.Oct),
+            (r'0[bB](?:_?[01])+', Number.Bin),
+            (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
+            (r'\d(?:_?\d)*', Number.Integer),
+        ],
+        'fstringescape': [
+            include('stringescape'),
+        ],
+        'bytesescape': [
+            (r'\\([\\bfnrt"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+        ],
+        'stringescape': [
+            (r'\\(N\{.*?\}|u\{[a-fA-F0-9]{1,6}\})', String.Escape),
+            include('bytesescape')
+        ],
+        'fstrings-single': fstring_rules(String.Single),
+        'fstrings-double': fstring_rules(String.Double),
+        'strings-single': innerstring_rules(String.Single),
+        'strings-double': innerstring_rules(String.Double),
+        'dqf': [
+            (r'"', String.Double, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
+            include('fstrings-double')
+        ],
+        'sqf': [
+            (r"'", String.Single, '#pop'),
+            (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
+            include('fstrings-single')
+        ],
+        'dqs': [
+            (r'"', String.Double, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
+            include('strings-double')
+        ],
+        'sqs': [
+            (r"'", String.Single, '#pop'),
+            (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
+            include('strings-single')
+        ],
+        'tdqf': [
+            (r'"""', String.Double, '#pop'),
+            include('fstrings-double'),
+            (r'\n', String.Double)
+        ],
+        'tsqf': [
+            (r"'''", String.Single, '#pop'),
+            include('fstrings-single'),
+            (r'\n', String.Single)
+        ],
+        'tdqs': [
+            (r'"""', String.Double, '#pop'),
+            include('strings-double'),
+            (r'\n', String.Double)
+        ],
+        'tsqs': [
+            (r"'''", String.Single, '#pop'),
+            include('strings-single'),
+            (r'\n', String.Single)
+        ],
+
+        'expr-inside-fstring': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            # without format specifier
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r'\}', String.Interpol, '#pop'),
+            # with format specifier
+            # we'll catch the remaining '}' in the outer scope
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r':', String.Interpol, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'expr-inside-fstring-inner': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            (r'[])}]', Punctuation, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'keywords': [
+            (words((
+                'into', 'case', 'type', 'module', 'internal',
+            ), suffix=r'\b'),
+                Keyword),
+            (words(('true', 'false', 'null'), suffix=r'\b'), Keyword.Constant),
+        ],
+        'functions': [
+            (words((
+                "min", "max", "sum", "average", "stddev", "every", "any",
+                "concat_array", "count", "lag", "lead", "first", "last",
+                "rank", "rank_dense", "row_number", "round", "as", "in",
+                "tuple_every", "tuple_map", "tuple_zip", "_eq", "_is_null",
+                "from_text", "lower", "upper", "read_parquet", "read_csv"),
+                suffix=r'\b'),
+             Name.Function),
+        ],
+
+        'comment': [
+            (r'-(?!\})', Comment.Multiline),
+            (r'\{-', Comment.Multiline, 'comment'),
+            (r'[^-}]', Comment.Multiline),
+            (r'-\}', Comment.Multiline, '#pop'),
+        ],
+
+        'imports': [
+            (r'\w+(\.\w+)*', Name.Class, '#pop'),
+        ],
+    }
diff --git a/lib/pygments/lexers/ptx.py b/lib/pygments/lexers/ptx.py
new file mode 100644
index 0000000..784ca13
--- /dev/null
+++ b/lib/pygments/lexers/ptx.py
@@ -0,0 +1,119 @@
+"""
+    pygments.lexers.ptx
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for other PTX language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Keyword, Name, String, Number, \
+    Punctuation, Whitespace, Operator
+
+__all__ = ["PtxLexer"]
+
+
+class PtxLexer(RegexLexer):
+    """
+    For NVIDIA `PTX `_
+    source.
+    """
+    name = 'PTX'
+    url = "https://docs.nvidia.com/cuda/parallel-thread-execution/"
+    filenames = ['*.ptx']
+    aliases = ['ptx']
+    mimetypes = ['text/x-ptx']
+    version_added = '2.16'
+
+    #: optional Comment or Whitespace
+    string = r'"[^"]*?"'
+    followsym = r'[a-zA-Z0-9_$]'
+    identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
+    block_label = r'(' + identifier + r'|(\d+))'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+
+            (block_label + r'\s*:', Name.Label),
+
+            include('keyword'),
+
+            (r'%' + identifier, Name.Variable),
+            (r'%\d+', Name.Variable.Anonymous),
+            (r'c?' + string, String),
+            (identifier, Name.Variable),
+            (r';', Punctuation),
+            (r'[*+-/]', Operator),
+
+            (r'0[xX][a-fA-F0-9]+', Number),
+            (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
+
+            (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
+
+        ],
+        'whitespace': [
+            (r'(\n|\s+)+', Whitespace),
+            (r'//.*?\n', Comment)
+        ],
+
+        'keyword': [
+            # Instruction keywords
+            (words((
+                'abs', 'discard', 'min', 'shf', 'vadd',
+                'activemask', 'div', 'mma', 'shfl', 'vadd2',
+                'add', 'dp2a', 'mov', 'shl', 'vadd4',
+                'addc', 'dp4a', 'movmatrix', 'shr', 'vavrg2',
+                'alloca', 'elect', 'mul', 'sin', 'vavrg4',
+                'and', 'ex2', 'mul24', 'slct', 'vmad',
+                'applypriority', 'exit', 'multimem', 'sqrt', 'vmax',
+                'atom', 'fence', 'nanosleep', 'st', 'vmax2',
+                'bar', 'fma', 'neg', 'stackrestore', 'vmax4',
+                'barrier', 'fns', 'not', 'stacksave', 'vmin',
+                'bfe', 'getctarank', 'or', 'stmatrix', 'vmin2',
+                'bfi', 'griddepcontrol', 'pmevent', 'sub', 'vmin4',
+                'bfind', 'isspacep', 'popc', 'subc', 'vote',
+                'bmsk', 'istypep', 'prefetch', 'suld', 'vset',
+                'bra', 'ld', 'prefetchu', 'suq', 'vset2',
+                'brev', 'ldmatrix', 'prmt', 'sured', 'vset4',
+                'brkpt', 'ldu', 'rcp', 'sust', 'vshl',
+                'brx', 'lg2', 'red', 'szext', 'vshr',
+                'call', 'lop3', 'redux', 'tanh', 'vsub',
+                'clz', 'mad', 'rem', 'testp', 'vsub2',
+                'cnot', 'mad24', 'ret', 'tex', 'vsub4',
+                'copysign', 'madc', 'rsqrt', 'tld4', 'wgmma',
+                'cos', 'mapa', 'sad', 'trap', 'wmma',
+                'cp', 'match', 'selp', 'txq', 'xor',
+                'createpolicy', 'max', 'set', 'vabsdiff', 'cvt',
+                'mbarrier', 'setmaxnreg', 'vabsdiff2', 'cvta',
+                'membar', 'setp', 'vabsdiff4')), Keyword),
+            # State Spaces and Suffixes
+            (words((
+                'reg', '.sreg', '.const', '.global',
+                '.local', '.param', '.shared', '.tex',
+                '.wide', '.loc'
+            )), Keyword.Pseudo),
+            # PTX Directives
+            (words((
+                '.address_size', '.explicitcluster', '.maxnreg', '.section',
+                '.alias', '.extern', '.maxntid', '.shared',
+                '.align', '.file', '.minnctapersm', '.sreg',
+                '.branchtargets', '.func', '.noreturn', '.target',
+                '.callprototype', '.global', '.param', '.tex',
+                '.calltargets', '.loc', '.pragma', '.version',
+                '.common', '.local', '.reg', '.visible',
+                '.const', '.maxclusterrank', '.reqnctapercluster', '.weak',
+                '.entry', '.maxnctapersm', '.reqntid')), Keyword.Reserved),
+            # Fundamental Types
+            (words((
+                '.s8', '.s16', '.s32', '.s64',
+                '.u8', '.u16', '.u32', '.u64',
+                '.f16', '.f16x2', '.f32', '.f64',
+                '.b8', '.b16', '.b32', '.b64',
+                '.pred'
+            )), Keyword.Type)
+        ],
+
+    }
diff --git a/lib/pygments/lexers/python.py b/lib/pygments/lexers/python.py
new file mode 100644
index 0000000..805f6ff
--- /dev/null
+++ b/lib/pygments/lexers/python.py
@@ -0,0 +1,1201 @@
+"""
+    pygments.lexers.python
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Python and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import keyword
+
+from pygments.lexer import DelegatingLexer, RegexLexer, include, \
+    bygroups, using, default, words, combined, this
+from pygments.util import get_bool_opt, shebang_matches
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Generic, Other, Error, Whitespace
+from pygments import unistring as uni
+
+__all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
+           'Python2Lexer', 'Python2TracebackLexer',
+           'CythonLexer', 'DgLexer', 'NumPyLexer']
+
+
+class PythonLexer(RegexLexer):
+    """
+    For Python source code (version 3.x).
+
+    .. versionchanged:: 2.5
+       This is now the default ``PythonLexer``.  It is still available as the
+       alias ``Python3Lexer``.
+    """
+
+    name = 'Python'
+    url = 'https://www.python.org'
+    aliases = ['python', 'py', 'sage', 'python3', 'py3', 'bazel', 'starlark', 'pyi']
+    filenames = [
+        '*.py',
+        '*.pyw',
+        # Type stubs
+        '*.pyi',
+        # Jython
+        '*.jy',
+        # Sage
+        '*.sage',
+        # SCons
+        '*.sc',
+        'SConstruct',
+        'SConscript',
+        # Skylark/Starlark (used by Bazel, Buck, and Pants)
+        '*.bzl',
+        'BUCK',
+        'BUILD',
+        'BUILD.bazel',
+        'WORKSPACE',
+        # Twisted Application infrastructure
+        '*.tac',
+    ]
+    mimetypes = ['text/x-python', 'application/x-python',
+                 'text/x-python3', 'application/x-python3']
+    version_added = '0.10'
+
+    uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*"
+
+    def innerstring_rules(ttype):
+        return [
+            # the old style '%s' % (...) string formatting (still valid in Py3)
+            (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+             '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
+            # the new style '{}'.format(...) string formatting
+            (r'\{'
+             r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?'  # field name
+             r'(\![sra])?'                       # conversion
+             r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
+             r'\}', String.Interpol),
+
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"%{\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # unhandled string formatting sign
+            (r'%|(\{{1,2})', ttype)
+            # newlines are an error (use "nl" state)
+        ]
+
+    def fstring_rules(ttype):
+        return [
+            # Assuming that a '}' is the closing brace after format specifier.
+            # Sadly, this means that we won't detect syntax error. But it's
+            # more important to parse correct syntax correctly, than to
+            # highlight invalid syntax.
+            (r'\}', String.Interpol),
+            (r'\{', String.Interpol, 'expr-inside-fstring'),
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"{}\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # newlines are an error (use "nl" state)
+        ]
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
+             bygroups(Whitespace, String.Affix, String.Doc)),
+            (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
+             bygroups(Whitespace, String.Affix, String.Doc)),
+            (r'\A#!.+$', Comment.Hashbang),
+            (r'#.*$', Comment.Single),
+            (r'\\\n', Text),
+            (r'\\', Text),
+            include('keywords'),
+            include('soft-keywords'),
+            (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'fromimport'),
+            (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'import'),
+            include('expr'),
+        ],
+        'expr': [
+            # raw f-strings
+            ('(?i)(rf|fr)(""")',
+             bygroups(String.Affix, String.Double),
+             combined('rfstringescape', 'tdqf')),
+            ("(?i)(rf|fr)(''')",
+             bygroups(String.Affix, String.Single),
+             combined('rfstringescape', 'tsqf')),
+            ('(?i)(rf|fr)(")',
+             bygroups(String.Affix, String.Double),
+             combined('rfstringescape', 'dqf')),
+            ("(?i)(rf|fr)(')",
+             bygroups(String.Affix, String.Single),
+             combined('rfstringescape', 'sqf')),
+            # non-raw f-strings
+            ('([fF])(""")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'tdqf')),
+            ("([fF])(''')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'tsqf')),
+            ('([fF])(")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'dqf')),
+            ("([fF])(')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'sqf')),
+            # raw bytes and strings
+            ('(?i)(rb|br|r)(""")',
+             bygroups(String.Affix, String.Double), 'tdqs'),
+            ("(?i)(rb|br|r)(''')",
+             bygroups(String.Affix, String.Single), 'tsqs'),
+            ('(?i)(rb|br|r)(")',
+             bygroups(String.Affix, String.Double), 'dqs'),
+            ("(?i)(rb|br|r)(')",
+             bygroups(String.Affix, String.Single), 'sqs'),
+            # non-raw strings
+            ('([uU]?)(""")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'tdqs')),
+            ("([uU]?)(''')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'tsqs')),
+            ('([uU]?)(")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'dqs')),
+            ("([uU]?)(')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'sqs')),
+            # non-raw bytes
+            ('([bB])(""")', bygroups(String.Affix, String.Double),
+             combined('bytesescape', 'tdqs')),
+            ("([bB])(''')", bygroups(String.Affix, String.Single),
+             combined('bytesescape', 'tsqs')),
+            ('([bB])(")', bygroups(String.Affix, String.Double),
+             combined('bytesescape', 'dqs')),
+            ("([bB])(')", bygroups(String.Affix, String.Single),
+             combined('bytesescape', 'sqs')),
+
+            (r'[^\S\n]+', Text),
+            include('numbers'),
+            (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            include('expr-keywords'),
+            include('builtins'),
+            include('magicfuncs'),
+            include('magicvars'),
+            include('name'),
+        ],
+        'expr-inside-fstring': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            # without format specifier
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r'(\![sraf])?'     # conversion
+             r'\}', String.Interpol, '#pop'),
+            # with format specifier
+            # we'll catch the remaining '}' in the outer scope
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r'(\![sraf])?'     # conversion
+             r':', String.Interpol, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'expr-inside-fstring-inner': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            (r'[])}]', Punctuation, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'expr-keywords': [
+            # Based on https://docs.python.org/3/reference/expressions.html
+            (words((
+                'async for', 'await', 'else', 'for', 'if', 'lambda',
+                'yield', 'yield from'), suffix=r'\b'),
+             Keyword),
+            (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
+                'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
+                'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
+                'yield from', 'as', 'with'), suffix=r'\b'),
+             Keyword),
+            (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
+        ],
+        'soft-keywords': [
+            # `match`, `case` and `_` soft keywords
+            (r'(^[ \t]*)'              # at beginning of line + possible indentation
+             r'(match|case)\b'         # a possible keyword
+             r'(?![ \t]*(?:'           # not followed by...
+             r'[:,;=^&|@~)\]}]|(?:' +  # characters and keywords that mean this isn't
+                                       # pattern matching (but None/True/False is ok)
+             r'|'.join(k for k in keyword.kwlist if k[0].islower()) + r')\b))',
+             bygroups(Text, Keyword), 'soft-keywords-inner'),
+        ],
+        'soft-keywords-inner': [
+            # optional `_` keyword
+            (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)),
+            default('#pop')
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'aiter', 'all', 'any', 'bin', 'bool', 'bytearray',
+                'breakpoint', 'bytes', 'callable', 'chr', 'classmethod', 'compile',
+                'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
+                'filter', 'float', 'format', 'frozenset', 'getattr', 'globals',
+                'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'isinstance',
+                'issubclass', 'iter', 'len', 'list', 'locals', 'map', 'max',
+                'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow',
+                'print', 'property', 'range', 'repr', 'reversed', 'round', 'set',
+                'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
+                'tuple', 'type', 'vars', 'zip'), prefix=r'(?>|[-~+/*%=<>&^|.]', Operator),
+            include('keywords'),
+            (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'fromimport'),
+            (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'import'),
+            include('builtins'),
+            include('magicfuncs'),
+            include('magicvars'),
+            include('backtick'),
+            ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
+             bygroups(String.Affix, String.Double), 'tdqs'),
+            ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
+             bygroups(String.Affix, String.Single), 'tsqs'),
+            ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
+             bygroups(String.Affix, String.Double), 'dqs'),
+            ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
+             bygroups(String.Affix, String.Single), 'sqs'),
+            ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'tdqs')),
+            ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'tsqs')),
+            ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'dqs')),
+            ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
+                'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
+                'print', 'raise', 'return', 'try', 'while', 'yield',
+                'yield from', 'as', 'with'), suffix=r'\b'),
+             Keyword),
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
+                'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
+                'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
+                'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
+                'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
+                'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
+                'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
+                'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
+                'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
+                'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
+                'unichr', 'unicode', 'vars', 'xrange', 'zip'),
+                prefix=r'(?>> )(.*\n)', bygroups(Generic.Prompt, Other.Code), 'continuations'),
+            # This happens, e.g., when tracebacks are embedded in documentation;
+            # trailing whitespaces are often stripped in such contexts.
+            (r'(>>>)(\n)', bygroups(Generic.Prompt, Whitespace)),
+            (r'(\^C)?Traceback \(most recent call last\):\n', Other.Traceback, 'traceback'),
+            # SyntaxError starts with this
+            (r'  File "[^"]+", line \d+', Other.Traceback, 'traceback'),
+            (r'.*\n', Generic.Output),
+        ],
+        'continuations': [
+            (r'(\.\.\. )(.*\n)', bygroups(Generic.Prompt, Other.Code)),
+            # See above.
+            (r'(\.\.\.)(\n)', bygroups(Generic.Prompt, Whitespace)),
+            default('#pop'),
+        ],
+        'traceback': [
+            # As soon as we see a traceback, consume everything until the next
+            # >>> prompt.
+            (r'(?=>>>( |$))', Text, '#pop'),
+            (r'(KeyboardInterrupt)(\n)', bygroups(Name.Class, Whitespace)),
+            (r'.*\n', Other.Traceback),
+        ],
+    }
+
+
+class PythonConsoleLexer(DelegatingLexer):
+    """
+    For Python console output or doctests, such as:
+
+    .. sourcecode:: pycon
+
+        >>> a = 'foo'
+        >>> print(a)
+        foo
+        >>> 1 / 0
+        Traceback (most recent call last):
+          File "", line 1, in 
+        ZeroDivisionError: integer division or modulo by zero
+
+    Additional options:
+
+    `python3`
+        Use Python 3 lexer for code.  Default is ``True``.
+
+        .. versionadded:: 1.0
+        .. versionchanged:: 2.5
+           Now defaults to ``True``.
+    """
+
+    name = 'Python console session'
+    aliases = ['pycon', 'python-console']
+    mimetypes = ['text/x-python-doctest']
+    url = 'https://python.org'
+    version_added = ''
+
+    def __init__(self, **options):
+        python3 = get_bool_opt(options, 'python3', True)
+        if python3:
+            pylexer = PythonLexer
+            tblexer = PythonTracebackLexer
+        else:
+            pylexer = Python2Lexer
+            tblexer = Python2TracebackLexer
+        # We have two auxiliary lexers. Use DelegatingLexer twice with
+        # different tokens.  TODO: DelegatingLexer should support this
+        # directly, by accepting a tuplet of auxiliary lexers and a tuple of
+        # distinguishing tokens. Then we wouldn't need this intermediary
+        # class.
+        class _ReplaceInnerCode(DelegatingLexer):
+            def __init__(self, **options):
+                super().__init__(pylexer, _PythonConsoleLexerBase, Other.Code, **options)
+        super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
+
+
+class PythonTracebackLexer(RegexLexer):
+    """
+    For Python 3.x tracebacks, with support for chained exceptions.
+
+    .. versionchanged:: 2.5
+       This is now the default ``PythonTracebackLexer``.  It is still available
+       as the alias ``Python3TracebackLexer``.
+    """
+
+    name = 'Python Traceback'
+    aliases = ['pytb', 'py3tb']
+    filenames = ['*.pytb', '*.py3tb']
+    mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
+    url = 'https://python.org'
+    version_added = '1.0'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'^(\^C)?Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
+            (r'^During handling of the above exception, another '
+             r'exception occurred:\n\n', Generic.Traceback),
+            (r'^The above exception was the direct cause of the '
+             r'following exception:\n\n', Generic.Traceback),
+            (r'^(?=  File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
+            (r'^.*\n', Other),
+        ],
+        'intb': [
+            (r'^(  File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
+            (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
+            (r'^(    )(.+)(\n)',
+             bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'),
+            (r'^([ \t]*)(\.\.\.)(\n)',
+             bygroups(Whitespace, Comment, Whitespace)),  # for doctests...
+            (r'^([^:]+)(: )(.+)(\n)',
+             bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
+            (r'^([a-zA-Z_][\w.]*)(:?\n)',
+             bygroups(Generic.Error, Whitespace), '#pop'),
+            default('#pop'),
+        ],
+        'markers': [
+            # Either `PEP 657 `
+            # error locations in Python 3.11+, or single-caret markers
+            # for syntax errors before that.
+            (r'^( {4,})([~^]+)(\n)',
+             bygroups(Whitespace, Punctuation.Marker, Whitespace),
+             '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+Python3TracebackLexer = PythonTracebackLexer
+
+
+class Python2TracebackLexer(RegexLexer):
+    """
+    For Python tracebacks.
+
+    .. versionchanged:: 2.5
+       This class has been renamed from ``PythonTracebackLexer``.
+       ``PythonTracebackLexer`` now refers to the Python 3 variant.
+    """
+
+    name = 'Python 2.x Traceback'
+    aliases = ['py2tb']
+    filenames = ['*.py2tb']
+    mimetypes = ['text/x-python2-traceback']
+    url = 'https://python.org'
+    version_added = '0.7'
+
+    tokens = {
+        'root': [
+            # Cover both (most recent call last) and (innermost last)
+            # The optional ^C allows us to catch keyboard interrupt signals.
+            (r'^(\^C)?(Traceback.*\n)',
+             bygroups(Text, Generic.Traceback), 'intb'),
+            # SyntaxError starts with this.
+            (r'^(?=  File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
+            (r'^.*\n', Other),
+        ],
+        'intb': [
+            (r'^(  File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
+            (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
+            (r'^(    )(.+)(\n)',
+             bygroups(Text, using(Python2Lexer), Whitespace), 'marker'),
+            (r'^([ \t]*)(\.\.\.)(\n)',
+             bygroups(Text, Comment, Whitespace)),  # for doctests...
+            (r'^([^:]+)(: )(.+)(\n)',
+             bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
+            (r'^([a-zA-Z_]\w*)(:?\n)',
+             bygroups(Generic.Error, Whitespace), '#pop')
+        ],
+        'marker': [
+            # For syntax errors.
+            (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+class CythonLexer(RegexLexer):
+    """
+    For Pyrex and Cython source code.
+    """
+
+    name = 'Cython'
+    url = 'https://cython.org'
+    aliases = ['cython', 'pyx', 'pyrex']
+    filenames = ['*.pyx', '*.pxd', '*.pxi']
+    mimetypes = ['text/x-cython', 'application/x-cython']
+    version_added = '1.1'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)),
+            (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)),
+            (r'[^\S\n]+', Text),
+            (r'#.*$', Comment),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'\\\n', Whitespace),
+            (r'\\', Text),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            (r'(<)([a-zA-Z0-9.?]+)(>)',
+             bygroups(Punctuation, Keyword.Type, Punctuation)),
+            (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
+            (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
+             bygroups(Keyword, Number.Integer, Operator, Whitespace, Operator,
+                      Name, Punctuation)),
+            include('keywords'),
+            (r'(def|property)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(cp?def)(\s+)', bygroups(Keyword, Whitespace), 'cdef'),
+            # (should actually start a block with only cdefs)
+            (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
+            (r'(class|struct)(\s+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(from)(\s+)', bygroups(Keyword, Whitespace), 'fromimport'),
+            (r'(c?import)(\s+)', bygroups(Keyword, Whitespace), 'import'),
+            include('builtins'),
+            include('backtick'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
+            ('[uU]?"""', String, combined('stringescape', 'tdqs')),
+            ("[uU]?'''", String, combined('stringescape', 'tsqs')),
+            ('[uU]?"', String, combined('stringescape', 'dqs')),
+            ("[uU]?'", String, combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif',
+                'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
+                'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print',
+                'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'),
+             Keyword),
+            (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint',
+                'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr',
+                'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
+                'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit',
+                'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
+                'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
+                'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
+                'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t',
+                'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
+                'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
+                'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned',
+                'vars', 'xrange', 'zip'), prefix=r'(??/\\:']?:)(\s*)(\{)",
+             bygroups(Name.Function, Whitespace, Operator, Whitespace, Punctuation),
+             "functions"),
+            # Variable Names
+            (r"([.]?[a-zA-Z][\w.]*)(\s*)([-.~=!@#$%^&*_+|,<>?/\\:']?:)",
+             bygroups(Name.Variable, Whitespace, Operator)),
+            # Functions
+            (r"\{", Punctuation, "functions"),
+            # Parentheses
+            (r"\(", Punctuation, "parentheses"),
+            # Brackets
+            (r"\[", Punctuation, "brackets"),
+            # Errors
+            (r"'`([a-zA-Z][\w.]*)?", Name.Exception),
+            # File Symbols
+            (r"`:([a-zA-Z/][\w./]*)?", String.Symbol),
+            # Symbols
+            (r"`([a-zA-Z][\w.]*)?", String.Symbol),
+            # Numbers
+            include("numbers"),
+            # Variable Names
+            (r"[a-zA-Z][\w.]*", Name),
+            # Operators
+            (r"[-=+*#$%@!~^&:.,<>'\\|/?_]", Operator),
+            # Punctuation
+            (r";", Punctuation),
+        ],
+        "functions": [
+            include("root"),
+            (r"\}", Punctuation, "#pop"),
+        ],
+        "parentheses": [
+            include("root"),
+            (r"\)", Punctuation, "#pop"),
+        ],
+        "brackets": [
+            include("root"),
+            (r"\]", Punctuation, "#pop"),
+        ],
+        "numbers": [
+            # Binary Values
+            (r"[01]+b", Number.Bin),
+            # Nulls/Infinities
+            (r"0[nNwW][cefghijmndzuvtp]?", Number),
+            # Timestamps
+            ((r"(?:[0-9]{4}[.][0-9]{2}[.][0-9]{2}|[0-9]+)"
+              "D(?:[0-9](?:[0-9](?::[0-9]{2}"
+              "(?::[0-9]{2}(?:[.][0-9]*)?)?)?)?)?"), Literal.Date),
+            # Datetimes
+            ((r"[0-9]{4}[.][0-9]{2}"
+              "(?:m|[.][0-9]{2}(?:T(?:[0-9]{2}:[0-9]{2}"
+              "(?::[0-9]{2}(?:[.][0-9]*)?)?)?)?)"), Literal.Date),
+            # Times
+            (r"[0-9]{2}:[0-9]{2}(?::[0-9]{2}(?:[.][0-9]{1,3})?)?",
+             Literal.Date),
+            # GUIDs
+            (r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+             Number.Hex),
+            # Byte Vectors
+            (r"0x[0-9a-fA-F]+", Number.Hex),
+            # Floats
+            (r"([0-9]*[.]?[0-9]+|[0-9]+[.]?[0-9]*)[eE][+-]?[0-9]+[ef]?",
+             Number.Float),
+            (r"([0-9]*[.][0-9]+|[0-9]+[.][0-9]*)[ef]?", Number.Float),
+            (r"[0-9]+[ef]", Number.Float),
+            # Characters
+            (r"[0-9]+c", Number),
+            # Integers
+            (r"[0-9]+[ihtuv]", Number.Integer),
+            # Long Integers
+            (r"[0-9]+[jnp]?", Number.Integer.Long),
+        ],
+        "comments": [
+            (r"[^\\]+", Comment.Multiline),
+            (r"^\\", Comment.Multiline, "#pop"),
+            (r"\\", Comment.Multiline),
+        ],
+        "strings": [
+            (r'[^"\\]+', String.Double),
+            (r"\\.", String.Escape),
+            (r'"', String.Double, "#pop"),
+        ],
+    }
+
+
+class QLexer(KLexer):
+    """
+    For `Q `_ source code.
+    """
+
+    name = "Q"
+    aliases = ["q"]
+    filenames = ["*.q"]
+    version_added = '2.12'
+
+    tokens = {
+        "root": [
+            (words(("aj", "aj0", "ajf", "ajf0", "all", "and", "any", "asc",
+                    "asof", "attr", "avgs", "ceiling", "cols", "count", "cross",
+                    "csv", "cut", "deltas", "desc", "differ", "distinct", "dsave",
+                    "each", "ej", "ema", "eval", "except", "fby", "fills", "first",
+                    "fkeys", "flip", "floor", "get", "group", "gtime", "hclose",
+                    "hcount", "hdel", "hsym", "iasc", "idesc", "ij", "ijf",
+                    "inter", "inv", "key", "keys", "lj", "ljf", "load", "lower",
+                    "lsq", "ltime", "ltrim", "mavg", "maxs", "mcount", "md5",
+                    "mdev", "med", "meta", "mins", "mmax", "mmin", "mmu", "mod",
+                    "msum", "neg", "next", "not", "null", "or", "over", "parse",
+                    "peach", "pj", "prds", "prior", "prev", "rand", "rank", "ratios",
+                    "raze", "read0", "read1", "reciprocal", "reval", "reverse",
+                    "rload", "rotate", "rsave", "rtrim", "save", "scan", "scov",
+                    "sdev", "set", "show", "signum", "ssr", "string", "sublist",
+                    "sums", "sv", "svar", "system", "tables", "til", "trim", "txf",
+                    "type", "uj", "ujf", "ungroup", "union", "upper", "upsert",
+                    "value", "view", "views", "vs", "where", "wj", "wj1", "ww",
+                    "xasc", "xbar", "xcol", "xcols", "xdesc", "xgroup", "xkey",
+                    "xlog", "xprev", "xrank"),
+                    suffix=r"\b"), Name.Builtin,
+            ),
+            inherit,
+        ],
+    }
diff --git a/lib/pygments/lexers/qlik.py b/lib/pygments/lexers/qlik.py
new file mode 100644
index 0000000..a29f89f
--- /dev/null
+++ b/lib/pygments/lexers/qlik.py
@@ -0,0 +1,117 @@
+"""
+    pygments.lexers.qlik
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the qlik scripting language
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+from pygments.lexers._qlik_builtins import OPERATORS_LIST, STATEMENT_LIST, \
+    SCRIPT_FUNCTIONS, CONSTANT_LIST
+
+__all__ = ["QlikLexer"]
+
+
+class QlikLexer(RegexLexer):
+    """
+    Lexer for qlik code, including .qvs files
+    """
+
+    name = "Qlik"
+    aliases = ["qlik", "qlikview", "qliksense", "qlikscript"]
+    filenames = ["*.qvs", "*.qvw"]
+    url = "https://qlik.com"
+    version_added = '2.12'
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        # Handle multi-line comments
+        "comment": [
+            (r"\*/", Comment.Multiline, "#pop"),
+            (r"[^*]+", Comment.Multiline),
+        ],
+        # Handle numbers
+        "numerics": [
+            (r"\b\d+\.\d+(e\d+)?[fd]?\b", Number.Float),
+            (r"\b\d+\b", Number.Integer),
+        ],
+        # Handle variable names in things
+        "interp": [
+            (
+                r"(\$\()(\w+)(\))",
+                bygroups(String.Interpol, Name.Variable, String.Interpol),
+            ),
+        ],
+        # Handle strings
+        "string": [
+            (r"'", String, "#pop"),
+            include("interp"),
+            (r"[^'$]+", String),
+            (r"\$", String),
+        ],
+        #
+        "assignment": [
+            (r";", Punctuation, "#pop"),
+            include("root"),
+        ],
+        "field_name_quote": [
+            (r'"', String.Symbol, "#pop"),
+            include("interp"),
+            (r"[^\"$]+", String.Symbol),
+            (r"\$", String.Symbol),
+        ],
+        "field_name_bracket": [
+            (r"\]", String.Symbol, "#pop"),
+            include("interp"),
+            (r"[^\]$]+", String.Symbol),
+            (r"\$", String.Symbol),
+        ],
+        "function": [(r"\)", Punctuation, "#pop"), include("root")],
+        "root": [
+            # Whitespace and comments
+            (r"\s+", Text.Whitespace),
+            (r"/\*", Comment.Multiline, "comment"),
+            (r"//.*\n", Comment.Single),
+            # variable assignment
+            (r"(let|set)(\s+)", bygroups(Keyword.Declaration, Text.Whitespace),
+             "assignment"),
+            # Word operators
+            (words(OPERATORS_LIST["words"], prefix=r"\b", suffix=r"\b"),
+             Operator.Word),
+            # Statements
+            (words(STATEMENT_LIST, suffix=r"\b"), Keyword),
+            # Table names
+            (r"[a-z]\w*:", Keyword.Declaration),
+            # Constants
+            (words(CONSTANT_LIST, suffix=r"\b"), Keyword.Constant),
+            # Functions
+            (words(SCRIPT_FUNCTIONS, suffix=r"(?=\s*\()"), Name.Builtin,
+             "function"),
+            # interpolation - e.g. $(variableName)
+            include("interp"),
+            # Quotes denote a field/file name
+            (r'"', String.Symbol, "field_name_quote"),
+            # Square brackets denote a field/file name
+            (r"\[", String.Symbol, "field_name_bracket"),
+            # Strings
+            (r"'", String, "string"),
+            # Numbers
+            include("numerics"),
+            # Operator symbols
+            (words(OPERATORS_LIST["symbols"]), Operator),
+            # Strings denoted by single quotes
+            (r"'.+?'", String),
+            # Words as text
+            (r"\b\w+\b", Text),
+            # Basic punctuation
+            (r"[,;.()\\/]", Punctuation),
+        ],
+    }
diff --git a/lib/pygments/lexers/qvt.py b/lib/pygments/lexers/qvt.py
new file mode 100644
index 0000000..302d1b6
--- /dev/null
+++ b/lib/pygments/lexers/qvt.py
@@ -0,0 +1,153 @@
+"""
+    pygments.lexers.qvt
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for QVT Operational language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include, combined, default, \
+    words
+from pygments.token import Text, Comment, Operator, Keyword, Punctuation, \
+    Name, String, Number
+
+__all__ = ['QVToLexer']
+
+
+class QVToLexer(RegexLexer):
+    """
+    For the QVT Operational Mapping language.
+
+    Reference for implementing this: «Meta Object Facility (MOF) 2.0
+    Query/View/Transformation Specification», Version 1.1 - January 2011
+    (https://www.omg.org/spec/QVT/1.1/), see §8.4, «Concrete Syntax» in
+    particular.
+
+    Notable tokens assignments:
+
+    - Name.Class is assigned to the identifier following any of the following
+      keywords: metamodel, class, exception, primitive, enum, transformation
+      or library
+
+    - Name.Function is assigned to the names of mappings and queries
+
+    - Name.Builtin.Pseudo is assigned to the pre-defined variables 'this',
+      'self' and 'result'.
+    """
+    # With obvious borrowings & inspiration from the Java, Python and C lexers
+
+    name = 'QVTO'
+    aliases = ['qvto', 'qvt']
+    filenames = ['*.qvto']
+    url = 'https://www.omg.org/spec/QVT/1.1'
+    version_added = ''
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'(--|//)(\s*)(directive:)?(.*)$',
+             bygroups(Comment, Comment, Comment.Preproc, Comment)),
+            # Uncomment the following if you want to distinguish between
+            # '/*' and '/**', à la javadoc
+            # (r'/[*]{2}(.|\n)*?[*]/', Comment.Multiline),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'\\\n', Text),
+            (r'(and|not|or|xor|##?)\b', Operator.Word),
+            (r'(:{1,2}=|[-+]=)\b', Operator.Word),
+            (r'(@|<<|>>)\b', Keyword),  # stereotypes
+            (r'!=|<>|==|=|!->|->|>=|<=|[.]{3}|[+/*%=<>&|.~]', Operator),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'(true|false|unlimited|null)\b', Keyword.Constant),
+            (r'(this|self|result)\b', Name.Builtin.Pseudo),
+            (r'(var)\b', Keyword.Declaration),
+            (r'(from|import)\b', Keyword.Namespace, 'fromimport'),
+            (r'(metamodel|class|exception|primitive|enum|transformation|'
+             r'library)(\s+)(\w+)',
+             bygroups(Keyword.Word, Text, Name.Class)),
+            (r'(exception)(\s+)(\w+)',
+             bygroups(Keyword.Word, Text, Name.Exception)),
+            (r'(main)\b', Name.Function),
+            (r'(mapping|helper|query)(\s+)',
+             bygroups(Keyword.Declaration, Text), 'operation'),
+            (r'(assert)(\s+)\b', bygroups(Keyword, Text), 'assert'),
+            (r'(Bag|Collection|Dict|OrderedSet|Sequence|Set|Tuple|List)\b',
+             Keyword.Type),
+            include('keywords'),
+            ('"', String, combined('stringescape', 'dqs')),
+            ("'", String, combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+            # (r'([a-zA-Z_]\w*)(::)([a-zA-Z_]\w*)',
+            # bygroups(Text, Text, Text)),
+        ],
+
+        'fromimport': [
+            (r'(?:[ \t]|\\\n)+', Text),
+            (r'[a-zA-Z_][\w.]*', Name.Namespace),
+            default('#pop'),
+        ],
+
+        'operation': [
+            (r'::', Text),
+            (r'(.*::)([a-zA-Z_]\w*)([ \t]*)(\()',
+             bygroups(Text, Name.Function, Text, Punctuation), '#pop')
+        ],
+
+        'assert': [
+            (r'(warning|error|fatal)\b', Keyword, '#pop'),
+            default('#pop'),  # all else: go back
+        ],
+
+        'keywords': [
+            (words((
+                'abstract', 'access', 'any', 'assert', 'blackbox', 'break',
+                'case', 'collect', 'collectNested', 'collectOne', 'collectselect',
+                'collectselectOne', 'composes', 'compute', 'configuration',
+                'constructor', 'continue', 'datatype', 'default', 'derived',
+                'disjuncts', 'do', 'elif', 'else', 'end', 'endif', 'except',
+                'exists', 'extends', 'forAll', 'forEach', 'forOne', 'from', 'if',
+                'implies', 'in', 'inherits', 'init', 'inout', 'intermediate',
+                'invresolve', 'invresolveIn', 'invresolveone', 'invresolveoneIn',
+                'isUnique', 'iterate', 'late', 'let', 'literal', 'log', 'map',
+                'merges', 'modeltype', 'new', 'object', 'one', 'ordered', 'out',
+                'package', 'population', 'property', 'raise', 'readonly',
+                'references', 'refines', 'reject', 'resolve', 'resolveIn',
+                'resolveone', 'resolveoneIn', 'return', 'select', 'selectOne',
+                'sortedBy', 'static', 'switch', 'tag', 'then', 'try', 'typedef',
+                'unlimited', 'uses', 'when', 'where', 'while', 'with', 'xcollect',
+                'xmap', 'xselect'), suffix=r'\b'), Keyword),
+        ],
+
+        # There is no need to distinguish between String.Single and
+        # String.Double: 'strings' is factorised for 'dqs' and 'sqs'
+        'strings': [
+            (r'[^\\\'"\n]+', String),
+            # quotes, percents and backslashes must be parsed one at a time
+            (r'[\'"\\]', String),
+        ],
+        'stringescape': [
+            (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape)
+        ],
+        'dqs': [  # double-quoted string
+            (r'"', String, '#pop'),
+            (r'\\\\|\\"', String.Escape),
+            include('strings')
+        ],
+        'sqs': [  # single-quoted string
+            (r"'", String, '#pop'),
+            (r"\\\\|\\'", String.Escape),
+            include('strings')
+        ],
+        'name': [
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+        # numbers: excerpt taken from the python lexer
+        'numbers': [
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+', Number.Float),
+            (r'\d+', Number.Integer)
+        ],
+    }
diff --git a/lib/pygments/lexers/r.py b/lib/pygments/lexers/r.py
new file mode 100644
index 0000000..d3f65ba
--- /dev/null
+++ b/lib/pygments/lexers/r.py
@@ -0,0 +1,196 @@
+"""
+    pygments.lexers.r
+    ~~~~~~~~~~~~~~~~~
+
+    Lexers for the R/S languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, do_insertions
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Generic, Whitespace
+
+__all__ = ['RConsoleLexer', 'SLexer', 'RdLexer']
+
+
+line_re  = re.compile('.*?\n')
+
+
+class RConsoleLexer(Lexer):
+    """
+    For R console transcripts or R CMD BATCH output files.
+    """
+
+    name = 'RConsole'
+    aliases = ['rconsole', 'rout']
+    filenames = ['*.Rout']
+    url = 'https://www.r-project.org'
+    version_added = ''
+    _example = "rconsole/r-console-transcript.Rout"
+
+    def get_tokens_unprocessed(self, text):
+        slexer = SLexer(**self.options)
+
+        current_code_block = ''
+        insertions = []
+
+        for match in line_re.finditer(text):
+            line = match.group()
+            if line.startswith('>') or line.startswith('+'):
+                # Colorize the prompt as such,
+                # then put rest of line into current_code_block
+                insertions.append((len(current_code_block),
+                                   [(0, Generic.Prompt, line[:2])]))
+                current_code_block += line[2:]
+            else:
+                # We have reached a non-prompt line!
+                # If we have stored prompt lines, need to process them first.
+                if current_code_block:
+                    # Weave together the prompts and highlight code.
+                    yield from do_insertions(
+                        insertions, slexer.get_tokens_unprocessed(current_code_block))
+                    # Reset vars for next code block.
+                    current_code_block = ''
+                    insertions = []
+                # Now process the actual line itself, this is output from R.
+                yield match.start(), Generic.Output, line
+
+        # If we happen to end on a code block with nothing after it, need to
+        # process the last code block. This is neither elegant nor DRY so
+        # should be changed.
+        if current_code_block:
+            yield from do_insertions(
+                insertions, slexer.get_tokens_unprocessed(current_code_block))
+
+
+class SLexer(RegexLexer):
+    """
+    For S, S-plus, and R source code.
+    """
+
+    name = 'S'
+    aliases = ['splus', 's', 'r']
+    filenames = ['*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron']
+    mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r',
+                 'text/x-R', 'text/x-r-history', 'text/x-r-profile']
+    url = 'https://www.r-project.org'
+    version_added = '0.10'
+
+    valid_name = r'`[^`\\]*(?:\\.[^`\\]*)*`|(?:[a-zA-Z]|\.[A-Za-z_.])[\w.]*|\.'
+    tokens = {
+        'comments': [
+            (r'#.*$', Comment.Single),
+        ],
+        'valid_name': [
+            (valid_name, Name),
+        ],
+        'function_name': [
+            (rf'({valid_name})\s*(?=\()', Name.Function),
+        ],
+        'punctuation': [
+            (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation),
+        ],
+        'keywords': [
+            (r'(if|else|for|while|repeat|in|next|break|return|switch|function)'
+             r'(?![\w.])',
+             Keyword.Reserved),
+        ],
+        'operators': [
+            (r'<>?|-|==|<=|>=|\|>|<|>|&&?|!=|\|\|?|\?', Operator),
+            (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator),
+        ],
+        'builtin_symbols': [
+            (r'(NULL|NA(_(integer|real|complex|character)_)?|'
+             r'letters|LETTERS|Inf|TRUE|FALSE|NaN|pi|\.\.(\.|[0-9]+))'
+             r'(?![\w.])',
+             Keyword.Constant),
+            (r'(T|F)\b', Name.Builtin.Pseudo),
+        ],
+        'numbers': [
+            # hex number
+            (r'0[xX][a-fA-F0-9]+([pP][0-9]+)?[Li]?', Number.Hex),
+            # decimal number
+            (r'[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[Li]?',
+             Number),
+        ],
+        'statements': [
+            include('comments'),
+            # whitespaces
+            (r'\s+', Whitespace),
+            (r'\'', String, 'string_squote'),
+            (r'\"', String, 'string_dquote'),
+            include('builtin_symbols'),
+            include('keywords'),
+            include('function_name'),
+            include('valid_name'),
+            include('numbers'),
+            include('punctuation'),
+            include('operators'),
+        ],
+        'root': [
+            # calls:
+            include('statements'),
+            # blocks:
+            (r'\{|\}', Punctuation),
+            # (r'\{', Punctuation, 'block'),
+            (r'.', Text),
+        ],
+        # 'block': [
+        #    include('statements'),
+        #    ('\{', Punctuation, '#push'),
+        #    ('\}', Punctuation, '#pop')
+        # ],
+        'string_squote': [
+            (r'([^\'\\]|\\.)*\'', String, '#pop'),
+        ],
+        'string_dquote': [
+            (r'([^"\\]|\\.)*"', String, '#pop'),
+        ],
+    }
+
+    def analyse_text(text):
+        if re.search(r'[a-z0-9_\])\s]<-(?!-)', text):
+            return 0.11
+
+
+class RdLexer(RegexLexer):
+    """
+    Pygments Lexer for R documentation (Rd) files
+
+    This is a very minimal implementation, highlighting little more
+    than the macros. A description of Rd syntax is found in `Writing R
+    Extensions `_
+    and `Parsing Rd files `_.
+    """
+    name = 'Rd'
+    aliases = ['rd']
+    filenames = ['*.Rd']
+    mimetypes = ['text/x-r-doc']
+    url = 'http://cran.r-project.org/doc/manuals/R-exts.html'
+    version_added = '1.6'
+
+    # To account for verbatim / LaTeX-like / and R-like areas
+    # would require parsing.
+    tokens = {
+        'root': [
+            # catch escaped brackets and percent sign
+            (r'\\[\\{}%]', String.Escape),
+            # comments
+            (r'%.*$', Comment),
+            # special macros with no arguments
+            (r'\\(?:cr|l?dots|R|tab)\b', Keyword.Constant),
+            # macros
+            (r'\\[a-zA-Z]+\b', Keyword),
+            # special preprocessor macros
+            (r'^\s*#(?:ifn?def|endif).*\b', Comment.Preproc),
+            # non-escaped brackets
+            (r'[{}]', Name.Builtin),
+            # everything else
+            (r'[^\\%\n{}]+', Text),
+            (r'.', Text),
+        ]
+    }
diff --git a/lib/pygments/lexers/rdf.py b/lib/pygments/lexers/rdf.py
new file mode 100644
index 0000000..4930c1b
--- /dev/null
+++ b/lib/pygments/lexers/rdf.py
@@ -0,0 +1,468 @@
+"""
+    pygments.lexers.rdf
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for semantic web and RDF query languages and markup.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default
+from pygments.token import Keyword, Punctuation, String, Number, Operator, \
+    Generic, Whitespace, Name, Literal, Comment, Text
+
+__all__ = ['SparqlLexer', 'TurtleLexer', 'ShExCLexer']
+
+
+class SparqlLexer(RegexLexer):
+    """
+    Lexer for SPARQL query language.
+    """
+    name = 'SPARQL'
+    aliases = ['sparql']
+    filenames = ['*.rq', '*.sparql']
+    mimetypes = ['application/sparql-query']
+    url = 'https://www.w3.org/TR/sparql11-query'
+    version_added = '2.0'
+
+    # character group definitions ::
+
+    PN_CHARS_BASE_GRP = ('a-zA-Z'
+                         '\u00c0-\u00d6'
+                         '\u00d8-\u00f6'
+                         '\u00f8-\u02ff'
+                         '\u0370-\u037d'
+                         '\u037f-\u1fff'
+                         '\u200c-\u200d'
+                         '\u2070-\u218f'
+                         '\u2c00-\u2fef'
+                         '\u3001-\ud7ff'
+                         '\uf900-\ufdcf'
+                         '\ufdf0-\ufffd')
+
+    PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+    PN_CHARS_GRP = (PN_CHARS_U_GRP +
+                    r'\-' +
+                    r'0-9' +
+                    '\u00b7' +
+                    '\u0300-\u036f' +
+                    '\u203f-\u2040')
+
+    HEX_GRP = '0-9A-Fa-f'
+
+    PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%'
+
+    # terminal productions ::
+
+    PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+    PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']'
+
+    PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+    HEX = '[' + HEX_GRP + ']'
+
+    PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+    IRIREF = r'<(?:[^<>"{}|^`\\\x00-\x20])*>'
+
+    BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \
+                       '.]*' + PN_CHARS + ')?'
+
+    PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+    VARNAME = '[0-9' + PN_CHARS_U_GRP + '][' + PN_CHARS_U_GRP + \
+              '0-9\u00b7\u0300-\u036f\u203f-\u2040]*'
+
+    PERCENT = '%' + HEX + HEX
+
+    PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+    PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+    PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+                '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+                PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+    EXPONENT = r'[eE][+-]?\d+'
+
+    # Lexer token definitions ::
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            # keywords ::
+            (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|'
+             r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|'
+             r'offset|values|bindings|load|into|clear|drop|create|add|move|copy|'
+             r'insert\s+data|delete\s+data|delete\s+where|with|delete|insert|'
+             r'using\s+named|using|graph|default|named|all|optional|service|'
+             r'silent|bind|undef|union|not\s+in|in|as|having|to|prefix|base)\b', Keyword),
+            (r'(a)\b', Keyword),
+            # IRIs ::
+            ('(' + IRIREF + ')', Name.Label),
+            # blank nodes ::
+            ('(' + BLANK_NODE_LABEL + ')', Name.Label),
+            #  # variables ::
+            ('[?$]' + VARNAME, Name.Variable),
+            # prefixed names ::
+            (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+            # function names ::
+            (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|'
+             r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|'
+             r'contains|strstarts|strends|strbefore|strafter|year|month|day|'
+             r'hours|minutes|seconds|timezone|tz|now|uuid|struuid|md5|sha1|sha256|sha384|'
+             r'sha512|coalesce|if|strlang|strdt|sameterm|isiri|isuri|isblank|'
+             r'isliteral|isnumeric|regex|substr|replace|exists|not\s+exists|'
+             r'count|sum|min|max|avg|sample|group_concat|separator)\b',
+             Name.Function),
+            # boolean literals ::
+            (r'(true|false)', Keyword.Constant),
+            # double literals ::
+            (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float),
+            # decimal literals ::
+            (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float),
+            # integer literals ::
+            (r'[+\-]?\d+', Number.Integer),
+            # operators ::
+            (r'(\|\||&&|=|\*|\-|\+|/|!=|<=|>=|!|<|>)', Operator),
+            # punctuation characters ::
+            (r'[(){}.;,:^\[\]]', Punctuation),
+            # line comments ::
+            (r'#[^\n]*', Comment),
+            # strings ::
+            (r'"""', String, 'triple-double-quoted-string'),
+            (r'"', String, 'single-double-quoted-string'),
+            (r"'''", String, 'triple-single-quoted-string'),
+            (r"'", String, 'single-single-quoted-string'),
+        ],
+        'triple-double-quoted-string': [
+            (r'"""', String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-double-quoted-string': [
+            (r'"', String, 'end-of-string'),
+            (r'[^"\\\n]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'triple-single-quoted-string': [
+            (r"'''", String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String.Escape, 'string-escape'),
+        ],
+        'single-single-quoted-string': [
+            (r"'", String, 'end-of-string'),
+            (r"[^'\\\n]+", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'string-escape': [
+            (r'u' + HEX + '{4}', String.Escape, '#pop'),
+            (r'U' + HEX + '{8}', String.Escape, '#pop'),
+            (r'.', String.Escape, '#pop'),
+        ],
+        'end-of-string': [
+            (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+             bygroups(Operator, Name.Function), '#pop:2'),
+            (r'\^\^', Operator, '#pop:2'),
+            default('#pop:2'),
+        ],
+    }
+
+
+class TurtleLexer(RegexLexer):
+    """
+    Lexer for Turtle data language.
+    """
+    name = 'Turtle'
+    aliases = ['turtle']
+    filenames = ['*.ttl']
+    mimetypes = ['text/turtle', 'application/x-turtle']
+    url = 'https://www.w3.org/TR/turtle'
+    version_added = '2.1'
+
+    # character group definitions ::
+    PN_CHARS_BASE_GRP = ('a-zA-Z'
+                         '\u00c0-\u00d6'
+                         '\u00d8-\u00f6'
+                         '\u00f8-\u02ff'
+                         '\u0370-\u037d'
+                         '\u037f-\u1fff'
+                         '\u200c-\u200d'
+                         '\u2070-\u218f'
+                         '\u2c00-\u2fef'
+                         '\u3001-\ud7ff'
+                         '\uf900-\ufdcf'
+                         '\ufdf0-\ufffd')
+
+    PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+    PN_CHARS_GRP = (PN_CHARS_U_GRP +
+                    r'\-' +
+                    r'0-9' +
+                    '\u00b7' +
+                    '\u0300-\u036f' +
+                    '\u203f-\u2040')
+
+    PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+    PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+    PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+    HEX_GRP = '0-9A-Fa-f'
+
+    HEX = '[' + HEX_GRP + ']'
+
+    PERCENT = '%' + HEX + HEX
+
+    PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%'
+
+    PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+    PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+    PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+    PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+                '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+                PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+    patterns = {
+        'PNAME_NS': r'((?:[a-zA-Z][\w-]*)?\:)',  # Simplified character range
+        'IRIREF': r'(<[^<>"{}|^`\\\x00-\x20]*>)'
+    }
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+
+            # Base / prefix
+            (r'(@base|BASE)(\s+){IRIREF}(\s*)(\.?)'.format(**patterns),
+             bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
+                      Punctuation)),
+            (r'(@prefix|PREFIX)(\s+){PNAME_NS}(\s+){IRIREF}(\s*)(\.?)'.format(**patterns),
+             bygroups(Keyword, Whitespace, Name.Namespace, Whitespace,
+                      Name.Variable, Whitespace, Punctuation)),
+
+            # The shorthand predicate 'a'
+            (r'(?<=\s)a(?=\s)', Keyword.Type),
+
+            # IRIREF
+            (r'{IRIREF}'.format(**patterns), Name.Variable),
+
+            # PrefixedName
+            (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+
+            # BlankNodeLabel
+            (r'(_)(:)([' + PN_CHARS_U_GRP + r'0-9]([' + PN_CHARS_GRP + r'.]*' + PN_CHARS + ')?)',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+
+            # Comment
+            (r'#[^\n]+', Comment),
+
+            (r'\b(true|false)\b', Literal),
+            (r'[+\-]?\d*\.\d+', Number.Float),
+            (r'[+\-]?\d*(:?\.\d+)?E[+\-]?\d+', Number.Float),
+            (r'[+\-]?\d+', Number.Integer),
+            (r'[\[\](){}.;,:^]', Punctuation),
+
+            (r'"""', String, 'triple-double-quoted-string'),
+            (r'"', String, 'single-double-quoted-string'),
+            (r"'''", String, 'triple-single-quoted-string'),
+            (r"'", String, 'single-single-quoted-string'),
+        ],
+        'triple-double-quoted-string': [
+            (r'"""', String, 'end-of-string'),
+            (r'[^\\]+(?=""")', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-double-quoted-string': [
+            (r'"', String, 'end-of-string'),
+            (r'[^"\\\n]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'triple-single-quoted-string': [
+            (r"'''", String, 'end-of-string'),
+            (r"[^\\]+(?=''')", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-single-quoted-string': [
+            (r"'", String, 'end-of-string'),
+            (r"[^'\\\n]+", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'string-escape': [
+            (r'.', String, '#pop'),
+        ],
+        'end-of-string': [
+            (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+             bygroups(Operator, Generic.Emph), '#pop:2'),
+
+            (r'(\^\^){IRIREF}'.format(**patterns), bygroups(Operator, Generic.Emph), '#pop:2'),
+
+            default('#pop:2'),
+
+        ],
+    }
+
+    # Turtle and Tera Term macro files share the same file extension
+    # but each has a recognizable and distinct syntax.
+    def analyse_text(text):
+        for t in ('@base ', 'BASE ', '@prefix ', 'PREFIX '):
+            if re.search(rf'^\s*{t}', text):
+                return 0.80
+
+
+class ShExCLexer(RegexLexer):
+    """
+    Lexer for ShExC shape expressions language syntax.
+    """
+    name = 'ShExC'
+    aliases = ['shexc', 'shex']
+    filenames = ['*.shex']
+    mimetypes = ['text/shex']
+    url = 'https://shex.io/shex-semantics/#shexc'
+    version_added = ''
+
+    # character group definitions ::
+
+    PN_CHARS_BASE_GRP = ('a-zA-Z'
+                         '\u00c0-\u00d6'
+                         '\u00d8-\u00f6'
+                         '\u00f8-\u02ff'
+                         '\u0370-\u037d'
+                         '\u037f-\u1fff'
+                         '\u200c-\u200d'
+                         '\u2070-\u218f'
+                         '\u2c00-\u2fef'
+                         '\u3001-\ud7ff'
+                         '\uf900-\ufdcf'
+                         '\ufdf0-\ufffd')
+
+    PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+    PN_CHARS_GRP = (PN_CHARS_U_GRP +
+                    r'\-' +
+                    r'0-9' +
+                    '\u00b7' +
+                    '\u0300-\u036f' +
+                    '\u203f-\u2040')
+
+    HEX_GRP = '0-9A-Fa-f'
+
+    PN_LOCAL_ESC_CHARS_GRP = r"_~.\-!$&'()*+,;=/?#@%"
+
+    # terminal productions ::
+
+    PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+    PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']'
+
+    PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+    HEX = '[' + HEX_GRP + ']'
+
+    PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+    UCHAR_NO_BACKSLASH = '(?:u' + HEX + '{4}|U' + HEX + '{8})'
+
+    UCHAR = r'\\' + UCHAR_NO_BACKSLASH
+
+    IRIREF = r'<(?:[^\x00-\x20<>"{}|^`\\]|' + UCHAR + ')*>'
+
+    BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \
+                       '.]*' + PN_CHARS + ')?'
+
+    PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+    PERCENT = '%' + HEX + HEX
+
+    PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+    PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+    PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+                '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+                PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+    EXPONENT = r'[eE][+-]?\d+'
+
+    # Lexer token definitions ::
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            # keywords ::
+            (r'(?i)(base|prefix|start|external|'
+             r'literal|iri|bnode|nonliteral|length|minlength|maxlength|'
+             r'mininclusive|minexclusive|maxinclusive|maxexclusive|'
+             r'totaldigits|fractiondigits|'
+             r'closed|extra)\b', Keyword),
+            (r'(a)\b', Keyword),
+            # IRIs ::
+            ('(' + IRIREF + ')', Name.Label),
+            # blank nodes ::
+            ('(' + BLANK_NODE_LABEL + ')', Name.Label),
+            # prefixed names ::
+            (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + ')?',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+            # boolean literals ::
+            (r'(true|false)', Keyword.Constant),
+            # double literals ::
+            (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float),
+            # decimal literals ::
+            (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float),
+            # integer literals ::
+            (r'[+\-]?\d+', Number.Integer),
+            # operators ::
+            (r'[@|$&=*+?^\-~]', Operator),
+            # operator keywords ::
+            (r'(?i)(and|or|not)\b', Operator.Word),
+            # punctuation characters ::
+            (r'[(){}.;,:^\[\]]', Punctuation),
+            # line comments ::
+            (r'#[^\n]*', Comment),
+            # strings ::
+            (r'"""', String, 'triple-double-quoted-string'),
+            (r'"', String, 'single-double-quoted-string'),
+            (r"'''", String, 'triple-single-quoted-string'),
+            (r"'", String, 'single-single-quoted-string'),
+        ],
+        'triple-double-quoted-string': [
+            (r'"""', String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-double-quoted-string': [
+            (r'"', String, 'end-of-string'),
+            (r'[^"\\\n]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'triple-single-quoted-string': [
+            (r"'''", String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String.Escape, 'string-escape'),
+        ],
+        'single-single-quoted-string': [
+            (r"'", String, 'end-of-string'),
+            (r"[^'\\\n]+", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'string-escape': [
+            (UCHAR_NO_BACKSLASH, String.Escape, '#pop'),
+            (r'.', String.Escape, '#pop'),
+        ],
+        'end-of-string': [
+            (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+             bygroups(Operator, Name.Function), '#pop:2'),
+            (r'\^\^', Operator, '#pop:2'),
+            default('#pop:2'),
+        ],
+    }
diff --git a/lib/pygments/lexers/rebol.py b/lib/pygments/lexers/rebol.py
new file mode 100644
index 0000000..4b37a74
--- /dev/null
+++ b/lib/pygments/lexers/rebol.py
@@ -0,0 +1,419 @@
+"""
+    pygments.lexers.rebol
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the REBOL and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Generic, Whitespace
+
+__all__ = ['RebolLexer', 'RedLexer']
+
+
+class RebolLexer(RegexLexer):
+    """
+    A REBOL lexer.
+    """
+    name = 'REBOL'
+    aliases = ['rebol']
+    filenames = ['*.r', '*.r3', '*.reb']
+    mimetypes = ['text/x-rebol']
+    url = 'http://www.rebol.com'
+    version_added = '1.1'
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)'
+
+    def word_callback(lexer, match):
+        word = match.group()
+
+        if re.match(".*:$", word):
+            yield match.start(), Generic.Subheading, word
+        elif re.match(
+            r'(native|alias|all|any|as-string|as-binary|bind|bound\?|case|'
+            r'catch|checksum|comment|debase|dehex|exclude|difference|disarm|'
+            r'either|else|enbase|foreach|remove-each|form|free|get|get-env|if|'
+            r'in|intersect|loop|minimum-of|maximum-of|mold|new-line|'
+            r'new-line\?|not|now|prin|print|reduce|compose|construct|repeat|'
+            r'reverse|save|script\?|set|shift|switch|throw|to-hex|trace|try|'
+            r'type\?|union|unique|unless|unprotect|unset|until|use|value\?|'
+            r'while|compress|decompress|secure|open|close|read|read-io|'
+            r'write-io|write|update|query|wait|input\?|exp|log-10|log-2|'
+            r'log-e|square-root|cosine|sine|tangent|arccosine|arcsine|'
+            r'arctangent|protect|lowercase|uppercase|entab|detab|connected\?|'
+            r'browse|launch|stats|get-modes|set-modes|to-local-file|'
+            r'to-rebol-file|encloak|decloak|create-link|do-browser|bind\?|'
+            r'hide|draw|show|size-text|textinfo|offset-to-caret|'
+            r'caret-to-offset|local-request-file|rgb-to-hsv|hsv-to-rgb|'
+            r'crypt-strength\?|dh-make-key|dh-generate-key|dh-compute-key|'
+            r'dsa-make-key|dsa-generate-key|dsa-make-signature|'
+            r'dsa-verify-signature|rsa-make-key|rsa-generate-key|'
+            r'rsa-encrypt)$', word):
+            yield match.start(), Name.Builtin, word
+        elif re.match(
+            r'(add|subtract|multiply|divide|remainder|power|and~|or~|xor~|'
+            r'minimum|maximum|negate|complement|absolute|random|head|tail|'
+            r'next|back|skip|at|pick|first|second|third|fourth|fifth|sixth|'
+            r'seventh|eighth|ninth|tenth|last|path|find|select|make|to|copy\*|'
+            r'insert|remove|change|poke|clear|trim|sort|min|max|abs|cp|'
+            r'copy)$', word):
+            yield match.start(), Name.Function, word
+        elif re.match(
+            r'(error|source|input|license|help|install|echo|Usage|with|func|'
+            r'throw-on-error|function|does|has|context|probe|\?\?|as-pair|'
+            r'mod|modulo|round|repend|about|set-net|append|join|rejoin|reform|'
+            r'remold|charset|array|replace|move|extract|forskip|forall|alter|'
+            r'first+|also|take|for|forever|dispatch|attempt|what-dir|'
+            r'change-dir|clean-path|list-dir|dirize|rename|split-path|delete|'
+            r'make-dir|delete-dir|in-dir|confirm|dump-obj|upgrade|what|'
+            r'build-tag|process-source|build-markup|decode-cgi|read-cgi|'
+            r'write-user|save-user|set-user-name|protect-system|parse-xml|'
+            r'cvs-date|cvs-version|do-boot|get-net-info|desktop|layout|'
+            r'scroll-para|get-face|alert|set-face|uninstall|unfocus|'
+            r'request-dir|center-face|do-events|net-error|decode-url|'
+            r'parse-header|parse-header-date|parse-email-addrs|import-email|'
+            r'send|build-attach-body|resend|show-popup|hide-popup|open-events|'
+            r'find-key-face|do-face|viewtop|confine|find-window|'
+            r'insert-event-func|remove-event-func|inform|dump-pane|dump-face|'
+            r'flag-face|deflag-face|clear-fields|read-net|vbug|path-thru|'
+            r'read-thru|load-thru|do-thru|launch-thru|load-image|'
+            r'request-download|do-face-alt|set-font|set-para|get-style|'
+            r'set-style|make-face|stylize|choose|hilight-text|hilight-all|'
+            r'unlight-text|focus|scroll-drag|clear-face|reset-face|scroll-face|'
+            r'resize-face|load-stock|load-stock-block|notify|request|flash|'
+            r'request-color|request-pass|request-text|request-list|'
+            r'request-date|request-file|dbug|editor|link-relative-path|'
+            r'emailer|parse-error)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match(
+            r'(halt|quit|do|load|q|recycle|call|run|ask|parse|view|unview|'
+            r'return|exit|break)$', word):
+            yield match.start(), Name.Exception, word
+        elif re.match('REBOL$', word):
+            yield match.start(), Generic.Heading, word
+        elif re.match("to-.*", word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$',
+                      word):
+            yield match.start(), Operator, word
+        elif re.match(r".*\?$", word):
+            yield match.start(), Keyword, word
+        elif re.match(r".*\!$", word):
+            yield match.start(), Keyword.Type, word
+        elif re.match("'.*", word):
+            yield match.start(), Name.Variable.Instance, word  # lit-word
+        elif re.match("#.*", word):
+            yield match.start(), Name.Label, word  # issue
+        elif re.match("%.*", word):
+            yield match.start(), Name.Decorator, word  # file
+        else:
+            yield match.start(), Name.Variable, word
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'#"', String.Char, 'char'),
+            (r'#\{[0-9a-f]*\}', Number.Hex),
+            (r'2#\{', Number.Hex, 'bin2'),
+            (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex),
+            (r'"', String, 'string'),
+            (r'\{', String, 'string2'),
+            (r';#+.*\n', Comment.Special),
+            (r';\*+.*\n', Comment.Preproc),
+            (r';.*\n', Comment),
+            (r'%"', Name.Decorator, 'stringFile'),
+            (r'%[^(^{")\s\[\]]+', Name.Decorator),
+            (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float),  # money
+            (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other),    # time
+            (r'\d+[\-/][0-9a-z]+[\-/]\d+(\/\d+\:\d+((\:\d+)?'
+             r'([.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other),   # date
+            (r'\d+(\.\d+)+\.\d+', Keyword.Constant),             # tuple
+            (r'\d+X\d+', Keyword.Constant),                   # pair
+            (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float),
+            (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float),
+            (r'[+-]?\d+(\'\d+)?', Number),
+            (r'[\[\]()]', Generic.Strong),
+            (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator),  # url
+            (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),  # url
+            (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),         # email
+            (r'comment\s"', Comment, 'commentString1'),
+            (r'comment\s\{', Comment, 'commentString2'),
+            (r'comment\s\[', Comment, 'commentBlock'),
+            (r'comment\s[^(\s{"\[]+', Comment),
+            (r'/[^(^{")\s/[\]]*', Name.Attribute),
+            (r'([^(^{")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+            (r'<[\w:.-]*>', Name.Tag),
+            (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+            (r'([^(^{")\s]+)', Text),
+        ],
+        'string': [
+            (r'[^(^")]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'"', String, '#pop'),
+        ],
+        'string2': [
+            (r'[^(^{})]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'\{', String, '#push'),
+            (r'\}', String, '#pop'),
+        ],
+        'stringFile': [
+            (r'[^(^")]+', Name.Decorator),
+            (escape_re, Name.Decorator),
+            (r'\^.', Name.Decorator),
+            (r'"', Name.Decorator, '#pop'),
+        ],
+        'char': [
+            (escape_re + '"', String.Char, '#pop'),
+            (r'\^."', String.Char, '#pop'),
+            (r'."', String.Char, '#pop'),
+        ],
+        'tag': [
+            (escape_re, Name.Tag),
+            (r'"', Name.Tag, 'tagString'),
+            (r'[^(<>\r\n")]+', Name.Tag),
+            (r'>', Name.Tag, '#pop'),
+        ],
+        'tagString': [
+            (r'[^(^")]+', Name.Tag),
+            (escape_re, Name.Tag),
+            (r'[(|)]+', Name.Tag),
+            (r'\^.', Name.Tag),
+            (r'"', Name.Tag, '#pop'),
+        ],
+        'tuple': [
+            (r'(\d+\.)+', Keyword.Constant),
+            (r'\d+', Keyword.Constant, '#pop'),
+        ],
+        'bin2': [
+            (r'\s+', Number.Hex),
+            (r'([01]\s*){8}', Number.Hex),
+            (r'\}', Number.Hex, '#pop'),
+        ],
+        'commentString1': [
+            (r'[^(^")]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'"', Comment, '#pop'),
+        ],
+        'commentString2': [
+            (r'[^(^{})]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'\{', Comment, '#push'),
+            (r'\}', Comment, '#pop'),
+        ],
+        'commentBlock': [
+            (r'\[', Comment, '#push'),
+            (r'\]', Comment, '#pop'),
+            (r'"', Comment, "commentString1"),
+            (r'\{', Comment, "commentString2"),
+            (r'[^(\[\]"{)]+', Comment),
+        ],
+    }
+
+    def analyse_text(text):
+        """
+        Check if code contains REBOL header and so it probably not R code
+        """
+        if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE):
+            # The code starts with REBOL header
+            return 1.0
+        elif re.search(r'\s*REBOL\s*\[', text, re.IGNORECASE):
+            # The code contains REBOL header but also some text before it
+            return 0.5
+
+
+class RedLexer(RegexLexer):
+    """
+    A Red-language lexer.
+    """
+    name = 'Red'
+    aliases = ['red', 'red/system']
+    filenames = ['*.red', '*.reds']
+    mimetypes = ['text/x-red', 'text/x-red-system']
+    url = 'https://www.red-lang.org'
+    version_added = '2.0'
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)'
+
+    def word_callback(lexer, match):
+        word = match.group()
+
+        if re.match(".*:$", word):
+            yield match.start(), Generic.Subheading, word
+        elif re.match(r'(if|unless|either|any|all|while|until|loop|repeat|'
+                      r'foreach|forall|func|function|does|has|switch|'
+                      r'case|reduce|compose|get|set|print|prin|equal\?|'
+                      r'not-equal\?|strict-equal\?|lesser\?|greater\?|lesser-or-equal\?|'
+                      r'greater-or-equal\?|same\?|not|type\?|stats|'
+                      r'bind|union|replace|charset|routine)$', word):
+            yield match.start(), Name.Builtin, word
+        elif re.match(r'(make|random|reflect|to|form|mold|absolute|add|divide|multiply|negate|'
+                      r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|'
+                      r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|'
+                      r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|'
+                      r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|'
+                      r'update|write)$', word):
+            yield match.start(), Name.Function, word
+        elif re.match(r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|'
+                      r'none|crlf|dot|null-byte)$', word):
+            yield match.start(), Name.Builtin.Pseudo, word
+        elif re.match(r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|'
+                      r'#switch|#default|#get-definition)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match(r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|'
+                      r'raise-error|return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|'
+                      r'quote|forever)$', word):
+            yield match.start(), Name.Exception, word
+        elif re.match(r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|'
+                      r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|'
+                      r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|'
+                      r'any-struct\?|none\?|word\?|any-series\?)$', word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match("to-.*", word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|'
+                      r'<<<|>>>|<<|>>|<|>%)$', word):
+            yield match.start(), Operator, word
+        elif re.match(r".*\!$", word):
+            yield match.start(), Keyword.Type, word
+        elif re.match("'.*", word):
+            yield match.start(), Name.Variable.Instance, word  # lit-word
+        elif re.match("#.*", word):
+            yield match.start(), Name.Label, word  # issue
+        elif re.match("%.*", word):
+            yield match.start(), Name.Decorator, word  # file
+        elif re.match(":.*", word):
+            yield match.start(), Generic.Subheading, word  # get-word
+        else:
+            yield match.start(), Name.Variable, word
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'#"', String.Char, 'char'),
+            (r'#\{[0-9a-f\s]*\}', Number.Hex),
+            (r'2#\{', Number.Hex, 'bin2'),
+            (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex),
+            (r'([0-9a-f]+)(h)((\s)|(?=[\[\]{}"()]))',
+             bygroups(Number.Hex, Name.Variable, Whitespace)),
+            (r'"', String, 'string'),
+            (r'\{', String, 'string2'),
+            (r';#+.*\n', Comment.Special),
+            (r';\*+.*\n', Comment.Preproc),
+            (r';.*\n', Comment),
+            (r'%"', Name.Decorator, 'stringFile'),
+            (r'%[^(^{")\s\[\]]+', Name.Decorator),
+            (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float),  # money
+            (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other),    # time
+            (r'\d+[\-/][0-9a-z]+[\-/]\d+(/\d+:\d+((:\d+)?'
+             r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other),   # date
+            (r'\d+(\.\d+)+\.\d+', Keyword.Constant),             # tuple
+            (r'\d+X\d+', Keyword.Constant),                   # pair
+            (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float),
+            (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float),
+            (r'[+-]?\d+(\'\d+)?', Number),
+            (r'[\[\]()]', Generic.Strong),
+            (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator),  # url
+            (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),  # url
+            (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),         # email
+            (r'comment\s"', Comment, 'commentString1'),
+            (r'comment\s\{', Comment, 'commentString2'),
+            (r'comment\s\[', Comment, 'commentBlock'),
+            (r'comment\s[^(\s{"\[]+', Comment),
+            (r'/[^(^{^")\s/[\]]*', Name.Attribute),
+            (r'([^(^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+            (r'<[\w:.-]*>', Name.Tag),
+            (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+            (r'([^(^{")\s]+)', Text),
+        ],
+        'string': [
+            (r'[^(^")]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'"', String, '#pop'),
+        ],
+        'string2': [
+            (r'[^(^{})]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'\{', String, '#push'),
+            (r'\}', String, '#pop'),
+        ],
+        'stringFile': [
+            (r'[^(^")]+', Name.Decorator),
+            (escape_re, Name.Decorator),
+            (r'\^.', Name.Decorator),
+            (r'"', Name.Decorator, '#pop'),
+        ],
+        'char': [
+            (escape_re + '"', String.Char, '#pop'),
+            (r'\^."', String.Char, '#pop'),
+            (r'."', String.Char, '#pop'),
+        ],
+        'tag': [
+            (escape_re, Name.Tag),
+            (r'"', Name.Tag, 'tagString'),
+            (r'[^(<>\r\n")]+', Name.Tag),
+            (r'>', Name.Tag, '#pop'),
+        ],
+        'tagString': [
+            (r'[^(^")]+', Name.Tag),
+            (escape_re, Name.Tag),
+            (r'[(|)]+', Name.Tag),
+            (r'\^.', Name.Tag),
+            (r'"', Name.Tag, '#pop'),
+        ],
+        'tuple': [
+            (r'(\d+\.)+', Keyword.Constant),
+            (r'\d+', Keyword.Constant, '#pop'),
+        ],
+        'bin2': [
+            (r'\s+', Number.Hex),
+            (r'([01]\s*){8}', Number.Hex),
+            (r'\}', Number.Hex, '#pop'),
+        ],
+        'commentString1': [
+            (r'[^(^")]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'"', Comment, '#pop'),
+        ],
+        'commentString2': [
+            (r'[^(^{})]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'\{', Comment, '#push'),
+            (r'\}', Comment, '#pop'),
+        ],
+        'commentBlock': [
+            (r'\[', Comment, '#push'),
+            (r'\]', Comment, '#pop'),
+            (r'"', Comment, "commentString1"),
+            (r'\{', Comment, "commentString2"),
+            (r'[^(\[\]"{)]+', Comment),
+        ],
+    }
diff --git a/lib/pygments/lexers/rego.py b/lib/pygments/lexers/rego.py
new file mode 100644
index 0000000..6f2e3e9
--- /dev/null
+++ b/lib/pygments/lexers/rego.py
@@ -0,0 +1,57 @@
+"""
+    pygments.lexers.rego
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Rego policy languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Operator, Keyword, Name, String, Number, Punctuation, Whitespace
+
+class RegoLexer(RegexLexer):
+    """
+    For Rego source.
+    """
+    name = 'Rego'
+    url = 'https://www.openpolicyagent.org/docs/latest/policy-language/'
+    filenames = ['*.rego']
+    aliases = ['rego']
+    mimetypes = ['text/x-rego']
+    version_added = '2.19'
+
+    reserved_words = (
+        'as', 'contains', 'data', 'default', 'else', 'every', 'false',
+        'if', 'in', 'import', 'package', 'not', 'null',
+        'some', 'true', 'with'
+    )
+
+    builtins = (
+        # https://www.openpolicyagent.org/docs/latest/philosophy/#the-opa-document-model
+        'data',  # Global variable for accessing base and virtual documents
+        'input', # Represents synchronously pushed base documents
+    )
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'#.*?$', Comment.Single),
+            (words(reserved_words, suffix=r'\b'), Keyword),
+            (words(builtins, suffix=r'\b'), Name.Builtin),
+            (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r'`[^`]*`', String.Backtick),
+            (r'-?\d+(\.\d+)?', Number),
+            (r'(==|!=|<=|>=|:=)', Operator),  # Compound operators
+            (r'[=<>+\-*/%&|]', Operator),     # Single-character operators
+            (r'[\[\]{}(),.:;]', Punctuation),
+        ]
+    }
+
+__all__ = ['RegoLexer']
+
+
+
diff --git a/lib/pygments/lexers/resource.py b/lib/pygments/lexers/resource.py
new file mode 100644
index 0000000..9593c21
--- /dev/null
+++ b/lib/pygments/lexers/resource.py
@@ -0,0 +1,83 @@
+"""
+    pygments.lexers.resource
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for resource definition files.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Comment, String, Number, Operator, Text, \
+    Keyword, Name
+
+__all__ = ['ResourceLexer']
+
+
+class ResourceLexer(RegexLexer):
+    """Lexer for ICU Resource bundles.
+    """
+    name = 'ResourceBundle'
+    aliases = ['resourcebundle', 'resource']
+    filenames = []
+    url = 'https://unicode-org.github.io/icu/userguide/locale/resources.html'
+    version_added = '2.0'
+
+    _types = (':table', ':array', ':string', ':bin', ':import', ':intvector',
+              ':int', ':alias')
+
+    flags = re.MULTILINE | re.IGNORECASE
+    tokens = {
+        'root': [
+            (r'//.*?$', Comment),
+            (r'"', String, 'string'),
+            (r'-?\d+', Number.Integer),
+            (r'[,{}]', Operator),
+            (r'([^\s{{:]+)(\s*)({}?)'.format('|'.join(_types)),
+             bygroups(Name, Text, Keyword)),
+            (r'\s+', Text),
+            (words(_types), Keyword),
+        ],
+        'string': [
+            (r'(\\x[0-9a-f]{2}|\\u[0-9a-f]{4}|\\U00[0-9a-f]{6}|'
+             r'\\[0-7]{1,3}|\\c.|\\[abtnvfre\'"?\\]|\\\{|[^"{\\])+', String),
+            (r'\{', String.Escape, 'msgname'),
+            (r'"', String, '#pop')
+        ],
+        'msgname': [
+            (r'([^{},]+)(\s*)', bygroups(Name, String.Escape), ('#pop', 'message'))
+        ],
+        'message': [
+            (r'\{', String.Escape, 'msgname'),
+            (r'\}', String.Escape, '#pop'),
+            (r'(,)(\s*)([a-z]+)(\s*\})',
+             bygroups(Operator, String.Escape, Keyword, String.Escape), '#pop'),
+            (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)(offset)(\s*)(:)(\s*)(-?\d+)(\s*)',
+             bygroups(Operator, String.Escape, Keyword, String.Escape, Operator,
+                      String.Escape, Operator.Word, String.Escape, Operator,
+                      String.Escape, Number.Integer, String.Escape), 'choice'),
+            (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)',
+             bygroups(Operator, String.Escape, Keyword, String.Escape, Operator,
+                      String.Escape), 'choice'),
+            (r'\s+', String.Escape)
+        ],
+        'choice': [
+            (r'(=|<|>|<=|>=|!=)(-?\d+)(\s*\{)',
+             bygroups(Operator, Number.Integer, String.Escape), 'message'),
+            (r'([a-z]+)(\s*\{)', bygroups(Keyword.Type, String.Escape), 'str'),
+            (r'\}', String.Escape, ('#pop', '#pop')),
+            (r'\s+', String.Escape)
+        ],
+        'str': [
+            (r'\}', String.Escape, '#pop'),
+            (r'\{', String.Escape, 'msgname'),
+            (r'[^{}]+', String)
+        ]
+    }
+
+    def analyse_text(text):
+        if text.startswith('root:table'):
+            return 1.0
diff --git a/lib/pygments/lexers/ride.py b/lib/pygments/lexers/ride.py
new file mode 100644
index 0000000..4d60c29
--- /dev/null
+++ b/lib/pygments/lexers/ride.py
@@ -0,0 +1,138 @@
+"""
+    pygments.lexers.ride
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Ride programming language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Keyword, Name, Number, Punctuation, \
+    String, Text
+
+__all__ = ['RideLexer']
+
+
+class RideLexer(RegexLexer):
+    """
+    For Ride source code.
+    """
+
+    name = 'Ride'
+    aliases = ['ride']
+    filenames = ['*.ride']
+    mimetypes = ['text/x-ride']
+    url = 'https://docs.waves.tech/en/ride'
+    version_added = '2.6'
+
+    validName = r'[a-zA-Z_][a-zA-Z0-9_\']*'
+
+    builtinOps = (
+        '||', '|', '>=', '>', '==', '!',
+        '=', '<=', '<', '::', ':+', ':', '!=', '/',
+        '.', '=>', '-', '+', '*', '&&', '%', '++',
+    )
+
+    globalVariablesName = (
+        'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512',
+        'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit',
+        'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN',
+        'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP',
+    )
+
+    typesName = (
+        'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias',
+        'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction',
+        'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction',
+        'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction',
+        'TransferTransaction', 'SetAssetScriptTransaction',
+        'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction',
+        'LeaseCancelTransaction', 'CreateAliasTransaction',
+        'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction',
+        'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet',
+        'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue',
+        'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512',
+        'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry',
+        'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling',
+        'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up',
+    )
+
+    functionsName = (
+        'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight',
+        'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value',
+        'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split',
+        'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256',
+        'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String',
+        'fromBase64String', 'transactionById', 'transactionHeightById',
+        'getInteger', 'getBoolean', 'getBinary', 'getString',
+        'addressFromPublicKey', 'addressFromString', 'addressFromRecipient',
+        'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue',
+        'getBinaryValue', 'getStringValue', 'addressFromStringValue',
+        'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median',
+        'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow',
+        'toBase16String', 'fromBase16String', 'blockInfoByHeight',
+        'transferTransactionById',
+    )
+
+    reservedWords = words((
+        'match', 'case', 'else', 'func', 'if',
+        'let', 'then', '@Callable', '@Verifier',
+    ), suffix=r'\b')
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'#.*', Comment.Single),
+            # Whitespace
+            (r'\s+', Text),
+            # Strings
+            (r'"', String, 'doublequote'),
+            (r'utf8\'', String, 'utf8quote'),
+            (r'base(58|64|16)\'', String, 'singlequote'),
+            # Keywords
+            (reservedWords, Keyword.Reserved),
+            (r'\{-#.*?#-\}', Keyword.Reserved),
+            (r'FOLD<\d+>', Keyword.Reserved),
+            # Types
+            (words(typesName), Keyword.Type),
+            # Main
+            # (specialName, Keyword.Reserved),
+            # Prefix Operators
+            (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function),
+            # Infix Operators
+            (words(builtinOps), Name.Function),
+            (words(globalVariablesName), Name.Function),
+            (words(functionsName), Name.Function),
+            # Numbers
+            include('numbers'),
+            # Variable Names
+            (validName, Name.Variable),
+            # Parens
+            (r'[,()\[\]{}]', Punctuation),
+        ],
+
+        'doublequote': [
+            (r'\\u[0-9a-fA-F]{4}', String.Escape),
+            (r'\\[nrfvb\\"]', String.Escape),
+            (r'[^"]', String),
+            (r'"', String, '#pop'),
+        ],
+
+        'utf8quote': [
+            (r'\\u[0-9a-fA-F]{4}', String.Escape),
+            (r'\\[nrfvb\\\']', String.Escape),
+            (r'[^\']', String),
+            (r'\'', String, '#pop'),
+        ],
+
+        'singlequote': [
+            (r'[^\']', String),
+            (r'\'', String, '#pop'),
+        ],
+
+        'numbers': [
+            (r'_?\d+', Number.Integer),
+        ],
+    }
diff --git a/lib/pygments/lexers/rita.py b/lib/pygments/lexers/rita.py
new file mode 100644
index 0000000..536aaff
--- /dev/null
+++ b/lib/pygments/lexers/rita.py
@@ -0,0 +1,42 @@
+"""
+    pygments.lexers.rita
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for RITA language
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Comment, Operator, Keyword, Name, Literal, \
+    Punctuation, Whitespace
+
+__all__ = ['RitaLexer']
+
+
+class RitaLexer(RegexLexer):
+    """
+    Lexer for RITA.
+    """
+    name = 'Rita'
+    url = 'https://github.com/zaibacu/rita-dsl'
+    filenames = ['*.rita']
+    aliases = ['rita']
+    mimetypes = ['text/rita']
+    version_added = '2.11'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'#(.*?)\n', Comment.Single),
+            (r'@(.*?)\n', Operator),  # Yes, whole line as an operator
+            (r'"(\w|\d|\s|(\\")|[\'_\-./,\?\!])+?"', Literal),
+            (r'\'(\w|\d|\s|(\\\')|["_\-./,\?\!])+?\'', Literal),
+            (r'([A-Z_]+)', Keyword),
+            (r'([a-z0-9_]+)', Name),
+            (r'((->)|[!?+*|=])', Operator),
+            (r'[\(\),\{\}]', Punctuation)
+        ]
+    }
diff --git a/lib/pygments/lexers/rnc.py b/lib/pygments/lexers/rnc.py
new file mode 100644
index 0000000..b7a06bb
--- /dev/null
+++ b/lib/pygments/lexers/rnc.py
@@ -0,0 +1,66 @@
+"""
+    pygments.lexers.rnc
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Relax-NG Compact syntax
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Punctuation
+
+__all__ = ['RNCCompactLexer']
+
+
+class RNCCompactLexer(RegexLexer):
+    """
+    For RelaxNG-compact syntax.
+    """
+
+    name = 'Relax-NG Compact'
+    url = 'http://relaxng.org'
+    aliases = ['rng-compact', 'rnc']
+    filenames = ['*.rnc']
+    version_added = '2.2'
+
+    tokens = {
+        'root': [
+            (r'namespace\b', Keyword.Namespace),
+            (r'(?:default|datatypes)\b', Keyword.Declaration),
+            (r'##.*$', Comment.Preproc),
+            (r'#.*$', Comment.Single),
+            (r'"[^"]*"', String.Double),
+            # TODO single quoted strings and escape sequences outside of
+            # double-quoted strings
+            (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'),
+            (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'),
+            (r'[,?&*=|~]|>>', Operator),
+            (r'[(){}]', Punctuation),
+            (r'.', Text),
+        ],
+
+        # a variable has been declared using `element` or `attribute`
+        'variable': [
+            (r'[^{]+', Name.Variable),
+            (r'\{', Punctuation, '#pop'),
+        ],
+
+        # after an xsd: declaration there may be attributes
+        'maybe_xsdattributes': [
+            (r'\{', Punctuation, 'xsdattributes'),
+            (r'\}', Punctuation, '#pop'),
+            (r'.', Text),
+        ],
+
+        # attributes take the form { key1 = value1 key2 = value2 ... }
+        'xsdattributes': [
+            (r'[^ =}]', Name.Attribute),
+            (r'=', Operator),
+            (r'"[^"]*"', String.Double),
+            (r'\}', Punctuation, '#pop'),
+            (r'.', Text),
+        ],
+    }
diff --git a/lib/pygments/lexers/roboconf.py b/lib/pygments/lexers/roboconf.py
new file mode 100644
index 0000000..31adba9
--- /dev/null
+++ b/lib/pygments/lexers/roboconf.py
@@ -0,0 +1,81 @@
+"""
+    pygments.lexers.roboconf
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Roboconf DSL.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, re
+from pygments.token import Text, Operator, Keyword, Name, Comment
+
+__all__ = ['RoboconfGraphLexer', 'RoboconfInstancesLexer']
+
+
+class RoboconfGraphLexer(RegexLexer):
+    """
+    Lexer for Roboconf graph files.
+    """
+    name = 'Roboconf Graph'
+    aliases = ['roboconf-graph']
+    filenames = ['*.graph']
+    url = 'https://roboconf.github.io/en/user-guide/graph-definition.html'
+    version_added = '2.1'
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            # Skip white spaces
+            (r'\s+', Text),
+
+            # There is one operator
+            (r'=', Operator),
+
+            # Keywords
+            (words(('facet', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
+            (words((
+                'installer', 'extends', 'exports', 'imports', 'facets',
+                'children'), suffix=r'\s*:?', prefix=r'\b'), Name),
+
+            # Comments
+            (r'#.*\n', Comment),
+
+            # Default
+            (r'[^#]', Text),
+            (r'.*\n', Text)
+        ]
+    }
+
+
+class RoboconfInstancesLexer(RegexLexer):
+    """
+    Lexer for Roboconf instances files.
+    """
+    name = 'Roboconf Instances'
+    aliases = ['roboconf-instances']
+    filenames = ['*.instances']
+    url = 'https://roboconf.github.io'
+    version_added = '2.1'
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+
+            # Skip white spaces
+            (r'\s+', Text),
+
+            # Keywords
+            (words(('instance of', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
+            (words(('name', 'count'), suffix=r's*:?', prefix=r'\b'), Name),
+            (r'\s*[\w.-]+\s*:', Name),
+
+            # Comments
+            (r'#.*\n', Comment),
+
+            # Default
+            (r'[^#]', Text),
+            (r'.*\n', Text)
+        ]
+    }
diff --git a/lib/pygments/lexers/robotframework.py b/lib/pygments/lexers/robotframework.py
new file mode 100644
index 0000000..f92d567
--- /dev/null
+++ b/lib/pygments/lexers/robotframework.py
@@ -0,0 +1,551 @@
+"""
+    pygments.lexers.robotframework
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Robot Framework.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+#  Copyright 2012 Nokia Siemens Networks Oyj
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.token import Token
+
+__all__ = ['RobotFrameworkLexer']
+
+
+HEADING = Token.Generic.Heading
+SETTING = Token.Keyword.Namespace
+IMPORT = Token.Name.Namespace
+TC_KW_NAME = Token.Generic.Subheading
+KEYWORD = Token.Name.Function
+ARGUMENT = Token.String
+VARIABLE = Token.Name.Variable
+COMMENT = Token.Comment
+SEPARATOR = Token.Punctuation
+SYNTAX = Token.Punctuation
+GHERKIN = Token.Generic.Emph
+ERROR = Token.Error
+
+
+def normalize(string, remove=''):
+    string = string.lower()
+    for char in remove + ' ':
+        if char in string:
+            string = string.replace(char, '')
+    return string
+
+
+class RobotFrameworkLexer(Lexer):
+    """
+    For Robot Framework test data.
+
+    Supports both space and pipe separated plain text formats.
+    """
+    name = 'RobotFramework'
+    url = 'http://robotframework.org'
+    aliases = ['robotframework']
+    filenames = ['*.robot', '*.resource']
+    mimetypes = ['text/x-robotframework']
+    version_added = '1.6'
+
+    def __init__(self, **options):
+        options['tabsize'] = 2
+        options['encoding'] = 'UTF-8'
+        Lexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        row_tokenizer = RowTokenizer()
+        var_tokenizer = VariableTokenizer()
+        index = 0
+        for row in text.splitlines():
+            for value, token in row_tokenizer.tokenize(row):
+                for value, token in var_tokenizer.tokenize(value, token):
+                    if value:
+                        yield index, token, str(value)
+                        index += len(value)
+
+
+class VariableTokenizer:
+
+    def tokenize(self, string, token):
+        var = VariableSplitter(string, identifiers='$@%&')
+        if var.start < 0 or token in (COMMENT, ERROR):
+            yield string, token
+            return
+        for value, token in self._tokenize(var, string, token):
+            if value:
+                yield value, token
+
+    def _tokenize(self, var, string, orig_token):
+        before = string[:var.start]
+        yield before, orig_token
+        yield var.identifier + '{', SYNTAX
+        yield from self.tokenize(var.base, VARIABLE)
+        yield '}', SYNTAX
+        if var.index is not None:
+            yield '[', SYNTAX
+            yield from self.tokenize(var.index, VARIABLE)
+            yield ']', SYNTAX
+        yield from self.tokenize(string[var.end:], orig_token)
+
+
+class RowTokenizer:
+
+    def __init__(self):
+        self._table = UnknownTable()
+        self._splitter = RowSplitter()
+        testcases = TestCaseTable()
+        settings = SettingTable(testcases.set_default_template)
+        variables = VariableTable()
+        keywords = KeywordTable()
+        self._tables = {'settings': settings, 'setting': settings,
+                        'metadata': settings,
+                        'variables': variables, 'variable': variables,
+                        'testcases': testcases, 'testcase': testcases,
+                        'tasks': testcases, 'task': testcases,
+                        'keywords': keywords, 'keyword': keywords,
+                        'userkeywords': keywords, 'userkeyword': keywords}
+
+    def tokenize(self, row):
+        commented = False
+        heading = False
+        for index, value in enumerate(self._splitter.split(row)):
+            # First value, and every second after that, is a separator.
+            index, separator = divmod(index-1, 2)
+            if value.startswith('#'):
+                commented = True
+            elif index == 0 and value.startswith('*'):
+                self._table = self._start_table(value)
+                heading = True
+            yield from self._tokenize(value, index, commented,
+                                      separator, heading)
+        self._table.end_row()
+
+    def _start_table(self, header):
+        name = normalize(header, remove='*')
+        return self._tables.get(name, UnknownTable())
+
+    def _tokenize(self, value, index, commented, separator, heading):
+        if commented:
+            yield value, COMMENT
+        elif separator:
+            yield value, SEPARATOR
+        elif heading:
+            yield value, HEADING
+        else:
+            yield from self._table.tokenize(value, index)
+
+
+class RowSplitter:
+    _space_splitter = re.compile('( {2,})')
+    _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))')
+
+    def split(self, row):
+        splitter = (row.startswith('| ') and self._split_from_pipes
+                    or self._split_from_spaces)
+        yield from splitter(row)
+        yield '\n'
+
+    def _split_from_spaces(self, row):
+        yield ''  # Start with (pseudo)separator similarly as with pipes
+        yield from self._space_splitter.split(row)
+
+    def _split_from_pipes(self, row):
+        _, separator, rest = self._pipe_splitter.split(row, 1)
+        yield separator
+        while self._pipe_splitter.search(rest):
+            cell, separator, rest = self._pipe_splitter.split(rest, 1)
+            yield cell
+            yield separator
+        yield rest
+
+
+class Tokenizer:
+    _tokens = None
+
+    def __init__(self):
+        self._index = 0
+
+    def tokenize(self, value):
+        values_and_tokens = self._tokenize(value, self._index)
+        self._index += 1
+        if isinstance(values_and_tokens, type(Token)):
+            values_and_tokens = [(value, values_and_tokens)]
+        return values_and_tokens
+
+    def _tokenize(self, value, index):
+        index = min(index, len(self._tokens) - 1)
+        return self._tokens[index]
+
+    def _is_assign(self, value):
+        if value.endswith('='):
+            value = value[:-1].strip()
+        var = VariableSplitter(value, identifiers='$@&')
+        return var.start == 0 and var.end == len(value)
+
+
+class Comment(Tokenizer):
+    _tokens = (COMMENT,)
+
+
+class Setting(Tokenizer):
+    _tokens = (SETTING, ARGUMENT)
+    _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
+                         'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition',
+                         'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate')
+    _import_settings = ('library', 'resource', 'variables')
+    _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
+                       'testtimeout','tasktimeout')
+    _custom_tokenizer = None
+
+    def __init__(self, template_setter=None):
+        Tokenizer.__init__(self)
+        self._template_setter = template_setter
+
+    def _tokenize(self, value, index):
+        if index == 1 and self._template_setter:
+            self._template_setter(value)
+        if index == 0:
+            normalized = normalize(value)
+            if normalized in self._keyword_settings:
+                self._custom_tokenizer = KeywordCall(support_assign=False)
+            elif normalized in self._import_settings:
+                self._custom_tokenizer = ImportSetting()
+            elif normalized not in self._other_settings:
+                return ERROR
+        elif self._custom_tokenizer:
+            return self._custom_tokenizer.tokenize(value)
+        return Tokenizer._tokenize(self, value, index)
+
+
+class ImportSetting(Tokenizer):
+    _tokens = (IMPORT, ARGUMENT)
+
+
+class TestCaseSetting(Setting):
+    _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
+                         'template')
+    _import_settings = ()
+    _other_settings = ('documentation', 'tags', 'timeout')
+
+    def _tokenize(self, value, index):
+        if index == 0:
+            type = Setting._tokenize(self, value[1:-1], index)
+            return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
+        return Setting._tokenize(self, value, index)
+
+
+class KeywordSetting(TestCaseSetting):
+    _keyword_settings = ('teardown',)
+    _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')
+
+
+class Variable(Tokenizer):
+    _tokens = (SYNTAX, ARGUMENT)
+
+    def _tokenize(self, value, index):
+        if index == 0 and not self._is_assign(value):
+            return ERROR
+        return Tokenizer._tokenize(self, value, index)
+
+
+class KeywordCall(Tokenizer):
+    _tokens = (KEYWORD, ARGUMENT)
+
+    def __init__(self, support_assign=True):
+        Tokenizer.__init__(self)
+        self._keyword_found = not support_assign
+        self._assigns = 0
+
+    def _tokenize(self, value, index):
+        if not self._keyword_found and self._is_assign(value):
+            self._assigns += 1
+            return SYNTAX  # VariableTokenizer tokenizes this later.
+        if self._keyword_found:
+            return Tokenizer._tokenize(self, value, index - self._assigns)
+        self._keyword_found = True
+        return GherkinTokenizer().tokenize(value, KEYWORD)
+
+
+class GherkinTokenizer:
+    _gherkin_prefix = re.compile('^(Given|When|Then|And|But) ', re.IGNORECASE)
+
+    def tokenize(self, value, token):
+        match = self._gherkin_prefix.match(value)
+        if not match:
+            return [(value, token)]
+        end = match.end()
+        return [(value[:end], GHERKIN), (value[end:], token)]
+
+
+class TemplatedKeywordCall(Tokenizer):
+    _tokens = (ARGUMENT,)
+
+
+class ForLoop(Tokenizer):
+
+    def __init__(self):
+        Tokenizer.__init__(self)
+        self._in_arguments = False
+
+    def _tokenize(self, value, index):
+        token = self._in_arguments and ARGUMENT or SYNTAX
+        if value.upper() in ('IN', 'IN RANGE'):
+            self._in_arguments = True
+        return token
+
+
+class _Table:
+    _tokenizer_class = None
+
+    def __init__(self, prev_tokenizer=None):
+        self._tokenizer = self._tokenizer_class()
+        self._prev_tokenizer = prev_tokenizer
+        self._prev_values_on_row = []
+
+    def tokenize(self, value, index):
+        if self._continues(value, index):
+            self._tokenizer = self._prev_tokenizer
+            yield value, SYNTAX
+        else:
+            yield from self._tokenize(value, index)
+        self._prev_values_on_row.append(value)
+
+    def _continues(self, value, index):
+        return value == '...' and all(self._is_empty(t)
+                                      for t in self._prev_values_on_row)
+
+    def _is_empty(self, value):
+        return value in ('', '\\')
+
+    def _tokenize(self, value, index):
+        return self._tokenizer.tokenize(value)
+
+    def end_row(self):
+        self.__init__(prev_tokenizer=self._tokenizer)
+
+
+class UnknownTable(_Table):
+    _tokenizer_class = Comment
+
+    def _continues(self, value, index):
+        return False
+
+
+class VariableTable(_Table):
+    _tokenizer_class = Variable
+
+
+class SettingTable(_Table):
+    _tokenizer_class = Setting
+
+    def __init__(self, template_setter, prev_tokenizer=None):
+        _Table.__init__(self, prev_tokenizer)
+        self._template_setter = template_setter
+
+    def _tokenize(self, value, index):
+        if index == 0 and normalize(value) == 'testtemplate':
+            self._tokenizer = Setting(self._template_setter)
+        return _Table._tokenize(self, value, index)
+
+    def end_row(self):
+        self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
+
+
+class TestCaseTable(_Table):
+    _setting_class = TestCaseSetting
+    _test_template = None
+    _default_template = None
+
+    @property
+    def _tokenizer_class(self):
+        if self._test_template or (self._default_template and
+                                   self._test_template is not False):
+            return TemplatedKeywordCall
+        return KeywordCall
+
+    def _continues(self, value, index):
+        return index > 0 and _Table._continues(self, value, index)
+
+    def _tokenize(self, value, index):
+        if index == 0:
+            if value:
+                self._test_template = None
+            return GherkinTokenizer().tokenize(value, TC_KW_NAME)
+        if index == 1 and self._is_setting(value):
+            if self._is_template(value):
+                self._test_template = False
+                self._tokenizer = self._setting_class(self.set_test_template)
+            else:
+                self._tokenizer = self._setting_class()
+        if index == 1 and self._is_for_loop(value):
+            self._tokenizer = ForLoop()
+        if index == 1 and self._is_empty(value):
+            return [(value, SYNTAX)]
+        return _Table._tokenize(self, value, index)
+
+    def _is_setting(self, value):
+        return value.startswith('[') and value.endswith(']')
+
+    def _is_template(self, value):
+        return normalize(value) == '[template]'
+
+    def _is_for_loop(self, value):
+        return value.startswith(':') and normalize(value, remove=':') == 'for'
+
+    def set_test_template(self, template):
+        self._test_template = self._is_template_set(template)
+
+    def set_default_template(self, template):
+        self._default_template = self._is_template_set(template)
+
+    def _is_template_set(self, template):
+        return normalize(template) not in ('', '\\', 'none', '${empty}')
+
+
+class KeywordTable(TestCaseTable):
+    _tokenizer_class = KeywordCall
+    _setting_class = KeywordSetting
+
+    def _is_template(self, value):
+        return False
+
+
+# Following code copied directly from Robot Framework 2.7.5.
+
+class VariableSplitter:
+
+    def __init__(self, string, identifiers):
+        self.identifier = None
+        self.base = None
+        self.index = None
+        self.start = -1
+        self.end = -1
+        self._identifiers = identifiers
+        self._may_have_internal_variables = False
+        try:
+            self._split(string)
+        except ValueError:
+            pass
+        else:
+            self._finalize()
+
+    def get_replaced_base(self, variables):
+        if self._may_have_internal_variables:
+            return variables.replace_string(self.base)
+        return self.base
+
+    def _finalize(self):
+        self.identifier = self._variable_chars[0]
+        self.base = ''.join(self._variable_chars[2:-1])
+        self.end = self.start + len(self._variable_chars)
+        if self._has_list_or_dict_variable_index():
+            self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])
+            self.end += len(self._list_and_dict_variable_index_chars)
+
+    def _has_list_or_dict_variable_index(self):
+        return self._list_and_dict_variable_index_chars\
+        and self._list_and_dict_variable_index_chars[-1] == ']'
+
+    def _split(self, string):
+        start_index, max_index = self._find_variable(string)
+        self.start = start_index
+        self._open_curly = 1
+        self._state = self._variable_state
+        self._variable_chars = [string[start_index], '{']
+        self._list_and_dict_variable_index_chars = []
+        self._string = string
+        start_index += 2
+        for index, char in enumerate(string[start_index:]):
+            index += start_index  # Giving start to enumerate only in Py 2.6+
+            try:
+                self._state(char, index)
+            except StopIteration:
+                return
+            if index  == max_index and not self._scanning_list_variable_index():
+                return
+
+    def _scanning_list_variable_index(self):
+        return self._state in [self._waiting_list_variable_index_state,
+                               self._list_variable_index_state]
+
+    def _find_variable(self, string):
+        max_end_index = string.rfind('}')
+        if max_end_index == -1:
+            raise ValueError('No variable end found')
+        if self._is_escaped(string, max_end_index):
+            return self._find_variable(string[:max_end_index])
+        start_index = self._find_start_index(string, 1, max_end_index)
+        if start_index == -1:
+            raise ValueError('No variable start found')
+        return start_index, max_end_index
+
+    def _find_start_index(self, string, start, end):
+        index = string.find('{', start, end) - 1
+        if index < 0:
+            return -1
+        if self._start_index_is_ok(string, index):
+            return index
+        return self._find_start_index(string, index+2, end)
+
+    def _start_index_is_ok(self, string, index):
+        return string[index] in self._identifiers\
+        and not self._is_escaped(string, index)
+
+    def _is_escaped(self, string, index):
+        escaped = False
+        while index > 0 and string[index-1] == '\\':
+            index -= 1
+            escaped = not escaped
+        return escaped
+
+    def _variable_state(self, char, index):
+        self._variable_chars.append(char)
+        if char == '}' and not self._is_escaped(self._string, index):
+            self._open_curly -= 1
+            if self._open_curly == 0:
+                if not self._is_list_or_dict_variable():
+                    raise StopIteration
+                self._state = self._waiting_list_variable_index_state
+        elif char in self._identifiers:
+            self._state = self._internal_variable_start_state
+
+    def _is_list_or_dict_variable(self):
+        return self._variable_chars[0] in ('@','&')
+
+    def _internal_variable_start_state(self, char, index):
+        self._state = self._variable_state
+        if char == '{':
+            self._variable_chars.append(char)
+            self._open_curly += 1
+            self._may_have_internal_variables = True
+        else:
+            self._variable_state(char, index)
+
+    def _waiting_list_variable_index_state(self, char, index):
+        if char != '[':
+            raise StopIteration
+        self._list_and_dict_variable_index_chars.append(char)
+        self._state = self._list_variable_index_state
+
+    def _list_variable_index_state(self, char, index):
+        self._list_and_dict_variable_index_chars.append(char)
+        if char == ']':
+            raise StopIteration
diff --git a/lib/pygments/lexers/ruby.py b/lib/pygments/lexers/ruby.py
new file mode 100644
index 0000000..72aaeb5
--- /dev/null
+++ b/lib/pygments/lexers/ruby.py
@@ -0,0 +1,518 @@
+"""
+    pygments.lexers.ruby
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Ruby and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, include, \
+    bygroups, default, LexerContext, do_insertions, words, line_re
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error, Generic, Whitespace
+from pygments.util import shebang_matches
+
+__all__ = ['RubyLexer', 'RubyConsoleLexer', 'FancyLexer']
+
+
+RUBY_OPERATORS = (
+    '*', '**', '-', '+', '-@', '+@', '/', '%', '&', '|', '^', '`', '~',
+    '[]', '[]=', '<<', '>>', '<', '<>', '<=>', '>', '>=', '==', '==='
+)
+
+
+class RubyLexer(ExtendedRegexLexer):
+    """
+    For Ruby source code.
+    """
+
+    name = 'Ruby'
+    url = 'http://www.ruby-lang.org'
+    aliases = ['ruby', 'rb', 'duby']
+    filenames = ['*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec',
+                 '*.rbx', '*.duby', 'Gemfile', 'Vagrantfile']
+    mimetypes = ['text/x-ruby', 'application/x-ruby']
+    version_added = ''
+
+    flags = re.DOTALL | re.MULTILINE
+
+    def heredoc_callback(self, match, ctx):
+        # okay, this is the hardest part of parsing Ruby...
+        # match: 1 = <<[-~]?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
+
+        start = match.start(1)
+        yield start, Operator, match.group(1)        # <<[-~]?
+        yield match.start(2), String.Heredoc, match.group(2)   # quote ", ', `
+        yield match.start(3), String.Delimiter, match.group(3) # heredoc name
+        yield match.start(4), String.Heredoc, match.group(4)   # quote again
+
+        heredocstack = ctx.__dict__.setdefault('heredocstack', [])
+        outermost = not bool(heredocstack)
+        heredocstack.append((match.group(1) in ('<<-', '<<~'), match.group(3)))
+
+        ctx.pos = match.start(5)
+        ctx.end = match.end(5)
+        # this may find other heredocs, so limit the recursion depth
+        if len(heredocstack) < 100:
+            yield from self.get_tokens_unprocessed(context=ctx)
+        else:
+            yield ctx.pos, String.Heredoc, match.group(5)
+        ctx.pos = match.end()
+
+        if outermost:
+            # this is the outer heredoc again, now we can process them all
+            for tolerant, hdname in heredocstack:
+                lines = []
+                for match in line_re.finditer(ctx.text, ctx.pos):
+                    if tolerant:
+                        check = match.group().strip()
+                    else:
+                        check = match.group().rstrip()
+                    if check == hdname:
+                        for amatch in lines:
+                            yield amatch.start(), String.Heredoc, amatch.group()
+                        yield match.start(), String.Delimiter, match.group()
+                        ctx.pos = match.end()
+                        break
+                    else:
+                        lines.append(match)
+                else:
+                    # end of heredoc not found -- error!
+                    for amatch in lines:
+                        yield amatch.start(), Error, amatch.group()
+            ctx.end = len(ctx.text)
+            del heredocstack[:]
+
+    def gen_rubystrings_rules():
+        def intp_regex_callback(self, match, ctx):
+            yield match.start(1), String.Regex, match.group(1)  # begin
+            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Regex, match.group(4)  # end[mixounse]*
+            ctx.pos = match.end()
+
+        def intp_string_callback(self, match, ctx):
+            yield match.start(1), String.Other, match.group(1)
+            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Other, match.group(4)  # end
+            ctx.pos = match.end()
+
+        states = {}
+        states['strings'] = [
+            # easy ones
+            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
+            (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
+            (r":'(\\\\|\\[^\\]|[^'\\])*'", String.Symbol),
+            (r':"', String.Symbol, 'simple-sym'),
+            (r'([a-zA-Z_]\w*)(:)(?!:)',
+             bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
+            (r'"', String.Double, 'simple-string-double'),
+            (r"'", String.Single, 'simple-string-single'),
+            (r'(?', '<>', 'ab'):
+            states[name+'-intp-string'] = [
+                (r'\\[\\' + bracecc + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                include('string-intp-escaped'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            states['strings'].append((r'%[QWx]?' + lbrace, String.Other,
+                                      name+'-intp-string'))
+            states[name+'-string'] = [
+                (r'\\[\\' + bracecc + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            states['strings'].append((r'%[qsw]' + lbrace, String.Other,
+                                      name+'-string'))
+            states[name+'-regex'] = [
+                (r'\\[\\' + bracecc + ']', String.Regex),
+                (lbrace, String.Regex, '#push'),
+                (rbrace + '[mixounse]*', String.Regex, '#pop'),
+                include('string-intp'),
+                (r'[\\#' + bracecc + ']', String.Regex),
+                (r'[^\\#' + bracecc + ']+', String.Regex),
+            ]
+            states['strings'].append((r'%r' + lbrace, String.Regex,
+                                      name+'-regex'))
+
+        # these must come after %!
+        states['strings'] += [
+            # %r regex
+            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)',
+             intp_regex_callback),
+            # regular fancy strings with qsw
+            (r'%[qsw]([\W_])((?:\\\1|(?!\1).)*)\1', String.Other),
+            (r'(%[QWx]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+            # special forms of fancy strings after operators or
+            # in method calls with braces
+            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Whitespace, String.Other, None)),
+            # and because of fixed width lookbehinds the whole thing a
+            # second time for line startings...
+            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Whitespace, String.Other, None)),
+            # all regular fancy strings without qsw
+            (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+        ]
+
+        return states
+
+    tokens = {
+        'root': [
+            (r'\A#!.+?$', Comment.Hashbang),
+            (r'#.*?$', Comment.Single),
+            (r'=begin\s.*?\n=end.*?$', Comment.Multiline),
+            # keywords
+            (words((
+                'BEGIN', 'END', 'alias', 'begin', 'break', 'case', 'defined?',
+                'do', 'else', 'elsif', 'end', 'ensure', 'for', 'if', 'in', 'next', 'redo',
+                'rescue', 'raise', 'retry', 'return', 'super', 'then', 'undef',
+                'unless', 'until', 'when', 'while', 'yield'), suffix=r'\b'),
+             Keyword),
+            # start of function, class and module names
+            (r'(module)(\s+)([a-zA-Z_]\w*'
+             r'(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Whitespace, Name.Namespace)),
+            (r'(def)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
+            (r'(class)(\s+)', bygroups(Keyword, Whitespace), 'classname'),
+            # special methods
+            (words((
+                'initialize', 'new', 'loop', 'include', 'extend', 'raise', 'attr_reader',
+                'attr_writer', 'attr_accessor', 'attr', 'catch', 'throw', 'private',
+                'module_function', 'public', 'protected', 'true', 'false', 'nil'),
+                suffix=r'\b'),
+             Keyword.Pseudo),
+            (r'(not|and|or)\b', Operator.Word),
+            (words((
+                'autoload', 'block_given', 'const_defined', 'eql', 'equal', 'frozen', 'include',
+                'instance_of', 'is_a', 'iterator', 'kind_of', 'method_defined', 'nil',
+                'private_method_defined', 'protected_method_defined',
+                'public_method_defined', 'respond_to', 'tainted'), suffix=r'\?'),
+             Name.Builtin),
+            (r'(chomp|chop|exit|gsub|sub)!', Name.Builtin),
+            (words((
+                'Array', 'Float', 'Integer', 'String', '__id__', '__send__', 'abort',
+                'ancestors', 'at_exit', 'autoload', 'binding', 'callcc', 'caller',
+                'catch', 'chomp', 'chop', 'class_eval', 'class_variables',
+                'clone', 'const_defined?', 'const_get', 'const_missing', 'const_set',
+                'constants', 'display', 'dup', 'eval', 'exec', 'exit', 'extend', 'fail', 'fork',
+                'format', 'freeze', 'getc', 'gets', 'global_variables', 'gsub',
+                'hash', 'id', 'included_modules', 'inspect', 'instance_eval',
+                'instance_method', 'instance_methods',
+                'instance_variable_get', 'instance_variable_set', 'instance_variables',
+                'lambda', 'load', 'local_variables', 'loop',
+                'method', 'method_missing', 'methods', 'module_eval', 'name',
+                'object_id', 'open', 'p', 'print', 'printf', 'private_class_method',
+                'private_instance_methods',
+                'private_methods', 'proc', 'protected_instance_methods',
+                'protected_methods', 'public_class_method',
+                'public_instance_methods', 'public_methods',
+                'putc', 'puts', 'raise', 'rand', 'readline', 'readlines', 'require',
+                'scan', 'select', 'self', 'send', 'set_trace_func', 'singleton_methods', 'sleep',
+                'split', 'sprintf', 'srand', 'sub', 'syscall', 'system', 'taint',
+                'test', 'throw', 'to_a', 'to_s', 'trace_var', 'trap', 'untaint',
+                'untrace_var', 'warn'), prefix=r'(?~!:])|'
+             r'(?<=(?:\s|;)when\s)|'
+             r'(?<=(?:\s|;)or\s)|'
+             r'(?<=(?:\s|;)and\s)|'
+             r'(?<=\.index\s)|'
+             r'(?<=\.scan\s)|'
+             r'(?<=\.sub\s)|'
+             r'(?<=\.sub!\s)|'
+             r'(?<=\.gsub\s)|'
+             r'(?<=\.gsub!\s)|'
+             r'(?<=\.match\s)|'
+             r'(?<=(?:\s|;)if\s)|'
+             r'(?<=(?:\s|;)elsif\s)|'
+             r'(?<=^when\s)|'
+             r'(?<=^index\s)|'
+             r'(?<=^scan\s)|'
+             r'(?<=^sub\s)|'
+             r'(?<=^gsub\s)|'
+             r'(?<=^sub!\s)|'
+             r'(?<=^gsub!\s)|'
+             r'(?<=^match\s)|'
+             r'(?<=^if\s)|'
+             r'(?<=^elsif\s)'
+             r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
+            # multiline regex (in method calls or subscripts)
+            (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
+            # multiline regex (this time the funny no whitespace rule)
+            (r'(\s+)(/)(?![\s=])', bygroups(Whitespace, String.Regex),
+             'multiline-regex'),
+            # lex numbers and ignore following regular expressions which
+            # are division operators in fact (grrrr. i hate that. any
+            # better ideas?)
+            # since pygments 0.7 we also eat a "?" operator after numbers
+            # so that the char operator does not work. Chars are not allowed
+            # there so that you can use the ternary operator.
+            # stupid example:
+            #   x>=0?n[x]:""
+            (r'(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
+             bygroups(Number.Oct, Whitespace, Operator)),
+            (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
+             bygroups(Number.Hex, Whitespace, Operator)),
+            (r'(0b[01]+(?:_[01]+)*)(\s*)([/?])?',
+             bygroups(Number.Bin, Whitespace, Operator)),
+            (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
+             bygroups(Number.Integer, Whitespace, Operator)),
+            # Names
+            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+            (r'\$\w+', Name.Variable.Global),
+            (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
+            (r'\$-[0adFiIlpvw]', Name.Variable.Global),
+            (r'::', Operator),
+            include('strings'),
+            # chars
+            (r'\?(\\[MC]-)*'  # modifiers
+             r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
+             r'(?!\w)',
+             String.Char),
+            (r'[A-Z]\w+', Name.Constant),
+            # this is needed because ruby attributes can look
+            # like keywords (class) or like this: ` ?!?
+            (words(RUBY_OPERATORS, prefix=r'(\.|::)'),
+             bygroups(Operator, Name.Operator)),
+            (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
+             bygroups(Operator, Name)),
+            (r'[a-zA-Z_]\w*[!?]?', Name),
+            (r'(\[|\]|\*\*|<>?|>=|<=|<=>|=~|={3}|'
+             r'!~|&&?|\|\||\.{1,3})', Operator),
+            (r'[-+/*%=<>&!^|~]=?', Operator),
+            (r'[(){};,/?:\\]', Punctuation),
+            (r'\s+', Whitespace)
+        ],
+        'funcname': [
+            (r'\(', Punctuation, 'defexpr'),
+            (r'(?:([a-zA-Z_]\w*)(\.))?'  # optional scope name, like "self."
+             r'('
+                r'[a-zA-Z\u0080-\uffff][a-zA-Z0-9_\u0080-\uffff]*[!?=]?'  # method name
+                r'|!=|!~|=~|\*\*?|[-+!~]@?|[/%&|^]|<=>|<[<=]?|>[>=]?|===?'  # or operator override
+                r'|\[\]=?'  # or element reference/assignment override
+                r'|`'  # or the undocumented backtick override
+             r')',
+             bygroups(Name.Class, Operator, Name.Function), '#pop'),
+            default('#pop')
+        ],
+        'classname': [
+            (r'\(', Punctuation, 'defexpr'),
+            (r'<<', Operator, '#pop'),
+            (r'[A-Z_]\w*', Name.Class, '#pop'),
+            default('#pop')
+        ],
+        'defexpr': [
+            (r'(\))(\.|::)?', bygroups(Punctuation, Operator), '#pop'),
+            (r'\(', Operator, '#push'),
+            include('root')
+        ],
+        'in-intp': [
+            (r'\{', String.Interpol, '#push'),
+            (r'\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        'string-intp': [
+            (r'#\{', String.Interpol, 'in-intp'),
+            (r'#@@?[a-zA-Z_]\w*', String.Interpol),
+            (r'#\$[a-zA-Z_]\w*', String.Interpol)
+        ],
+        'string-intp-escaped': [
+            include('string-intp'),
+            (r'\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})',
+             String.Escape)
+        ],
+        'interpolated-regex': [
+            include('string-intp'),
+            (r'[\\#]', String.Regex),
+            (r'[^\\#]+', String.Regex),
+        ],
+        'interpolated-string': [
+            include('string-intp'),
+            (r'[\\#]', String.Other),
+            (r'[^\\#]+', String.Other),
+        ],
+        'multiline-regex': [
+            include('string-intp'),
+            (r'\\\\', String.Regex),
+            (r'\\/', String.Regex),
+            (r'[\\#]', String.Regex),
+            (r'[^\\/#]+', String.Regex),
+            (r'/[mixounse]*', String.Regex, '#pop'),
+        ],
+        'end-part': [
+            (r'.+', Comment.Preproc, '#pop')
+        ]
+    }
+    tokens.update(gen_rubystrings_rules())
+
+    def analyse_text(text):
+        return shebang_matches(text, r'ruby(1\.\d)?')
+
+
+class RubyConsoleLexer(Lexer):
+    """
+    For Ruby interactive console (**irb**) output.
+    """
+    name = 'Ruby irb session'
+    aliases = ['rbcon', 'irb']
+    mimetypes = ['text/x-ruby-shellsession']
+    url = 'https://www.ruby-lang.org'
+    version_added = ''
+    _example = 'rbcon/console'
+
+    _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] '
+                            r'|>> |\?> ')
+
+    def get_tokens_unprocessed(self, text):
+        rblexer = RubyLexer(**self.options)
+
+        curcode = ''
+        insertions = []
+        for match in line_re.finditer(text):
+            line = match.group()
+            m = self._prompt_re.match(line)
+            if m is not None:
+                end = m.end()
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:end])]))
+                curcode += line[end:]
+            else:
+                if curcode:
+                    yield from do_insertions(
+                        insertions, rblexer.get_tokens_unprocessed(curcode))
+                    curcode = ''
+                    insertions = []
+                yield match.start(), Generic.Output, line
+        if curcode:
+            yield from do_insertions(
+                insertions, rblexer.get_tokens_unprocessed(curcode))
+
+
+class FancyLexer(RegexLexer):
+    """
+    Pygments Lexer For Fancy.
+
+    Fancy is a self-hosted, pure object-oriented, dynamic,
+    class-based, concurrent general-purpose programming language
+    running on Rubinius, the Ruby VM.
+    """
+    name = 'Fancy'
+    url = 'https://github.com/bakkdoor/fancy'
+    filenames = ['*.fy', '*.fancypack']
+    aliases = ['fancy', 'fy']
+    mimetypes = ['text/x-fancysrc']
+    version_added = '1.5'
+
+    tokens = {
+        # copied from PerlLexer:
+        'balanced-regex': [
+            (r'/(\\\\|\\[^\\]|[^/\\])*/[egimosx]*', String.Regex, '#pop'),
+            (r'!(\\\\|\\[^\\]|[^!\\])*![egimosx]*', String.Regex, '#pop'),
+            (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
+            (r'\{(\\\\|\\[^\\]|[^}\\])*\}[egimosx]*', String.Regex, '#pop'),
+            (r'<(\\\\|\\[^\\]|[^>\\])*>[egimosx]*', String.Regex, '#pop'),
+            (r'\[(\\\\|\\[^\\]|[^\]\\])*\][egimosx]*', String.Regex, '#pop'),
+            (r'\((\\\\|\\[^\\]|[^)\\])*\)[egimosx]*', String.Regex, '#pop'),
+            (r'@(\\\\|\\[^\\]|[^@\\])*@[egimosx]*', String.Regex, '#pop'),
+            (r'%(\\\\|\\[^\\]|[^%\\])*%[egimosx]*', String.Regex, '#pop'),
+            (r'\$(\\\\|\\[^\\]|[^$\\])*\$[egimosx]*', String.Regex, '#pop'),
+        ],
+        'root': [
+            (r'\s+', Whitespace),
+
+            # balanced delimiters (copied from PerlLexer):
+            (r's\{(\\\\|\\[^\\]|[^}\\])*\}\s*', String.Regex, 'balanced-regex'),
+            (r's<(\\\\|\\[^\\]|[^>\\])*>\s*', String.Regex, 'balanced-regex'),
+            (r's\[(\\\\|\\[^\\]|[^\]\\])*\]\s*', String.Regex, 'balanced-regex'),
+            (r's\((\\\\|\\[^\\]|[^)\\])*\)\s*', String.Regex, 'balanced-regex'),
+            (r'm?/(\\\\|\\[^\\]|[^///\n])*/[gcimosx]*', String.Regex),
+            (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),
+
+            # Comments
+            (r'#(.*?)\n', Comment.Single),
+            # Symbols
+            (r'\'([^\'\s\[\](){}]+|\[\])', String.Symbol),
+            # Multi-line DoubleQuotedString
+            (r'"""(\\\\|\\[^\\]|[^\\])*?"""', String),
+            # DoubleQuotedString
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # keywords
+            (r'(def|class|try|catch|finally|retry|return|return_local|match|'
+             r'case|->|=>)\b', Keyword),
+            # constants
+            (r'(self|super|nil|false|true)\b', Name.Constant),
+            (r'[(){};,/?|:\\]', Punctuation),
+            # names
+            (words((
+                'Object', 'Array', 'Hash', 'Directory', 'File', 'Class', 'String',
+                'Number', 'Enumerable', 'FancyEnumerable', 'Block', 'TrueClass',
+                'NilClass', 'FalseClass', 'Tuple', 'Symbol', 'Stack', 'Set',
+                'FancySpec', 'Method', 'Package', 'Range'), suffix=r'\b'),
+             Name.Builtin),
+            # functions
+            (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function),
+            # operators, must be below functions
+            (r'[-+*/~,<>=&!?%^\[\].$]+', Operator),
+            (r'[A-Z]\w*', Name.Constant),
+            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+            ('@@?', Operator),
+            (r'[a-zA-Z_]\w*', Name),
+            # numbers - / checks are necessary to avoid mismarking regexes,
+            # see comment in RubyLexer
+            (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
+             bygroups(Number.Oct, Whitespace, Operator)),
+            (r'(0[xX][0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
+             bygroups(Number.Hex, Whitespace, Operator)),
+            (r'(0[bB][01]+(?:_[01]+)*)(\s*)([/?])?',
+             bygroups(Number.Bin, Whitespace, Operator)),
+            (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
+             bygroups(Number.Integer, Whitespace, Operator)),
+            (r'\d+([eE][+-]?[0-9]+)|\d+\.\d+([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+', Number.Integer)
+        ]
+    }
diff --git a/lib/pygments/lexers/rust.py b/lib/pygments/lexers/rust.py
new file mode 100644
index 0000000..6341047
--- /dev/null
+++ b/lib/pygments/lexers/rust.py
@@ -0,0 +1,222 @@
+"""
+    pygments.lexers.rust
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Rust language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['RustLexer']
+
+
+class RustLexer(RegexLexer):
+    """
+    Lexer for the Rust programming language (version 1.47).
+    """
+    name = 'Rust'
+    url = 'https://www.rust-lang.org/'
+    filenames = ['*.rs', '*.rs.in']
+    aliases = ['rust', 'rs']
+    mimetypes = ['text/rust', 'text/x-rust']
+    version_added = '1.6'
+
+    keyword_types = (words((
+        'u8', 'u16', 'u32', 'u64', 'u128', 'i8', 'i16', 'i32', 'i64', 'i128',
+        'usize', 'isize', 'f32', 'f64', 'char', 'str', 'bool',
+    ), suffix=r'\b'), Keyword.Type)
+
+    builtin_funcs_types = (words((
+        'Copy', 'Send', 'Sized', 'Sync', 'Unpin',
+        'Drop', 'Fn', 'FnMut', 'FnOnce', 'drop',
+        'Box', 'ToOwned', 'Clone',
+        'PartialEq', 'PartialOrd', 'Eq', 'Ord',
+        'AsRef', 'AsMut', 'Into', 'From', 'Default',
+        'Iterator', 'Extend', 'IntoIterator', 'DoubleEndedIterator',
+        'ExactSizeIterator',
+        'Option', 'Some', 'None',
+        'Result', 'Ok', 'Err',
+        'String', 'ToString', 'Vec',
+    ), suffix=r'\b'), Name.Builtin)
+
+    builtin_macros = (words((
+        'asm', 'assert', 'assert_eq', 'assert_ne', 'cfg', 'column',
+        'compile_error', 'concat', 'concat_idents', 'dbg', 'debug_assert',
+        'debug_assert_eq', 'debug_assert_ne', 'env', 'eprint', 'eprintln',
+        'file', 'format', 'format_args', 'format_args_nl', 'global_asm',
+        'include', 'include_bytes', 'include_str',
+        'is_aarch64_feature_detected',
+        'is_arm_feature_detected',
+        'is_mips64_feature_detected',
+        'is_mips_feature_detected',
+        'is_powerpc64_feature_detected',
+        'is_powerpc_feature_detected',
+        'is_x86_feature_detected',
+        'line', 'llvm_asm', 'log_syntax', 'macro_rules', 'matches',
+        'module_path', 'option_env', 'panic', 'print', 'println', 'stringify',
+        'thread_local', 'todo', 'trace_macros', 'unimplemented', 'unreachable',
+        'vec', 'write', 'writeln',
+    ), suffix=r'!'), Name.Function.Magic)
+
+    tokens = {
+        'root': [
+            # rust allows a file to start with a shebang, but if the first line
+            # starts with #![ then it's not a shebang but a crate attribute.
+            (r'#![^[\r\n].*$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            # Whitespace and Comments
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'//!.*?\n', String.Doc),
+            (r'///(\n|[^/].*?\n)', String.Doc),
+            (r'//(.*?)\n', Comment.Single),
+            (r'/\*\*(\n|[^/*])', String.Doc, 'doccomment'),
+            (r'/\*!', String.Doc, 'doccomment'),
+            (r'/\*', Comment.Multiline, 'comment'),
+
+            # Macro parameters
+            (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc),
+            # Keywords
+            (words(('as', 'async', 'await', 'box', 'const', 'crate', 'dyn',
+                    'else', 'extern', 'for', 'if', 'impl', 'in', 'loop',
+                    'match', 'move', 'mut', 'pub', 'ref', 'return', 'static',
+                    'super', 'trait', 'unsafe', 'use', 'where', 'while'),
+                   suffix=r'\b'), Keyword),
+            (words(('abstract', 'become', 'do', 'final', 'macro', 'override',
+                    'priv', 'typeof', 'try', 'unsized', 'virtual', 'yield'),
+                   suffix=r'\b'), Keyword.Reserved),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'self\b', Name.Builtin.Pseudo),
+            (r'mod\b', Keyword, 'modname'),
+            (r'let\b', Keyword.Declaration),
+            (r'fn\b', Keyword, 'funcname'),
+            (r'(struct|enum|type|union)\b', Keyword, 'typename'),
+            (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Whitespace, Keyword)),
+            keyword_types,
+            (r'[sS]elf\b', Name.Builtin.Pseudo),
+            # Prelude (taken from Rust's src/libstd/prelude.rs)
+            builtin_funcs_types,
+            builtin_macros,
+            # Path separators, so types don't catch them.
+            (r'::\b', Punctuation),
+            # Types in positions.
+            (r'(?::|->)', Punctuation, 'typename'),
+            # Labels
+            (r'(break|continue)(\b\s*)(\'[A-Za-z_]\w*)?',
+             bygroups(Keyword, Text.Whitespace, Name.Label)),
+
+            # Character literals
+            (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+             String.Char),
+            (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+             String.Char),
+
+            # Binary literals
+            (r'0b[01_]+', Number.Bin, 'number_lit'),
+            # Octal literals
+            (r'0o[0-7_]+', Number.Oct, 'number_lit'),
+            # Hexadecimal literals
+            (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'),
+            # Decimal literals
+            (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+             r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float,
+             'number_lit'),
+            (r'[0-9][0-9_]*', Number.Integer, 'number_lit'),
+
+            # String literals
+            (r'b"', String, 'bytestring'),
+            (r'"', String, 'string'),
+            (r'(?s)b?r(#*)".*?"\1', String),
+
+            # Lifetime names
+            (r"'", Operator, 'lifetime'),
+
+            # Operators and Punctuation
+            (r'\.\.=?', Operator),
+            (r'[{}()\[\],.;]', Punctuation),
+            (r'[+\-*/%&|<>^!~@=:?]', Operator),
+
+            # Identifiers
+            (r'[a-zA-Z_]\w*', Name),
+            # Raw identifiers
+            (r'r#[a-zA-Z_]\w*', Name),
+
+            # Attributes
+            (r'#!?\[', Comment.Preproc, 'attribute['),
+
+            # Misc
+            # Lone hashes: not used in Rust syntax, but allowed in macro
+            # arguments, most famously for quote::quote!()
+            (r'#', Punctuation),
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline),
+        ],
+        'doccomment': [
+            (r'[^*/]+', String.Doc),
+            (r'/\*', String.Doc, '#push'),
+            (r'\*/', String.Doc, '#pop'),
+            (r'[*/]', String.Doc),
+        ],
+        'modname': [
+            (r'\s+', Whitespace),
+            (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'),
+            default('#pop'),
+        ],
+        'funcname': [
+            (r'\s+', Whitespace),
+            (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
+            default('#pop'),
+        ],
+        'typename': [
+            (r'\s+', Whitespace),
+            (r'&', Keyword.Pseudo),
+            (r"'", Operator, 'lifetime'),
+            builtin_funcs_types,
+            keyword_types,
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+            default('#pop'),
+        ],
+        'lifetime': [
+            (r"(static|_)", Name.Builtin),
+            (r"[a-zA-Z_]+\w*", Name.Attribute),
+            default('#pop'),
+        ],
+        'number_lit': [
+            (r'[ui](8|16|32|64|size)', Keyword, '#pop'),
+            (r'f(32|64)', Keyword, '#pop'),
+            default('#pop'),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r"""\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}""", String.Escape),
+            (r'[^\\"]+', String),
+            (r'\\', String),
+        ],
+        'bytestring': [
+            (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape),
+            include('string'),
+        ],
+        'attribute_common': [
+            (r'"', String, 'string'),
+            (r'\[', Comment.Preproc, 'attribute['),
+        ],
+        'attribute[': [
+            include('attribute_common'),
+            (r'\]', Comment.Preproc, '#pop'),
+            (r'[^"\]\[]+', Comment.Preproc),
+        ],
+    }
diff --git a/lib/pygments/lexers/sas.py b/lib/pygments/lexers/sas.py
new file mode 100644
index 0000000..1b2ad43
--- /dev/null
+++ b/lib/pygments/lexers/sas.py
@@ -0,0 +1,227 @@
+"""
+    pygments.lexers.sas
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for SAS.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Keyword, Name, Number, String, Text, \
+    Other, Generic
+
+__all__ = ['SASLexer']
+
+
+class SASLexer(RegexLexer):
+    """
+    For SAS files.
+    """
+    # Syntax from syntax/sas.vim by James Kidd 
+
+    name      = 'SAS'
+    aliases   = ['sas']
+    filenames = ['*.SAS', '*.sas']
+    mimetypes = ['text/x-sas', 'text/sas', 'application/x-sas']
+    url = 'https://en.wikipedia.org/wiki/SAS_(software)'
+    version_added = '2.2'
+    flags     = re.IGNORECASE | re.MULTILINE
+
+    builtins_macros = (
+        "bquote", "nrbquote", "cmpres", "qcmpres", "compstor", "datatyp",
+        "display", "do", "else", "end", "eval", "global", "goto", "if",
+        "index", "input", "keydef", "label", "left", "length", "let",
+        "local", "lowcase", "macro", "mend", "nrquote",
+        "nrstr", "put", "qleft", "qlowcase", "qscan",
+        "qsubstr", "qsysfunc", "qtrim", "quote", "qupcase", "scan",
+        "str", "substr", "superq", "syscall", "sysevalf", "sysexec",
+        "sysfunc", "sysget", "syslput", "sysprod", "sysrc", "sysrput",
+        "then", "to", "trim", "unquote", "until", "upcase", "verify",
+        "while", "window"
+    )
+
+    builtins_conditionals = (
+        "do", "if", "then", "else", "end", "until", "while"
+    )
+
+    builtins_statements = (
+        "abort", "array", "attrib", "by", "call", "cards", "cards4",
+        "catname", "continue", "datalines", "datalines4", "delete", "delim",
+        "delimiter", "display", "dm", "drop", "endsas", "error", "file",
+        "filename", "footnote", "format", "goto", "in", "infile", "informat",
+        "input", "keep", "label", "leave", "length", "libname", "link",
+        "list", "lostcard", "merge", "missing", "modify", "options", "output",
+        "out", "page", "put", "redirect", "remove", "rename", "replace",
+        "retain", "return", "select", "set", "skip", "startsas", "stop",
+        "title", "update", "waitsas", "where", "window", "x", "systask"
+    )
+
+    builtins_sql = (
+        "add", "and", "alter", "as", "cascade", "check", "create",
+        "delete", "describe", "distinct", "drop", "foreign", "from",
+        "group", "having", "index", "insert", "into", "in", "key", "like",
+        "message", "modify", "msgtype", "not", "null", "on", "or",
+        "order", "primary", "references", "reset", "restrict", "select",
+        "set", "table", "unique", "update", "validate", "view", "where"
+    )
+
+    builtins_functions = (
+        "abs", "addr", "airy", "arcos", "arsin", "atan", "attrc",
+        "attrn", "band", "betainv", "blshift", "bnot", "bor",
+        "brshift", "bxor", "byte", "cdf", "ceil", "cexist", "cinv",
+        "close", "cnonct", "collate", "compbl", "compound",
+        "compress", "cos", "cosh", "css", "curobs", "cv", "daccdb",
+        "daccdbsl", "daccsl", "daccsyd", "dacctab", "dairy", "date",
+        "datejul", "datepart", "datetime", "day", "dclose", "depdb",
+        "depdbsl", "depsl", "depsyd",
+        "deptab", "dequote", "dhms", "dif", "digamma",
+        "dim", "dinfo", "dnum", "dopen", "doptname", "doptnum",
+        "dread", "dropnote", "dsname", "erf", "erfc", "exist", "exp",
+        "fappend", "fclose", "fcol", "fdelete", "fetch", "fetchobs",
+        "fexist", "fget", "fileexist", "filename", "fileref",
+        "finfo", "finv", "fipname", "fipnamel", "fipstate", "floor",
+        "fnonct", "fnote", "fopen", "foptname", "foptnum", "fpoint",
+        "fpos", "fput", "fread", "frewind", "frlen", "fsep", "fuzz",
+        "fwrite", "gaminv", "gamma", "getoption", "getvarc", "getvarn",
+        "hbound", "hms", "hosthelp", "hour", "ibessel", "index",
+        "indexc", "indexw", "input", "inputc", "inputn", "int",
+        "intck", "intnx", "intrr", "irr", "jbessel", "juldate",
+        "kurtosis", "lag", "lbound", "left", "length", "lgamma",
+        "libname", "libref", "log", "log10", "log2", "logpdf", "logpmf",
+        "logsdf", "lowcase", "max", "mdy", "mean", "min", "minute",
+        "mod", "month", "mopen", "mort", "n", "netpv", "nmiss",
+        "normal", "note", "npv", "open", "ordinal", "pathname",
+        "pdf", "peek", "peekc", "pmf", "point", "poisson", "poke",
+        "probbeta", "probbnml", "probchi", "probf", "probgam",
+        "probhypr", "probit", "probnegb", "probnorm", "probt",
+        "put", "putc", "putn", "qtr", "quote", "ranbin", "rancau",
+        "ranexp", "rangam", "range", "rank", "rannor", "ranpoi",
+        "rantbl", "rantri", "ranuni", "repeat", "resolve", "reverse",
+        "rewind", "right", "round", "saving", "scan", "sdf", "second",
+        "sign", "sin", "sinh", "skewness", "soundex", "spedis",
+        "sqrt", "std", "stderr", "stfips", "stname", "stnamel",
+        "substr", "sum", "symget", "sysget", "sysmsg", "sysprod",
+        "sysrc", "system", "tan", "tanh", "time", "timepart", "tinv",
+        "tnonct", "today", "translate", "tranwrd", "trigamma",
+        "trim", "trimn", "trunc", "uniform", "upcase", "uss", "var",
+        "varfmt", "varinfmt", "varlabel", "varlen", "varname",
+        "varnum", "varray", "varrayx", "vartype", "verify", "vformat",
+        "vformatd", "vformatdx", "vformatn", "vformatnx", "vformatw",
+        "vformatwx", "vformatx", "vinarray", "vinarrayx", "vinformat",
+        "vinformatd", "vinformatdx", "vinformatn", "vinformatnx",
+        "vinformatw", "vinformatwx", "vinformatx", "vlabel",
+        "vlabelx", "vlength", "vlengthx", "vname", "vnamex", "vtype",
+        "vtypex", "weekday", "year", "yyq", "zipfips", "zipname",
+        "zipnamel", "zipstate"
+    )
+
+    tokens = {
+        'root': [
+            include('comments'),
+            include('proc-data'),
+            include('cards-datalines'),
+            include('logs'),
+            include('general'),
+            (r'.', Text),
+        ],
+        # SAS is multi-line regardless, but * is ended by ;
+        'comments': [
+            (r'^\s*\*.*?;', Comment),
+            (r'/\*.*?\*/', Comment),
+            (r'^\s*\*(.|\n)*?;', Comment.Multiline),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+        ],
+        # Special highlight for proc, data, quit, run
+        'proc-data': [
+            (r'(^|;)\s*(proc \w+|data|run|quit)[\s;]',
+             Keyword.Reserved),
+        ],
+        # Special highlight cards and datalines
+        'cards-datalines': [
+            (r'^\s*(datalines|cards)\s*;\s*$', Keyword, 'data'),
+        ],
+        'data': [
+            (r'(.|\n)*^\s*;\s*$', Other, '#pop'),
+        ],
+        # Special highlight for put NOTE|ERROR|WARNING (order matters)
+        'logs': [
+            (r'\n?^\s*%?put ', Keyword, 'log-messages'),
+        ],
+        'log-messages': [
+            (r'NOTE(:|-).*', Generic, '#pop'),
+            (r'WARNING(:|-).*', Generic.Emph, '#pop'),
+            (r'ERROR(:|-).*', Generic.Error, '#pop'),
+            include('general'),
+        ],
+        'general': [
+            include('keywords'),
+            include('vars-strings'),
+            include('special'),
+            include('numbers'),
+        ],
+        # Keywords, statements, functions, macros
+        'keywords': [
+            (words(builtins_statements,
+                   prefix = r'\b',
+                   suffix = r'\b'),
+             Keyword),
+            (words(builtins_sql,
+                   prefix = r'\b',
+                   suffix = r'\b'),
+             Keyword),
+            (words(builtins_conditionals,
+                   prefix = r'\b',
+                   suffix = r'\b'),
+             Keyword),
+            (words(builtins_macros,
+                   prefix = r'%',
+                   suffix = r'\b'),
+             Name.Builtin),
+            (words(builtins_functions,
+                   prefix = r'\b',
+                   suffix = r'\('),
+             Name.Builtin),
+        ],
+        # Strings and user-defined variables and macros (order matters)
+        'vars-strings': [
+            (r'&[a-z_]\w{0,31}\.?', Name.Variable),
+            (r'%[a-z_]\w{0,31}', Name.Function),
+            (r'\'', String, 'string_squote'),
+            (r'"', String, 'string_dquote'),
+        ],
+        'string_squote': [
+            ('\'', String, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),
+            # AFAIK, macro variables are not evaluated in single quotes
+            # (r'&', Name.Variable, 'validvar'),
+            (r'[^$\'\\]+', String),
+            (r'[$\'\\]', String),
+        ],
+        'string_dquote': [
+            (r'"', String, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),
+            (r'&', Name.Variable, 'validvar'),
+            (r'[^$&"\\]+', String),
+            (r'[$"\\]', String),
+        ],
+        'validvar': [
+            (r'[a-z_]\w{0,31}\.?', Name.Variable, '#pop'),
+        ],
+        # SAS numbers and special variables
+        'numbers': [
+            (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)(E[+-]?[0-9]+)?i?\b',
+             Number),
+        ],
+        'special': [
+            (r'(null|missing|_all_|_automatic_|_character_|_n_|'
+             r'_infile_|_name_|_null_|_numeric_|_user_|_webout_)',
+             Keyword.Constant),
+        ],
+        # 'operators': [
+        #     (r'(-|=|<=|>=|<|>|<>|&|!=|'
+        #      r'\||\*|\+|\^|/|!|~|~=)', Operator)
+        # ],
+    }
diff --git a/lib/pygments/lexers/savi.py b/lib/pygments/lexers/savi.py
new file mode 100644
index 0000000..1e443ae
--- /dev/null
+++ b/lib/pygments/lexers/savi.py
@@ -0,0 +1,171 @@
+"""
+    pygments.lexers.savi
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Savi.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Whitespace, Keyword, Name, String, Number, \
+  Operator, Punctuation, Comment, Generic, Error
+
+__all__ = ['SaviLexer']
+
+
+# The canonical version of this file can be found in the following repository,
+# where it is kept in sync with any language changes, as well as the other
+# pygments-like lexers that are maintained for use with other tools:
+# - https://github.com/savi-lang/savi/blob/main/tooling/pygments/lexers/savi.py
+#
+# If you're changing this file in the pygments repository, please ensure that
+# any changes you make are also propagated to the official Savi repository,
+# in order to avoid accidental clobbering of your changes later when an update
+# from the Savi repository flows forward into the pygments repository.
+#
+# If you're changing this file in the Savi repository, please ensure that
+# any changes you make are also reflected in the other pygments-like lexers
+# (rouge, vscode, etc) so that all of the lexers can be kept cleanly in sync.
+
+class SaviLexer(RegexLexer):
+    """
+    For Savi source code.
+
+    .. versionadded: 2.10
+    """
+
+    name = 'Savi'
+    url = 'https://github.com/savi-lang/savi'
+    aliases = ['savi']
+    filenames = ['*.savi']
+    version_added = ''
+
+    tokens = {
+      "root": [
+        # Line Comment
+        (r'//.*?$', Comment.Single),
+
+        # Doc Comment
+        (r'::.*?$', Comment.Single),
+
+        # Capability Operator
+        (r'(\')(\w+)(?=[^\'])', bygroups(Operator, Name)),
+
+        # Double-Quote String
+        (r'\w?"', String.Double, "string.double"),
+
+        # Single-Char String
+        (r"'", String.Char, "string.char"),
+
+        # Type Name
+        (r'(_?[A-Z]\w*)', Name.Class),
+
+        # Nested Type Name
+        (r'(\.)(\s*)(_?[A-Z]\w*)', bygroups(Punctuation, Whitespace, Name.Class)),
+
+        # Declare
+        (r'^([ \t]*)(:\w+)',
+          bygroups(Whitespace, Name.Tag),
+          "decl"),
+
+        # Error-Raising Calls/Names
+        (r'((\w+|\+|\-|\*)\!)', Generic.Deleted),
+
+        # Numeric Values
+        (r'\b\d([\d_]*(\.[\d_]+)?)\b', Number),
+
+        # Hex Numeric Values
+        (r'\b0x([0-9a-fA-F_]+)\b', Number.Hex),
+
+        # Binary Numeric Values
+        (r'\b0b([01_]+)\b', Number.Bin),
+
+        # Function Call (with braces)
+        (r'\w+(?=\()', Name.Function),
+
+        # Function Call (with receiver)
+        (r'(\.)(\s*)(\w+)', bygroups(Punctuation, Whitespace, Name.Function)),
+
+        # Function Call (with self receiver)
+        (r'(@)(\w+)', bygroups(Punctuation, Name.Function)),
+
+        # Parenthesis
+        (r'\(', Punctuation, "root"),
+        (r'\)', Punctuation, "#pop"),
+
+        # Brace
+        (r'\{', Punctuation, "root"),
+        (r'\}', Punctuation, "#pop"),
+
+        # Bracket
+        (r'\[', Punctuation, "root"),
+        (r'(\])(\!)', bygroups(Punctuation, Generic.Deleted), "#pop"),
+        (r'\]', Punctuation, "#pop"),
+
+        # Punctuation
+        (r'[,;:\.@]', Punctuation),
+
+        # Piping Operators
+        (r'(\|\>)', Operator),
+
+        # Branching Operators
+        (r'(\&\&|\|\||\?\?|\&\?|\|\?|\.\?)', Operator),
+
+        # Comparison Operators
+        (r'(\<\=\>|\=\~|\=\=|\<\=|\>\=|\<|\>)', Operator),
+
+        # Arithmetic Operators
+        (r'(\+|\-|\/|\*|\%)', Operator),
+
+        # Assignment Operators
+        (r'(\=)', Operator),
+
+        # Other Operators
+        (r'(\!|\<\<|\<|\&|\|)', Operator),
+
+        # Identifiers
+        (r'\b\w+\b', Name),
+
+        # Whitespace
+        (r'[ \t\r]+\n*|\n+', Whitespace),
+      ],
+
+      # Declare (nested rules)
+      "decl": [
+        (r'\b[a-z_]\w*\b(?!\!)', Keyword.Declaration),
+        (r':', Punctuation, "#pop"),
+        (r'\n', Whitespace, "#pop"),
+        include("root"),
+      ],
+
+      # Double-Quote String (nested rules)
+      "string.double": [
+        (r'\\\(', String.Interpol, "string.interpolation"),
+        (r'\\u[0-9a-fA-F]{4}', String.Escape),
+        (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        (r'\\[bfnrt\\\']', String.Escape),
+        (r'\\"', String.Escape),
+        (r'"', String.Double, "#pop"),
+        (r'[^\\"]+', String.Double),
+        (r'.', Error),
+      ],
+
+      # Single-Char String (nested rules)
+      "string.char": [
+        (r'\\u[0-9a-fA-F]{4}', String.Escape),
+        (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        (r'\\[bfnrt\\\']', String.Escape),
+        (r"\\'", String.Escape),
+        (r"'", String.Char, "#pop"),
+        (r"[^\\']+", String.Char),
+        (r'.', Error),
+      ],
+
+      # Interpolation inside String (nested rules)
+      "string.interpolation": [
+        (r"\)", String.Interpol, "#pop"),
+        include("root"),
+      ]
+    }
diff --git a/lib/pygments/lexers/scdoc.py b/lib/pygments/lexers/scdoc.py
new file mode 100644
index 0000000..8e850d0
--- /dev/null
+++ b/lib/pygments/lexers/scdoc.py
@@ -0,0 +1,85 @@
+"""
+    pygments.lexers.scdoc
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for scdoc, a simple man page generator.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this
+from pygments.token import Text, Comment, Keyword, String, Generic
+
+__all__ = ['ScdocLexer']
+
+
+class ScdocLexer(RegexLexer):
+    """
+    `scdoc` is a simple man page generator for POSIX systems written in C99.
+    """
+    name = 'scdoc'
+    url = 'https://git.sr.ht/~sircmpwn/scdoc'
+    aliases = ['scdoc', 'scd']
+    filenames = ['*.scd', '*.scdoc']
+    version_added = '2.5'
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            # comment
+            (r'^(;.+\n)', bygroups(Comment)),
+
+            # heading with pound prefix
+            (r'^(#)([^#].+\n)', bygroups(Generic.Heading, Text)),
+            (r'^(#{2})(.+\n)', bygroups(Generic.Subheading, Text)),
+            # bulleted lists
+            (r'^(\s*)([*-])(\s)(.+\n)',
+            bygroups(Text, Keyword, Text, using(this, state='inline'))),
+            # numbered lists
+            (r'^(\s*)(\.+\.)( .+\n)',
+            bygroups(Text, Keyword, using(this, state='inline'))),
+            # quote
+            (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)),
+            # text block
+            (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)),
+
+            include('inline'),
+        ],
+        'inline': [
+            # escape
+            (r'\\.', Text),
+            # underlines
+            (r'(\s)(_[^_]+_)(\W|\n)', bygroups(Text, Generic.Emph, Text)),
+            # bold
+            (r'(\s)(\*[^*]+\*)(\W|\n)', bygroups(Text, Generic.Strong, Text)),
+            # inline code
+            (r'`[^`]+`', String.Backtick),
+
+            # general text, must come last!
+            (r'[^\\\s]+', Text),
+            (r'.', Text),
+        ],
+    }
+
+    def analyse_text(text):
+        """We checks for bold and underline text with * and _. Also
+        every scdoc file must start with a strictly defined first line."""
+        result = 0
+
+        if '*' in text:
+            result += 0.01
+
+        if '_' in text:
+            result += 0.01
+
+        # name(section) ["left_footer" ["center_header"]]
+        first_line = text.partition('\n')[0]
+        scdoc_preamble_pattern = r'^.*\([1-7]\)( "[^"]+"){0,2}$'
+
+        if re.search(scdoc_preamble_pattern, first_line):
+            result += 0.5
+
+        return result
diff --git a/lib/pygments/lexers/scripting.py b/lib/pygments/lexers/scripting.py
new file mode 100644
index 0000000..6e494c3
--- /dev/null
+++ b/lib/pygments/lexers/scripting.py
@@ -0,0 +1,1616 @@
+"""
+    pygments.lexers.scripting
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for scripting and embedded languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default, combined, \
+    words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error, Whitespace, Other
+from pygments.util import get_bool_opt, get_list_opt
+
+__all__ = ['LuaLexer', 'LuauLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLexer',
+           'AppleScriptLexer', 'RexxLexer', 'MOOCodeLexer', 'HybrisLexer',
+           'EasytrieveLexer', 'JclLexer', 'MiniScriptLexer']
+
+
+def all_lua_builtins():
+    from pygments.lexers._lua_builtins import MODULES
+    return [w for values in MODULES.values() for w in values]
+
+class LuaLexer(RegexLexer):
+    """
+    For Lua source code.
+
+    Additional options accepted:
+
+    `func_name_highlighting`
+        If given and ``True``, highlight builtin function names
+        (default: ``True``).
+    `disabled_modules`
+        If given, must be a list of module names whose function names
+        should not be highlighted. By default all modules are highlighted.
+
+        To get a list of allowed modules have a look into the
+        `_lua_builtins` module:
+
+        .. sourcecode:: pycon
+
+            >>> from pygments.lexers._lua_builtins import MODULES
+            >>> MODULES.keys()
+            ['string', 'coroutine', 'modules', 'io', 'basic', ...]
+    """
+
+    name = 'Lua'
+    url = 'https://www.lua.org/'
+    aliases = ['lua']
+    filenames = ['*.lua', '*.wlua']
+    mimetypes = ['text/x-lua', 'application/x-lua']
+    version_added = ''
+
+    _comment_multiline = r'(?:--\[(?P=*)\[[\w\W]*?\](?P=level)\])'
+    _comment_single = r'(?:--.*$)'
+    _space = r'(?:\s+(?!\s))'
+    _s = rf'(?:{_comment_multiline}|{_comment_single}|{_space})'
+    _name = r'(?:[^\W\d]\w*)'
+
+    tokens = {
+        'root': [
+            # Lua allows a file to start with a shebang.
+            (r'#!.*', Comment.Preproc),
+            default('base'),
+        ],
+        'ws': [
+            (_comment_multiline, Comment.Multiline),
+            (_comment_single, Comment.Single),
+            (_space, Whitespace),
+        ],
+        'base': [
+            include('ws'),
+
+            (r'(?i)0x[\da-f]*(\.[\da-f]*)?(p[+-]?\d+)?', Number.Hex),
+            (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
+            (r'(?i)\d+e[+-]?\d+', Number.Float),
+            (r'\d+', Number.Integer),
+
+            # multiline strings
+            (r'(?s)\[(=*)\[.*?\]\1\]', String),
+
+            (r'::', Punctuation, 'label'),
+            (r'\.{3}', Punctuation),
+            (r'[=<>|~&+\-*/%#^]+|\.\.', Operator),
+            (r'[\[\]{}().,:;]+', Punctuation),
+            (r'(and|or|not)\b', Operator.Word),
+
+            (words([
+                'break', 'do', 'else', 'elseif', 'end', 'for', 'if', 'in',
+                'repeat', 'return', 'then', 'until', 'while'
+            ], suffix=r'\b'), Keyword.Reserved),
+            (r'goto\b', Keyword.Reserved, 'goto'),
+            (r'(local)\b', Keyword.Declaration),
+            (r'(true|false|nil)\b', Keyword.Constant),
+
+            (r'(function)\b', Keyword.Reserved, 'funcname'),
+
+            (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
+            (fr'[A-Za-z_]\w*(?={_s}*[.:])', Name.Variable, 'varname'),
+            (fr'[A-Za-z_]\w*(?={_s}*\()', Name.Function),
+            (r'[A-Za-z_]\w*', Name.Variable),
+
+            ("'", String.Single, combined('stringescape', 'sqs')),
+            ('"', String.Double, combined('stringescape', 'dqs'))
+        ],
+
+        'varname': [
+            include('ws'),
+            (r'\.\.', Operator, '#pop'),
+            (r'[.:]', Punctuation),
+            (rf'{_name}(?={_s}*[.:])', Name.Property),
+            (rf'{_name}(?={_s}*\()', Name.Function, '#pop'),
+            (_name, Name.Property, '#pop'),
+        ],
+
+        'funcname': [
+            include('ws'),
+            (r'[.:]', Punctuation),
+            (rf'{_name}(?={_s}*[.:])', Name.Class),
+            (_name, Name.Function, '#pop'),
+            # inline function
+            (r'\(', Punctuation, '#pop'),
+        ],
+
+        'goto': [
+            include('ws'),
+            (_name, Name.Label, '#pop'),
+        ],
+
+        'label': [
+            include('ws'),
+            (r'::', Punctuation, '#pop'),
+            (_name, Name.Label),
+        ],
+
+        'stringescape': [
+            (r'\\([abfnrtv\\"\']|[\r\n]{1,2}|z\s*|x[0-9a-fA-F]{2}|\d{1,3}|'
+             r'u\{[0-9a-fA-F]+\})', String.Escape),
+        ],
+
+        'sqs': [
+            (r"'", String.Single, '#pop'),
+            (r"[^\\']+", String.Single),
+        ],
+
+        'dqs': [
+            (r'"', String.Double, '#pop'),
+            (r'[^\\"]+', String.Double),
+        ]
+    }
+
+    def __init__(self, **options):
+        self.func_name_highlighting = get_bool_opt(
+            options, 'func_name_highlighting', True)
+        self.disabled_modules = get_list_opt(options, 'disabled_modules', [])
+
+        self._functions = set()
+        if self.func_name_highlighting:
+            from pygments.lexers._lua_builtins import MODULES
+            for mod, func in MODULES.items():
+                if mod not in self.disabled_modules:
+                    self._functions.update(func)
+        RegexLexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name.Builtin and value not in self._functions:
+                if '.' in value:
+                    a, b = value.split('.')
+                    yield index, Name, a
+                    yield index + len(a), Punctuation, '.'
+                    yield index + len(a) + 1, Name, b
+                else:
+                    yield index, Name, value
+                continue
+            yield index, token, value
+
+def _luau_make_expression(should_pop, _s):
+    temp_list = [
+        (r'0[xX][\da-fA-F_]*', Number.Hex, '#pop'),
+        (r'0[bB][\d_]*', Number.Bin, '#pop'),
+        (r'\.?\d[\d_]*(?:\.[\d_]*)?(?:[eE][+-]?[\d_]+)?', Number.Float, '#pop'),
+
+        (words((
+            'true', 'false', 'nil'
+        ), suffix=r'\b'), Keyword.Constant, '#pop'),
+
+        (r'\[(=*)\[[.\n]*?\]\1\]', String, '#pop'),
+
+        (r'(\.)([a-zA-Z_]\w*)(?=%s*[({"\'])', bygroups(Punctuation, Name.Function), '#pop'),
+        (r'(\.)([a-zA-Z_]\w*)', bygroups(Punctuation, Name.Variable), '#pop'),
+
+        (rf'[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*(?={_s}*[({{"\'])', Name.Other, '#pop'),
+        (r'[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*', Name, '#pop'),
+    ]
+    if should_pop:
+        return temp_list
+    return [entry[:2] for entry in temp_list]
+
+def _luau_make_expression_special(should_pop):
+    temp_list = [
+        (r'\{', Punctuation, ('#pop', 'closing_brace_base', 'expression')),
+        (r'\(', Punctuation, ('#pop', 'closing_parenthesis_base', 'expression')),
+
+        (r'::?', Punctuation, ('#pop', 'type_end', 'type_start')),
+
+        (r"'", String.Single, ('#pop', 'string_single')),
+        (r'"', String.Double, ('#pop', 'string_double')),
+        (r'`', String.Backtick, ('#pop', 'string_interpolated')),
+    ]
+    if should_pop:
+        return temp_list
+    return [(entry[0], entry[1], entry[2][1:]) for entry in temp_list]
+
+class LuauLexer(RegexLexer):
+    """
+    For Luau source code.
+
+    Additional options accepted:
+
+    `include_luau_builtins`
+        If given and ``True``, automatically highlight Luau builtins
+        (default: ``True``).
+    `include_roblox_builtins`
+        If given and ``True``, automatically highlight Roblox-specific builtins
+        (default: ``False``).
+    `additional_builtins`
+        If given, must be a list of additional builtins to highlight.
+    `disabled_builtins`
+        If given, must be a list of builtins that will not be highlighted.
+    """
+
+    name = 'Luau'
+    url = 'https://luau-lang.org/'
+    aliases = ['luau']
+    filenames = ['*.luau']
+    version_added = '2.18'
+
+    _comment_multiline = r'(?:--\[(?P=*)\[[\w\W]*?\](?P=level)\])'
+    _comment_single = r'(?:--.*$)'
+    _s = r'(?:{}|{}|{})'.format(_comment_multiline, _comment_single, r'\s+')
+
+    tokens = {
+        'root': [
+            (r'#!.*', Comment.Hashbang, 'base'),
+            default('base'),
+        ],
+
+        'ws': [
+            (_comment_multiline, Comment.Multiline),
+            (_comment_single, Comment.Single),
+            (r'\s+', Whitespace),
+        ],
+
+        'base': [
+            include('ws'),
+
+            *_luau_make_expression_special(False),
+            (r'\.\.\.', Punctuation),
+
+            (rf'type\b(?={_s}+[a-zA-Z_])', Keyword.Reserved, 'type_declaration'),
+            (rf'export\b(?={_s}+[a-zA-Z_])', Keyword.Reserved),
+
+            (r'(?:\.\.|//|[+\-*\/%^<>=])=?', Operator, 'expression'),
+            (r'~=', Operator, 'expression'),
+
+            (words((
+                'and', 'or', 'not'
+            ), suffix=r'\b'), Operator.Word, 'expression'),
+
+            (words((
+                'elseif', 'for', 'if', 'in', 'repeat', 'return', 'until',
+                'while'), suffix=r'\b'), Keyword.Reserved, 'expression'),
+            (r'local\b', Keyword.Declaration, 'expression'),
+
+            (r'function\b', Keyword.Reserved, ('expression', 'func_name')),
+
+            (r'[\])};]+', Punctuation),
+
+            include('expression_static'),
+            *_luau_make_expression(False, _s),
+
+            (r'[\[.,]', Punctuation, 'expression'),
+        ],
+        'expression_static': [
+            (words((
+                'break', 'continue', 'do', 'else', 'elseif', 'end', 'for',
+                'if', 'in', 'repeat', 'return', 'then', 'until', 'while'),
+                suffix=r'\b'), Keyword.Reserved),
+        ],
+        'expression': [
+            include('ws'),
+
+            (r'if\b', Keyword.Reserved, ('ternary', 'expression')),
+
+            (r'local\b', Keyword.Declaration),
+            *_luau_make_expression_special(True),
+            (r'\.\.\.', Punctuation, '#pop'),
+
+            (r'function\b', Keyword.Reserved, 'func_name'),
+
+            include('expression_static'),
+            *_luau_make_expression(True, _s),
+
+            default('#pop'),
+        ],
+        'ternary': [
+            include('ws'),
+
+            (r'else\b', Keyword.Reserved, '#pop'),
+            (words((
+                'then', 'elseif',
+            ), suffix=r'\b'), Operator.Reserved, 'expression'),
+
+            default('#pop'),
+        ],
+
+        'closing_brace_pop': [
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'closing_parenthesis_pop': [
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'closing_gt_pop': [
+            (r'>', Punctuation, '#pop'),
+        ],
+
+        'closing_parenthesis_base': [
+            include('closing_parenthesis_pop'),
+            include('base'),
+        ],
+        'closing_parenthesis_type': [
+            include('closing_parenthesis_pop'),
+            include('type'),
+        ],
+        'closing_brace_base': [
+            include('closing_brace_pop'),
+            include('base'),
+        ],
+        'closing_brace_type': [
+            include('closing_brace_pop'),
+            include('type'),
+        ],
+        'closing_gt_type': [
+            include('closing_gt_pop'),
+            include('type'),
+        ],
+
+        'string_escape': [
+            (r'\\z\s*', String.Escape),
+            (r'\\(?:[abfnrtvz\\"\'`\{\n])|[\r\n]{1,2}|x[\da-fA-F]{2}|\d{1,3}|'
+             r'u\{\}[\da-fA-F]*\}', String.Escape),
+        ],
+        'string_single': [
+            include('string_escape'),
+
+            (r"'", String.Single, "#pop"),
+            (r"[^\\']+", String.Single),
+        ],
+        'string_double': [
+            include('string_escape'),
+
+            (r'"', String.Double, "#pop"),
+            (r'[^\\"]+', String.Double),
+        ],
+        'string_interpolated': [
+            include('string_escape'),
+
+            (r'\{', Punctuation, ('closing_brace_base', 'expression')),
+
+            (r'`', String.Backtick, "#pop"),
+            (r'[^\\`\{]+', String.Backtick),
+        ],
+
+        'func_name': [
+            include('ws'),
+
+            (r'[.:]', Punctuation),
+            (rf'[a-zA-Z_]\w*(?={_s}*[.:])', Name.Class),
+            (r'[a-zA-Z_]\w*', Name.Function),
+
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            (r'\(', Punctuation, '#pop'),
+        ],
+
+        'type': [
+            include('ws'),
+
+            (r'\(', Punctuation, 'closing_parenthesis_type'),
+            (r'\{', Punctuation, 'closing_brace_type'),
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            (r"'", String.Single, 'string_single'),
+            (r'"', String.Double, 'string_double'),
+
+            (r'[|&\.,\[\]:=]+', Punctuation),
+            (r'->', Punctuation),
+
+            (r'typeof\(', Name.Builtin, ('closing_parenthesis_base',
+                                         'expression')),
+            (r'[a-zA-Z_]\w*', Name.Class),
+        ],
+        'type_start': [
+            include('ws'),
+
+            (r'\(', Punctuation, ('#pop', 'closing_parenthesis_type')),
+            (r'\{', Punctuation, ('#pop', 'closing_brace_type')),
+            (r'<', Punctuation, ('#pop', 'closing_gt_type')),
+
+            (r"'", String.Single, ('#pop', 'string_single')),
+            (r'"', String.Double, ('#pop', 'string_double')),
+
+            (r'typeof\(', Name.Builtin, ('#pop', 'closing_parenthesis_base',
+                                         'expression')),
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+        ],
+        'type_end': [
+            include('ws'),
+
+            (r'[|&\.]', Punctuation, 'type_start'),
+            (r'->', Punctuation, 'type_start'),
+
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            default('#pop'),
+        ],
+        'type_declaration': [
+            include('ws'),
+
+            (r'[a-zA-Z_]\w*', Name.Class),
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            (r'=', Punctuation, ('#pop', 'type_end', 'type_start')),
+        ],
+    }
+
+    def __init__(self, **options):
+        self.include_luau_builtins = get_bool_opt(
+            options, 'include_luau_builtins', True)
+        self.include_roblox_builtins = get_bool_opt(
+            options, 'include_roblox_builtins', False)
+        self.additional_builtins = get_list_opt(options, 'additional_builtins', [])
+        self.disabled_builtins = get_list_opt(options, 'disabled_builtins', [])
+
+        self._builtins = set(self.additional_builtins)
+        if self.include_luau_builtins:
+            from pygments.lexers._luau_builtins import LUAU_BUILTINS
+            self._builtins.update(LUAU_BUILTINS)
+        if self.include_roblox_builtins:
+            from pygments.lexers._luau_builtins import ROBLOX_BUILTINS
+            self._builtins.update(ROBLOX_BUILTINS)
+        if self.additional_builtins:
+            self._builtins.update(self.additional_builtins)
+        self._builtins.difference_update(self.disabled_builtins)
+
+        RegexLexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name or token is Name.Other:
+                split_value = value.split('.')
+                complete_value = []
+                new_index = index
+                for position in range(len(split_value), 0, -1):
+                    potential_string = '.'.join(split_value[:position])
+                    if potential_string in self._builtins:
+                        yield index, Name.Builtin, potential_string
+                        new_index += len(potential_string)
+
+                        if complete_value:
+                            yield new_index, Punctuation, '.'
+                            new_index += 1
+                        break
+                    complete_value.insert(0, split_value[position - 1])
+
+                for position, substring in enumerate(complete_value):
+                    if position + 1 == len(complete_value):
+                        if token is Name:
+                            yield new_index, Name.Variable, substring
+                            continue
+                        yield new_index, Name.Function, substring
+                        continue
+                    yield new_index, Name.Variable, substring
+                    new_index += len(substring)
+                    yield new_index, Punctuation, '.'
+                    new_index += 1
+
+                continue
+            yield index, token, value
+
+class MoonScriptLexer(LuaLexer):
+    """
+    For MoonScript source code.
+    """
+
+    name = 'MoonScript'
+    url = 'http://moonscript.org'
+    aliases = ['moonscript', 'moon']
+    filenames = ['*.moon']
+    mimetypes = ['text/x-moonscript', 'application/x-moonscript']
+    version_added = '1.5'
+
+    tokens = {
+        'root': [
+            (r'#!(.*?)$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            ('--.*$', Comment.Single),
+            (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
+            (r'(?i)\d+e[+-]?\d+', Number.Float),
+            (r'(?i)0x[0-9a-f]*', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'\n', Whitespace),
+            (r'[^\S\n]+', Text),
+            (r'(?s)\[(=*)\[.*?\]\1\]', String),
+            (r'(->|=>)', Name.Function),
+            (r':[a-zA-Z_]\w*', Name.Variable),
+            (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator),
+            (r'[;,]', Punctuation),
+            (r'[\[\]{}()]', Keyword.Type),
+            (r'[a-zA-Z_]\w*:', Name.Variable),
+            (words((
+                'class', 'extends', 'if', 'then', 'super', 'do', 'with',
+                'import', 'export', 'while', 'elseif', 'return', 'for', 'in',
+                'from', 'when', 'using', 'else', 'and', 'or', 'not', 'switch',
+                'break'), suffix=r'\b'),
+             Keyword),
+            (r'(true|false|nil)\b', Keyword.Constant),
+            (r'(and|or|not)\b', Operator.Word),
+            (r'(self)\b', Name.Builtin.Pseudo),
+            (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class),
+            (r'[A-Z]\w*', Name.Class),  # proper name
+            (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
+            (r'[A-Za-z_]\w*', Name),
+            ("'", String.Single, combined('stringescape', 'sqs')),
+            ('"', String.Double, combined('stringescape', 'dqs'))
+        ],
+        'stringescape': [
+            (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape)
+        ],
+        'sqs': [
+            ("'", String.Single, '#pop'),
+            ("[^']+", String)
+        ],
+        'dqs': [
+            ('"', String.Double, '#pop'),
+            ('[^"]+', String)
+        ]
+    }
+
+    def get_tokens_unprocessed(self, text):
+        # set . as Operator instead of Punctuation
+        for index, token, value in LuaLexer.get_tokens_unprocessed(self, text):
+            if token == Punctuation and value == ".":
+                token = Operator
+            yield index, token, value
+
+
+class ChaiscriptLexer(RegexLexer):
+    """
+    For ChaiScript source code.
+    """
+
+    name = 'ChaiScript'
+    url = 'http://chaiscript.com/'
+    aliases = ['chaiscript', 'chai']
+    filenames = ['*.chai']
+    mimetypes = ['text/x-chaiscript', 'application/x-chaiscript']
+    version_added = '2.0'
+
+    flags = re.DOTALL | re.MULTILINE
+
+    tokens = {
+        'commentsandwhitespace': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'^\#.*?\n', Comment.Single)
+        ],
+        'slashstartsregex': [
+            include('commentsandwhitespace'),
+            (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+             r'([gim]+\b|\B)', String.Regex, '#pop'),
+            (r'(?=/)', Text, ('#pop', 'badregex')),
+            default('#pop')
+        ],
+        'badregex': [
+            (r'\n', Text, '#pop')
+        ],
+        'root': [
+            include('commentsandwhitespace'),
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|\.\.'
+             r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+            (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+            (r'[})\].]', Punctuation),
+            (r'[=+\-*/]', Operator),
+            (r'(for|in|while|do|break|return|continue|if|else|'
+             r'throw|try|catch'
+             r')\b', Keyword, 'slashstartsregex'),
+            (r'(var)\b', Keyword.Declaration, 'slashstartsregex'),
+            (r'(attr|def|fun)\b', Keyword.Reserved),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'(eval|throw)\b', Name.Builtin),
+            (r'`\S+`', Name.Builtin),
+            (r'[$a-zA-Z_]\w*', Name.Other),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[0-9]+', Number.Integer),
+            (r'"', String.Double, 'dqstring'),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+        ],
+        'dqstring': [
+            (r'\$\{[^"}]+?\}', String.Interpol),
+            (r'\$', String.Double),
+            (r'\\\\', String.Double),
+            (r'\\"', String.Double),
+            (r'[^\\"$]+', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+    }
+
+
+class LSLLexer(RegexLexer):
+    """
+    For Second Life's Linden Scripting Language source code.
+    """
+
+    name = 'LSL'
+    aliases = ['lsl']
+    filenames = ['*.lsl']
+    mimetypes = ['text/x-lsl']
+    url = 'https://wiki.secondlife.com/wiki/Linden_Scripting_Language'
+    version_added = '2.0'
+
+    flags = re.MULTILINE
+
+    lsl_keywords = r'\b(?:do|else|for|if|jump|return|while)\b'
+    lsl_types = r'\b(?:float|integer|key|list|quaternion|rotation|string|vector)\b'
+    lsl_states = r'\b(?:(?:state)\s+\w+|default)\b'
+    lsl_events = r'\b(?:state_(?:entry|exit)|touch(?:_(?:start|end))?|(?:land_)?collision(?:_(?:start|end))?|timer|listen|(?:no_)?sensor|control|(?:not_)?at_(?:rot_)?target|money|email|run_time_permissions|changed|attach|dataserver|moving_(?:start|end)|link_message|(?:on|object)_rez|remote_data|http_re(?:sponse|quest)|path_update|transaction_result)\b'
+    lsl_functions_builtin = r'\b(?:ll(?:ReturnObjectsBy(?:ID|Owner)|Json(?:2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(?:Mag|Norm|Dist)|Rot(?:Between|2(?:Euler|Fwd|Left|Up))|(?:Euler|Axes)2Rot|Whisper|(?:Region|Owner)?Say|Shout|Listen(?:Control|Remove)?|Sensor(?:Repeat|Remove)?|Detected(?:Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|(?:[GS]et)(?:AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(?:Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(?:Scale|Offset|Rotate)Texture|(?:Rot)?Target(?:Remove)?|(?:Stop)?MoveToTarget|Apply(?:Rotational)?Impulse|Set(?:KeyframedMotion|ContentType|RegionPos|(?:Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(?:Queueing|Radius)|Vehicle(?:Type|(?:Float|Vector|Rotation)Param)|(?:Touch|Sit)?Text|Camera(?:Eye|At)Offset|PrimitiveParams|ClickAction|Link(?:Alpha|Color|PrimitiveParams(?:Fast)?|Texture(?:Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get(?:(?:Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(?:PrimitiveParams|Number(?:OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(?:Details|PermMask|PrimCount)|Parcel(?:MaxPrims|Details|Prim(?:Count|Owners))|Attached|(?:SPMax|Free|Used)Memory|Region(?:Name|TimeDilation|FPS|Corner|AgentCount)|Root(?:Position|Rotation)|UnixTime|(?:Parcel|Region)Flags|(?:Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(?:Prims|NotecardLines|Sides)|Animation(?:List)?|(?:Camera|Local)(?:Pos|Rot)|Vel|Accel|Omega|Time(?:stamp|OfDay)|(?:Object|CenterOf)?Mass|MassMKS|Energy|Owner|(?:Owner)?Key|SunDirection|Texture(?:Offset|Scale|Rot)|Inventory(?:Number|Name|Key|Type|Creator|PermMask)|Permissions(?:Key)?|StartParameter|List(?:Length|EntryType)|Date|Agent(?:Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(?:Name|State))|(?:Get|Reset|GetAndReset)Time|PlaySound(?:Slave)?|LoopSound(?:Master|Slave)?|(?:Trigger|Stop|Preload)Sound|(?:(?:Get|Delete)Sub|Insert)String|To(?:Upper|Lower)|Give(?:InventoryList|Money)|RezObject|(?:Stop)?LookAt|Sleep|CollisionFilter|(?:Take|Release)Controls|DetachFromAvatar|AttachToAvatar(?:Temp)?|InstantMessage|(?:GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(?:Length|Trim)|(?:Start|Stop)Animation|TargetOmega|RequestPermissions|(?:Create|Break)Link|BreakAllLinks|(?:Give|Remove)Inventory|Water|PassTouches|Request(?:Agent|Inventory)Data|TeleportAgent(?:Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(?:Axis|Angle)|A(?:cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(?:CSV|Integer|Json|Float|String|Key|Vector|Rot|List(?:Strided)?)|DeleteSubList|List(?:Statistics|Sort|Randomize|(?:Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(?:CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(?:Slope|Normal|Contour)|GroundRepel|(?:Set|Remove)VehicleFlags|(?:AvatarOn)?(?:Link)?SitTarget|Script(?:Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(?:Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(?:Integer|String)ToBase64|XorBase64|Log(?:10)?|Base64To(?:String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(?:Load|Release|(?:E|Une)scape)URL|ParcelMedia(?:CommandList|Query)|ModPow|MapDestination|(?:RemoveFrom|AddTo|Reset)Land(?:Pass|Ban)List|(?:Set|Clear)CameraParams|HTTP(?:Request|Response)|TextBox|DetectedTouch(?:UV|Face|Pos|(?:N|Bin)ormal|ST)|(?:MD5|SHA1|DumpList2)String|Request(?:Secure)?URL|Clear(?:Prim|Link)Media|(?:Link)?ParticleSystem|(?:Get|Request)(?:Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(?:Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\b'
+    lsl_constants_float = r'\b(?:DEG_TO_RAD|PI(?:_BY_TWO)?|RAD_TO_DEG|SQRT2|TWO_PI)\b'
+    lsl_constants_integer = r'\b(?:JSON_APPEND|STATUS_(?:PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(?:_OBJECT)?|(?:DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(?:FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(?:_(?:BY_(?:LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(?:PARCEL(?:_OWNER)?|REGION)))?|CAMERA_(?:PITCH|DISTANCE|BEHINDNESS_(?:ANGLE|LAG)|(?:FOCUS|POSITION)(?:_(?:THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(?:ROOT|SET|ALL_(?:OTHERS|CHILDREN)|THIS)|ACTIVE|PASSIVE|SCRIPTED|CONTROL_(?:FWD|BACK|(?:ROT_)?(?:LEFT|RIGHT)|UP|DOWN|(?:ML_)?LBUTTON)|PERMISSION_(?:RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(?:CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(?:TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(?:INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(?:_START)?|TELEPORT|MEDIA)|OBJECT_(?:(?:PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_ON_REZ|NAME|DESC|POS|PRIM_EQUIVALENCE|RETURN_(?:PARCEL(?:_OWNER)?|REGION)|ROO?T|VELOCITY|OWNER|GROUP|CREATOR|ATTACHED_POINT|RENDER_WEIGHT|PATHFINDING_TYPE|(?:RUNNING|TOTAL)_SCRIPT_COUNT|SCRIPT_(?:MEMORY|TIME))|TYPE_(?:INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(?:DEBUG|PUBLIC)_CHANNEL|ATTACH_(?:AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](?:SHOULDER|HAND|FOOT|EAR|EYE|[UL](?:ARM|LEG)|HIP)|(?:LEFT|RIGHT)_PEC|HUD_(?:CENTER_[12]|TOP_(?:RIGHT|CENTER|LEFT)|BOTTOM(?:_(?:RIGHT|LEFT))?))|LAND_(?:LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(?:ONLINE|NAME|BORN|SIM_(?:POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(?:ON_FILE|USED)|REMOTE_DATA_(?:CHANNEL|REQUEST|REPLY)|PSYS_(?:PART_(?:BF_(?:ZERO|ONE(?:_MINUS_(?:DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(?:START|END)_(?:COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(?:RIBBON|WIND|INTERP_(?:COLOR|SCALE)|BOUNCE|FOLLOW_(?:SRC|VELOCITY)|TARGET_(?:POS|LINEAR)|EMISSIVE)_MASK)|SRC_(?:MAX_AGE|PATTERN|ANGLE_(?:BEGIN|END)|BURST_(?:RATE|PART_COUNT|RADIUS|SPEED_(?:MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(?:DROP|EXPLODE|ANGLE(?:_CONE(?:_EMPTY)?)?)))|VEHICLE_(?:REFERENCE_FRAME|TYPE_(?:NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(?:LINEAR|ANGULAR)_(?:FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(?:HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(?:LINEAR|ANGULAR)_(?:DEFLECTION_(?:EFFICIENCY|TIMESCALE)|MOTOR_(?:DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(?:EFFICIENCY|TIMESCALE)|BANKING_(?:EFFICIENCY|MIX|TIMESCALE)|FLAG_(?:NO_DEFLECTION_UP|LIMIT_(?:ROLL_ONLY|MOTOR_UP)|HOVER_(?:(?:WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(?:STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(?:TYPE(?:_(?:BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(?:DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(?:_(?:STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(?:NONE|LOW|MEDIUM|HIGH)|BUMP_(?:NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(?:DEFAULT|PLANAR)|SCULPT_(?:TYPE_(?:SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(?:MIRROR|INVERT))|PHYSICS(?:_(?:SHAPE_(?:CONVEX|NONE|PRIM|TYPE)))?|(?:POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(?:ALT_IMAGE_ENABLE|CONTROLS|(?:CURRENT|HOME)_URL|AUTO_(?:LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(?:WIDTH|HEIGHT)_PIXELS|WHITELIST(?:_ENABLE)?|PERMS_(?:INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(?:STANDARD|MINI)|PERM_(?:NONE|OWNER|GROUP|ANYONE)|MAX_(?:URL_LENGTH|WHITELIST_(?:SIZE|COUNT)|(?:WIDTH|HEIGHT)_PIXELS)))|MASK_(?:BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(?:TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(?:MEDIA_COMMAND_(?:STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(?:ALLOW_(?:FLY|(?:GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(?:GROUP_)?OBJECTS)|USE_(?:ACCESS_(?:GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(?:GROUP|ALL)_OBJECT_ENTRY)|COUNT_(?:TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(?:NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(?:MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(?:_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(?:HIDE|DEFAULT)|REGION_FLAG_(?:ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(?:COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(?:METHOD|MIMETYPE|BODY_(?:MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|STRING_(?:TRIM(?:_(?:HEAD|TAIL))?)|CLICK_ACTION_(?:NONE|TOUCH|SIT|BUY|PAY|OPEN(?:_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(?:NONE|SCRIPT_MEMORY)|RC_(?:DATA_FLAGS|DETECT_PHANTOM|GET_(?:LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(?:TYPES|AGENTS|(?:NON)?PHYSICAL|LAND))|RCERR_(?:CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(?:ALLOWED_(?:AGENT|GROUP)_(?:ADD|REMOVE)|BANNED_AGENT_(?:ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(?:COMMAND|CMD_(?:PLAY|STOP|PAUSE|SET_MODE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(?:GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(?:CMD_(?:(?:SMOOTH_)?STOP|JUMP)|DESIRED_(?:TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(?:_(?:[A-D]|NONE))?|MAX_(?:DECEL|TURN_RADIUS|(?:ACCEL|SPEED)))|PURSUIT_(?:OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(?:CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(?:EVADE_(?:HIDDEN|SPOTTED)|FAILURE_(?:DYNAMIC_PATHFINDING_DISABLED|INVALID_(?:GOAL|START)|NO_(?:NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(?:PARCEL_)?UNREACHABLE)|(?:GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(?:_(?:FAST|NONE|SLOW))?|CONTENT_TYPE_(?:ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(?:RADIUS|STATIC)|(?:PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(?:AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\b'
+    lsl_constants_integer_boolean = r'\b(?:FALSE|TRUE)\b'
+    lsl_constants_rotation = r'\b(?:ZERO_ROTATION)\b'
+    lsl_constants_string = r'\b(?:EOF|JSON_(?:ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(?:BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(?:GRANTED|DENIED))\b'
+    lsl_constants_vector = r'\b(?:TOUCH_INVALID_(?:TEXCOORD|VECTOR)|ZERO_VECTOR)\b'
+    lsl_invalid_broken = r'\b(?:LAND_(?:LARGE|MEDIUM|SMALL)_BRUSH)\b'
+    lsl_invalid_deprecated = r'\b(?:ATTACH_[LR]PEC|DATA_RATING|OBJECT_ATTACHMENT_(?:GEOMETRY_BYTES|SURFACE_AREA)|PRIM_(?:CAST_SHADOWS|MATERIAL_LIGHT|TYPE_LEGACY)|PSYS_SRC_(?:INNER|OUTER)ANGLE|VEHICLE_FLAG_NO_FLY_UP|ll(?:Cloud|Make(?:Explosion|Fountain|Smoke|Fire)|RemoteDataSetRegion|Sound(?:Preload)?|XorBase64Strings(?:Correct)?))\b'
+    lsl_invalid_illegal = r'\b(?:event)\b'
+    lsl_invalid_unimplemented = r'\b(?:CHARACTER_(?:MAX_ANGULAR_(?:ACCEL|SPEED)|TURN_SPEED_MULTIPLIER)|PERMISSION_(?:CHANGE_(?:JOINTS|PERMISSIONS)|RELEASE_OWNERSHIP|REMAP_CONTROLS)|PRIM_PHYSICS_MATERIAL|PSYS_SRC_OBJ_REL_MASK|ll(?:CollisionSprite|(?:Stop)?PointAt|(?:(?:Refresh|Set)Prim)URL|(?:Take|Release)Camera|RemoteLoadScript))\b'
+    lsl_reserved_godmode = r'\b(?:ll(?:GodLikeRezObject|Set(?:Inventory|Object)PermMask))\b'
+    lsl_reserved_log = r'\b(?:print)\b'
+    lsl_operators = r'\+\+|\-\-|<<|>>|&&?|\|\|?|\^|~|[!%<>=*+\-/]=?'
+
+    tokens = {
+        'root':
+        [
+            (r'//.*?\n',                          Comment.Single),
+            (r'/\*',                              Comment.Multiline, 'comment'),
+            (r'"',                                String.Double, 'string'),
+            (lsl_keywords,                        Keyword),
+            (lsl_types,                           Keyword.Type),
+            (lsl_states,                          Name.Class),
+            (lsl_events,                          Name.Builtin),
+            (lsl_functions_builtin,               Name.Function),
+            (lsl_constants_float,                 Keyword.Constant),
+            (lsl_constants_integer,               Keyword.Constant),
+            (lsl_constants_integer_boolean,       Keyword.Constant),
+            (lsl_constants_rotation,              Keyword.Constant),
+            (lsl_constants_string,                Keyword.Constant),
+            (lsl_constants_vector,                Keyword.Constant),
+            (lsl_invalid_broken,                  Error),
+            (lsl_invalid_deprecated,              Error),
+            (lsl_invalid_illegal,                 Error),
+            (lsl_invalid_unimplemented,           Error),
+            (lsl_reserved_godmode,                Keyword.Reserved),
+            (lsl_reserved_log,                    Keyword.Reserved),
+            (r'\b([a-zA-Z_]\w*)\b',     Name.Variable),
+            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float),
+            (r'(\d+\.\d*|\.\d+)',                 Number.Float),
+            (r'0[xX][0-9a-fA-F]+',                Number.Hex),
+            (r'\d+',                              Number.Integer),
+            (lsl_operators,                       Operator),
+            (r':=?',                              Error),
+            (r'[,;{}()\[\]]',                     Punctuation),
+            (r'\n+',                              Whitespace),
+            (r'\s+',                              Whitespace)
+        ],
+        'comment':
+        [
+            (r'[^*/]+',                           Comment.Multiline),
+            (r'/\*',                              Comment.Multiline, '#push'),
+            (r'\*/',                              Comment.Multiline, '#pop'),
+            (r'[*/]',                             Comment.Multiline)
+        ],
+        'string':
+        [
+            (r'\\([nt"\\])',                      String.Escape),
+            (r'"',                                String.Double, '#pop'),
+            (r'\\.',                              Error),
+            (r'[^"\\]+',                          String.Double),
+        ]
+    }
+
+
+class AppleScriptLexer(RegexLexer):
+    """
+    For AppleScript source code,
+    including `AppleScript Studio
+    `_.
+    Contributed by Andreas Amann .
+    """
+
+    name = 'AppleScript'
+    url = 'https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html'
+    aliases = ['applescript']
+    filenames = ['*.applescript']
+    version_added = '1.0'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    Identifiers = r'[a-zA-Z]\w*'
+
+    # XXX: use words() for all of these
+    Literals = ('AppleScript', 'current application', 'false', 'linefeed',
+                'missing value', 'pi', 'quote', 'result', 'return', 'space',
+                'tab', 'text item delimiters', 'true', 'version')
+    Classes = ('alias ', 'application ', 'boolean ', 'class ', 'constant ',
+               'date ', 'file ', 'integer ', 'list ', 'number ', 'POSIX file ',
+               'real ', 'record ', 'reference ', 'RGB color ', 'script ',
+               'text ', 'unit types', '(?:Unicode )?text', 'string')
+    BuiltIn = ('attachment', 'attribute run', 'character', 'day', 'month',
+               'paragraph', 'word', 'year')
+    HandlerParams = ('about', 'above', 'against', 'apart from', 'around',
+                     'aside from', 'at', 'below', 'beneath', 'beside',
+                     'between', 'for', 'given', 'instead of', 'on', 'onto',
+                     'out of', 'over', 'since')
+    Commands = ('ASCII (character|number)', 'activate', 'beep', 'choose URL',
+                'choose application', 'choose color', 'choose file( name)?',
+                'choose folder', 'choose from list',
+                'choose remote application', 'clipboard info',
+                'close( access)?', 'copy', 'count', 'current date', 'delay',
+                'delete', 'display (alert|dialog)', 'do shell script',
+                'duplicate', 'exists', 'get eof', 'get volume settings',
+                'info for', 'launch', 'list (disks|folder)', 'load script',
+                'log', 'make', 'mount volume', 'new', 'offset',
+                'open( (for access|location))?', 'path to', 'print', 'quit',
+                'random number', 'read', 'round', 'run( script)?',
+                'say', 'scripting components',
+                'set (eof|the clipboard to|volume)', 'store script',
+                'summarize', 'system attribute', 'system info',
+                'the clipboard', 'time to GMT', 'write', 'quoted form')
+    References = ('(in )?back of', '(in )?front of', '[0-9]+(st|nd|rd|th)',
+                  'first', 'second', 'third', 'fourth', 'fifth', 'sixth',
+                  'seventh', 'eighth', 'ninth', 'tenth', 'after', 'back',
+                  'before', 'behind', 'every', 'front', 'index', 'last',
+                  'middle', 'some', 'that', 'through', 'thru', 'where', 'whose')
+    Operators = ("and", "or", "is equal", "equals", "(is )?equal to", "is not",
+                 "isn't", "isn't equal( to)?", "is not equal( to)?",
+                 "doesn't equal", "does not equal", "(is )?greater than",
+                 "comes after", "is not less than or equal( to)?",
+                 "isn't less than or equal( to)?", "(is )?less than",
+                 "comes before", "is not greater than or equal( to)?",
+                 "isn't greater than or equal( to)?",
+                 "(is  )?greater than or equal( to)?", "is not less than",
+                 "isn't less than", "does not come before",
+                 "doesn't come before", "(is )?less than or equal( to)?",
+                 "is not greater than", "isn't greater than",
+                 "does not come after", "doesn't come after", "starts? with",
+                 "begins? with", "ends? with", "contains?", "does not contain",
+                 "doesn't contain", "is in", "is contained by", "is not in",
+                 "is not contained by", "isn't contained by", "div", "mod",
+                 "not", "(a  )?(ref( to)?|reference to)", "is", "does")
+    Control = ('considering', 'else', 'error', 'exit', 'from', 'if',
+               'ignoring', 'in', 'repeat', 'tell', 'then', 'times', 'to',
+               'try', 'until', 'using terms from', 'while', 'whith',
+               'with timeout( of)?', 'with transaction', 'by', 'continue',
+               'end', 'its?', 'me', 'my', 'return', 'of', 'as')
+    Declarations = ('global', 'local', 'prop(erty)?', 'set', 'get')
+    Reserved = ('but', 'put', 'returning', 'the')
+    StudioClasses = ('action cell', 'alert reply', 'application', 'box',
+                     'browser( cell)?', 'bundle', 'button( cell)?', 'cell',
+                     'clip view', 'color well', 'color-panel',
+                     'combo box( item)?', 'control',
+                     'data( (cell|column|item|row|source))?', 'default entry',
+                     'dialog reply', 'document', 'drag info', 'drawer',
+                     'event', 'font(-panel)?', 'formatter',
+                     'image( (cell|view))?', 'matrix', 'menu( item)?', 'item',
+                     'movie( view)?', 'open-panel', 'outline view', 'panel',
+                     'pasteboard', 'plugin', 'popup button',
+                     'progress indicator', 'responder', 'save-panel',
+                     'scroll view', 'secure text field( cell)?', 'slider',
+                     'sound', 'split view', 'stepper', 'tab view( item)?',
+                     'table( (column|header cell|header view|view))',
+                     'text( (field( cell)?|view))?', 'toolbar( item)?',
+                     'user-defaults', 'view', 'window')
+    StudioEvents = ('accept outline drop', 'accept table drop', 'action',
+                    'activated', 'alert ended', 'awake from nib', 'became key',
+                    'became main', 'begin editing', 'bounds changed',
+                    'cell value', 'cell value changed', 'change cell value',
+                    'change item value', 'changed', 'child of item',
+                    'choose menu item', 'clicked', 'clicked toolbar item',
+                    'closed', 'column clicked', 'column moved',
+                    'column resized', 'conclude drop', 'data representation',
+                    'deminiaturized', 'dialog ended', 'document nib name',
+                    'double clicked', 'drag( (entered|exited|updated))?',
+                    'drop', 'end editing', 'exposed', 'idle', 'item expandable',
+                    'item value', 'item value changed', 'items changed',
+                    'keyboard down', 'keyboard up', 'launched',
+                    'load data representation', 'miniaturized', 'mouse down',
+                    'mouse dragged', 'mouse entered', 'mouse exited',
+                    'mouse moved', 'mouse up', 'moved',
+                    'number of browser rows', 'number of items',
+                    'number of rows', 'open untitled', 'opened', 'panel ended',
+                    'parameters updated', 'plugin loaded', 'prepare drop',
+                    'prepare outline drag', 'prepare outline drop',
+                    'prepare table drag', 'prepare table drop',
+                    'read from file', 'resigned active', 'resigned key',
+                    'resigned main', 'resized( sub views)?',
+                    'right mouse down', 'right mouse dragged',
+                    'right mouse up', 'rows changed', 'scroll wheel',
+                    'selected tab view item', 'selection changed',
+                    'selection changing', 'should begin editing',
+                    'should close', 'should collapse item',
+                    'should end editing', 'should expand item',
+                    'should open( untitled)?',
+                    'should quit( after last window closed)?',
+                    'should select column', 'should select item',
+                    'should select row', 'should select tab view item',
+                    'should selection change', 'should zoom', 'shown',
+                    'update menu item', 'update parameters',
+                    'update toolbar item', 'was hidden', 'was miniaturized',
+                    'will become active', 'will close', 'will dismiss',
+                    'will display browser cell', 'will display cell',
+                    'will display item cell', 'will display outline cell',
+                    'will finish launching', 'will hide', 'will miniaturize',
+                    'will move', 'will open', 'will pop up', 'will quit',
+                    'will resign active', 'will resize( sub views)?',
+                    'will select tab view item', 'will show', 'will zoom',
+                    'write to file', 'zoomed')
+    StudioCommands = ('animate', 'append', 'call method', 'center',
+                      'close drawer', 'close panel', 'display',
+                      'display alert', 'display dialog', 'display panel', 'go',
+                      'hide', 'highlight', 'increment', 'item for',
+                      'load image', 'load movie', 'load nib', 'load panel',
+                      'load sound', 'localized string', 'lock focus', 'log',
+                      'open drawer', 'path for', 'pause', 'perform action',
+                      'play', 'register', 'resume', 'scroll', 'select( all)?',
+                      'show', 'size to fit', 'start', 'step back',
+                      'step forward', 'stop', 'synchronize', 'unlock focus',
+                      'update')
+    StudioProperties = ('accepts arrow key', 'action method', 'active',
+                        'alignment', 'allowed identifiers',
+                        'allows branch selection', 'allows column reordering',
+                        'allows column resizing', 'allows column selection',
+                        'allows customization',
+                        'allows editing text attributes',
+                        'allows empty selection', 'allows mixed state',
+                        'allows multiple selection', 'allows reordering',
+                        'allows undo', 'alpha( value)?', 'alternate image',
+                        'alternate increment value', 'alternate title',
+                        'animation delay', 'associated file name',
+                        'associated object', 'auto completes', 'auto display',
+                        'auto enables items', 'auto repeat',
+                        'auto resizes( outline column)?',
+                        'auto save expanded items', 'auto save name',
+                        'auto save table columns', 'auto saves configuration',
+                        'auto scroll', 'auto sizes all columns to fit',
+                        'auto sizes cells', 'background color', 'bezel state',
+                        'bezel style', 'bezeled', 'border rect', 'border type',
+                        'bordered', 'bounds( rotation)?', 'box type',
+                        'button returned', 'button type',
+                        'can choose directories', 'can choose files',
+                        'can draw', 'can hide',
+                        'cell( (background color|size|type))?', 'characters',
+                        'class', 'click count', 'clicked( data)? column',
+                        'clicked data item', 'clicked( data)? row',
+                        'closeable', 'collating', 'color( (mode|panel))',
+                        'command key down', 'configuration',
+                        'content(s| (size|view( margins)?))?', 'context',
+                        'continuous', 'control key down', 'control size',
+                        'control tint', 'control view',
+                        'controller visible', 'coordinate system',
+                        'copies( on scroll)?', 'corner view', 'current cell',
+                        'current column', 'current( field)?  editor',
+                        'current( menu)? item', 'current row',
+                        'current tab view item', 'data source',
+                        'default identifiers', 'delta (x|y|z)',
+                        'destination window', 'directory', 'display mode',
+                        'displayed cell', 'document( (edited|rect|view))?',
+                        'double value', 'dragged column', 'dragged distance',
+                        'dragged items', 'draws( cell)? background',
+                        'draws grid', 'dynamically scrolls', 'echos bullets',
+                        'edge', 'editable', 'edited( data)? column',
+                        'edited data item', 'edited( data)? row', 'enabled',
+                        'enclosing scroll view', 'ending page',
+                        'error handling', 'event number', 'event type',
+                        'excluded from windows menu', 'executable path',
+                        'expanded', 'fax number', 'field editor', 'file kind',
+                        'file name', 'file type', 'first responder',
+                        'first visible column', 'flipped', 'floating',
+                        'font( panel)?', 'formatter', 'frameworks path',
+                        'frontmost', 'gave up', 'grid color', 'has data items',
+                        'has horizontal ruler', 'has horizontal scroller',
+                        'has parent data item', 'has resize indicator',
+                        'has shadow', 'has sub menu', 'has vertical ruler',
+                        'has vertical scroller', 'header cell', 'header view',
+                        'hidden', 'hides when deactivated', 'highlights by',
+                        'horizontal line scroll', 'horizontal page scroll',
+                        'horizontal ruler view', 'horizontally resizable',
+                        'icon image', 'id', 'identifier',
+                        'ignores multiple clicks',
+                        'image( (alignment|dims when disabled|frame style|scaling))?',
+                        'imports graphics', 'increment value',
+                        'indentation per level', 'indeterminate', 'index',
+                        'integer value', 'intercell spacing', 'item height',
+                        'key( (code|equivalent( modifier)?|window))?',
+                        'knob thickness', 'label', 'last( visible)? column',
+                        'leading offset', 'leaf', 'level', 'line scroll',
+                        'loaded', 'localized sort', 'location', 'loop mode',
+                        'main( (bunde|menu|window))?', 'marker follows cell',
+                        'matrix mode', 'maximum( content)? size',
+                        'maximum visible columns',
+                        'menu( form representation)?', 'miniaturizable',
+                        'miniaturized', 'minimized image', 'minimized title',
+                        'minimum column width', 'minimum( content)? size',
+                        'modal', 'modified', 'mouse down state',
+                        'movie( (controller|file|rect))?', 'muted', 'name',
+                        'needs display', 'next state', 'next text',
+                        'number of tick marks', 'only tick mark values',
+                        'opaque', 'open panel', 'option key down',
+                        'outline table column', 'page scroll', 'pages across',
+                        'pages down', 'palette label', 'pane splitter',
+                        'parent data item', 'parent window', 'pasteboard',
+                        'path( (names|separator))?', 'playing',
+                        'plays every frame', 'plays selection only', 'position',
+                        'preferred edge', 'preferred type', 'pressure',
+                        'previous text', 'prompt', 'properties',
+                        'prototype cell', 'pulls down', 'rate',
+                        'released when closed', 'repeated',
+                        'requested print time', 'required file type',
+                        'resizable', 'resized column', 'resource path',
+                        'returns records', 'reuses columns', 'rich text',
+                        'roll over', 'row height', 'rulers visible',
+                        'save panel', 'scripts path', 'scrollable',
+                        'selectable( identifiers)?', 'selected cell',
+                        'selected( data)? columns?', 'selected data items?',
+                        'selected( data)? rows?', 'selected item identifier',
+                        'selection by rect', 'send action on arrow key',
+                        'sends action when done editing', 'separates columns',
+                        'separator item', 'sequence number', 'services menu',
+                        'shared frameworks path', 'shared support path',
+                        'sheet', 'shift key down', 'shows alpha',
+                        'shows state by', 'size( mode)?',
+                        'smart insert delete enabled', 'sort case sensitivity',
+                        'sort column', 'sort order', 'sort type',
+                        'sorted( data rows)?', 'sound', 'source( mask)?',
+                        'spell checking enabled', 'starting page', 'state',
+                        'string value', 'sub menu', 'super menu', 'super view',
+                        'tab key traverses cells', 'tab state', 'tab type',
+                        'tab view', 'table view', 'tag', 'target( printer)?',
+                        'text color', 'text container insert',
+                        'text container origin', 'text returned',
+                        'tick mark position', 'time stamp',
+                        'title(d| (cell|font|height|position|rect))?',
+                        'tool tip', 'toolbar', 'trailing offset', 'transparent',
+                        'treat packages as directories', 'truncated labels',
+                        'types', 'unmodified characters', 'update views',
+                        'use sort indicator', 'user defaults',
+                        'uses data source', 'uses ruler',
+                        'uses threaded animation',
+                        'uses title from previous column', 'value wraps',
+                        'version',
+                        'vertical( (line scroll|page scroll|ruler view))?',
+                        'vertically resizable', 'view',
+                        'visible( document rect)?', 'volume', 'width', 'window',
+                        'windows menu', 'wraps', 'zoomable', 'zoomed')
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'¬\n', String.Escape),
+            (r"'s\s+", Text),  # This is a possessive, consider moving
+            (r'(--|#).*?$', Comment),
+            (r'\(\*', Comment.Multiline, 'comment'),
+            (r'[(){}!,.:]', Punctuation),
+            (r'(«)([^»]+)(»)',
+             bygroups(Text, Name.Builtin, Text)),
+            (r'\b((?:considering|ignoring)\s*)'
+             r'(application responses|case|diacriticals|hyphens|'
+             r'numeric strings|punctuation|white space)',
+             bygroups(Keyword, Name.Builtin)),
+            (r'(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\^)', Operator),
+            (r"\b({})\b".format('|'.join(Operators)), Operator.Word),
+            (r'^(\s*(?:on|end)\s+)'
+             r'({})'.format('|'.join(StudioEvents[::-1])),
+             bygroups(Keyword, Name.Function)),
+            (r'^(\s*)(in|on|script|to)(\s+)', bygroups(Text, Keyword, Text)),
+            (r'\b(as )({})\b'.format('|'.join(Classes)),
+             bygroups(Keyword, Name.Class)),
+            (r'\b({})\b'.format('|'.join(Literals)), Name.Constant),
+            (r'\b({})\b'.format('|'.join(Commands)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(Control)), Keyword),
+            (r'\b({})\b'.format('|'.join(Declarations)), Keyword),
+            (r'\b({})\b'.format('|'.join(Reserved)), Name.Builtin),
+            (r'\b({})s?\b'.format('|'.join(BuiltIn)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(HandlerParams)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(StudioProperties)), Name.Attribute),
+            (r'\b({})s?\b'.format('|'.join(StudioClasses)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(StudioCommands)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(References)), Name.Builtin),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (rf'\b({Identifiers})\b', Name.Variable),
+            (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float),
+            (r'[-+]?\d+', Number.Integer),
+        ],
+        'comment': [
+            (r'\(\*', Comment.Multiline, '#push'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            ('[^*(]+', Comment.Multiline),
+            ('[*(]', Comment.Multiline),
+        ],
+    }
+
+
+class RexxLexer(RegexLexer):
+    """
+    Rexx is a scripting language available for
+    a wide range of different platforms with its roots found on mainframe
+    systems. It is popular for I/O- and data based tasks and can act as glue
+    language to bind different applications together.
+    """
+    name = 'Rexx'
+    url = 'http://www.rexxinfo.org/'
+    aliases = ['rexx', 'arexx']
+    filenames = ['*.rexx', '*.rex', '*.rx', '*.arexx']
+    mimetypes = ['text/x-rexx']
+    version_added = '2.0'
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r'\s+', Whitespace),
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"', String, 'string_double'),
+            (r"'", String, 'string_single'),
+            (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number),
+            (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b',
+             bygroups(Name.Function, Whitespace, Operator, Whitespace,
+                      Keyword.Declaration)),
+            (r'([a-z_]\w*)(\s*)(:)',
+             bygroups(Name.Label, Whitespace, Operator)),
+            include('function'),
+            include('keyword'),
+            include('operator'),
+            (r'[a-z_]\w*', Text),
+        ],
+        'function': [
+            (words((
+                'abbrev', 'abs', 'address', 'arg', 'b2x', 'bitand', 'bitor', 'bitxor',
+                'c2d', 'c2x', 'center', 'charin', 'charout', 'chars', 'compare',
+                'condition', 'copies', 'd2c', 'd2x', 'datatype', 'date', 'delstr',
+                'delword', 'digits', 'errortext', 'form', 'format', 'fuzz', 'insert',
+                'lastpos', 'left', 'length', 'linein', 'lineout', 'lines', 'max',
+                'min', 'overlay', 'pos', 'queued', 'random', 'reverse', 'right', 'sign',
+                'sourceline', 'space', 'stream', 'strip', 'substr', 'subword', 'symbol',
+                'time', 'trace', 'translate', 'trunc', 'value', 'verify', 'word',
+                'wordindex', 'wordlength', 'wordpos', 'words', 'x2b', 'x2c', 'x2d',
+                'xrange'), suffix=r'(\s*)(\()'),
+             bygroups(Name.Builtin, Whitespace, Operator)),
+        ],
+        'keyword': [
+            (r'(address|arg|by|call|do|drop|else|end|exit|for|forever|if|'
+             r'interpret|iterate|leave|nop|numeric|off|on|options|parse|'
+             r'pull|push|queue|return|say|select|signal|to|then|trace|until|'
+             r'while)\b', Keyword.Reserved),
+        ],
+        'operator': [
+            (r'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||'
+             r'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|'
+             r'¬>>|¬>|¬|\.|,)', Operator),
+        ],
+        'string_double': [
+            (r'[^"\n]+', String),
+            (r'""', String),
+            (r'"', String, '#pop'),
+            (r'\n', Text, '#pop'),  # Stray linefeed also terminates strings.
+        ],
+        'string_single': [
+            (r'[^\'\n]+', String),
+            (r'\'\'', String),
+            (r'\'', String, '#pop'),
+            (r'\n', Text, '#pop'),  # Stray linefeed also terminates strings.
+        ],
+        'comment': [
+            (r'[^*]+', Comment.Multiline),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ]
+    }
+
+    def _c(s):
+        return re.compile(s, re.MULTILINE)
+    _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b')
+    _ADDRESS_PATTERN = _c(r'^\s*address\s+')
+    _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b')
+    _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$')
+    _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b')
+    _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$')
+    _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b')
+    PATTERNS_AND_WEIGHTS = (
+        (_ADDRESS_COMMAND_PATTERN, 0.2),
+        (_ADDRESS_PATTERN, 0.05),
+        (_DO_WHILE_PATTERN, 0.1),
+        (_ELSE_DO_PATTERN, 0.1),
+        (_IF_THEN_DO_PATTERN, 0.1),
+        (_PROCEDURE_PATTERN, 0.5),
+        (_PARSE_ARG_PATTERN, 0.2),
+    )
+
+    def analyse_text(text):
+        """
+        Check for initial comment and patterns that distinguish Rexx from other
+        C-like languages.
+        """
+        if re.search(r'/\*\**\s*rexx', text, re.IGNORECASE):
+            # Header matches MVS Rexx requirements, this is certainly a Rexx
+            # script.
+            return 1.0
+        elif text.startswith('/*'):
+            # Header matches general Rexx requirements; the source code might
+            # still be any language using C comments such as C++, C# or Java.
+            lowerText = text.lower()
+            result = sum(weight
+                         for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS
+                         if pattern.search(lowerText)) + 0.01
+            return min(result, 1.0)
+
+
+class MOOCodeLexer(RegexLexer):
+    """
+    For MOOCode (the MOO scripting language).
+    """
+    name = 'MOOCode'
+    url = 'http://www.moo.mud.org/'
+    filenames = ['*.moo']
+    aliases = ['moocode', 'moo']
+    mimetypes = ['text/x-moocode']
+    version_added = '0.9'
+
+    tokens = {
+        'root': [
+            # Numbers
+            (r'(0|[1-9][0-9_]*)', Number.Integer),
+            # Strings
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # exceptions
+            (r'(E_PERM|E_DIV)', Name.Exception),
+            # db-refs
+            (r'((#[-0-9]+)|(\$\w+))', Name.Entity),
+            # Keywords
+            (r'\b(if|else|elseif|endif|for|endfor|fork|endfork|while'
+             r'|endwhile|break|continue|return|try'
+             r'|except|endtry|finally|in)\b', Keyword),
+            # builtins
+            (r'(random|length)', Name.Builtin),
+            # special variables
+            (r'(player|caller|this|args)', Name.Variable.Instance),
+            # skip whitespace
+            (r'\s+', Text),
+            (r'\n', Text),
+            # other operators
+            (r'([!;=,{}&|:.\[\]@()<>?]+)', Operator),
+            # function call
+            (r'(\w+)(\()', bygroups(Name.Function, Operator)),
+            # variables
+            (r'(\w+)', Text),
+        ]
+    }
+
+
+class HybrisLexer(RegexLexer):
+    """
+    For Hybris source code.
+    """
+
+    name = 'Hybris'
+    aliases = ['hybris']
+    filenames = ['*.hyb']
+    mimetypes = ['text/x-hybris', 'application/x-hybris']
+    url = 'https://github.com/evilsocket/hybris'
+    version_added = '1.4'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    tokens = {
+        'root': [
+            # method names
+            (r'^(\s*(?:function|method|operator\s+)+?)'
+             r'([a-zA-Z_]\w*)'
+             r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)),
+            (r'[^\S\n]+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'@[a-zA-Z_][\w.]*', Name.Decorator),
+            (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|'
+             r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword),
+            (r'(extends|private|protected|public|static|throws|function|method|'
+             r'operator)\b', Keyword.Declaration),
+            (r'(true|false|null|__FILE__|__LINE__|__VERSION__|__LIB_PATH__|'
+             r'__INC_PATH__)\b', Keyword.Constant),
+            (r'(class|struct)(\s+)',
+             bygroups(Keyword.Declaration, Text), 'class'),
+            (r'(import|include)(\s+)',
+             bygroups(Keyword.Namespace, Text), 'import'),
+            (words((
+                'gc_collect', 'gc_mm_items', 'gc_mm_usage', 'gc_collect_threshold',
+                'urlencode', 'urldecode', 'base64encode', 'base64decode', 'sha1', 'crc32',
+                'sha2', 'md5', 'md5_file', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos',
+                'cosh', 'exp', 'fabs', 'floor', 'fmod', 'log', 'log10', 'pow', 'sin',
+                'sinh', 'sqrt', 'tan', 'tanh', 'isint', 'isfloat', 'ischar', 'isstring',
+                'isarray', 'ismap', 'isalias', 'typeof', 'sizeof', 'toint', 'tostring',
+                'fromxml', 'toxml', 'binary', 'pack', 'load', 'eval', 'var_names',
+                'var_values', 'user_functions', 'dyn_functions', 'methods', 'call',
+                'call_method', 'mknod', 'mkfifo', 'mount', 'umount2', 'umount', 'ticks',
+                'usleep', 'sleep', 'time', 'strtime', 'strdate', 'dllopen', 'dlllink',
+                'dllcall', 'dllcall_argv', 'dllclose', 'env', 'exec', 'fork', 'getpid',
+                'wait', 'popen', 'pclose', 'exit', 'kill', 'pthread_create',
+                'pthread_create_argv', 'pthread_exit', 'pthread_join', 'pthread_kill',
+                'smtp_send', 'http_get', 'http_post', 'http_download', 'socket', 'bind',
+                'listen', 'accept', 'getsockname', 'getpeername', 'settimeout', 'connect',
+                'server', 'recv', 'send', 'close', 'print', 'println', 'printf', 'input',
+                'readline', 'serial_open', 'serial_fcntl', 'serial_get_attr',
+                'serial_get_ispeed', 'serial_get_ospeed', 'serial_set_attr',
+                'serial_set_ispeed', 'serial_set_ospeed', 'serial_write', 'serial_read',
+                'serial_close', 'xml_load', 'xml_parse', 'fopen', 'fseek', 'ftell',
+                'fsize', 'fread', 'fwrite', 'fgets', 'fclose', 'file', 'readdir',
+                'pcre_replace', 'size', 'pop', 'unmap', 'has', 'keys', 'values',
+                'length', 'find', 'substr', 'replace', 'split', 'trim', 'remove',
+                'contains', 'join'), suffix=r'\b'),
+             Name.Builtin),
+            (words((
+                'MethodReference', 'Runner', 'Dll', 'Thread', 'Pipe', 'Process',
+                'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket',
+                'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'),
+             Keyword.Type),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
+            (r'(\.)([a-zA-Z_]\w*)',
+             bygroups(Operator, Name.Attribute)),
+            (r'[a-zA-Z_]\w*:', Name.Label),
+            (r'[a-zA-Z_$]\w*', Name),
+            (r'[~^*!%&\[\](){}<>|+=:;,./?\-@]+', Operator),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-f]+', Number.Hex),
+            (r'[0-9]+L?', Number.Integer),
+            (r'\n', Text),
+        ],
+        'class': [
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+        ],
+        'import': [
+            (r'[\w.]+\*?', Name.Namespace, '#pop')
+        ],
+    }
+
+    def analyse_text(text):
+        """public method and private method don't seem to be quite common
+        elsewhere."""
+        result = 0
+        if re.search(r'\b(?:public|private)\s+method\b', text):
+            result += 0.01
+        return result
+
+
+
+class EasytrieveLexer(RegexLexer):
+    """
+    Easytrieve Plus is a programming language for extracting, filtering and
+    converting sequential data. Furthermore it can layout data for reports.
+    It is mainly used on mainframe platforms and can access several of the
+    mainframe's native file formats. It is somewhat comparable to awk.
+    """
+    name = 'Easytrieve'
+    aliases = ['easytrieve']
+    filenames = ['*.ezt', '*.mac']
+    mimetypes = ['text/x-easytrieve']
+    url = 'https://www.broadcom.com/products/mainframe/application-development/easytrieve-report-generator'
+    version_added = '2.1'
+    flags = 0
+
+    # Note: We cannot use r'\b' at the start and end of keywords because
+    # Easytrieve Plus delimiter characters are:
+    #
+    #   * space ( )
+    #   * apostrophe (')
+    #   * period (.)
+    #   * comma (,)
+    #   * parenthesis ( and )
+    #   * colon (:)
+    #
+    # Additionally words end once a '*' appears, indicatins a comment.
+    _DELIMITERS = r' \'.,():\n'
+    _DELIMITERS_OR_COMENT = _DELIMITERS + '*'
+    _DELIMITER_PATTERN = '[' + _DELIMITERS + ']'
+    _DELIMITER_PATTERN_CAPTURE = '(' + _DELIMITER_PATTERN + ')'
+    _NON_DELIMITER_OR_COMMENT_PATTERN = '[^' + _DELIMITERS_OR_COMENT + ']'
+    _OPERATORS_PATTERN = '[.+\\-/=\\[\\](){}<>;,&%¬]'
+    _KEYWORDS = [
+        'AFTER-BREAK', 'AFTER-LINE', 'AFTER-SCREEN', 'AIM', 'AND', 'ATTR',
+        'BEFORE', 'BEFORE-BREAK', 'BEFORE-LINE', 'BEFORE-SCREEN', 'BUSHU',
+        'BY', 'CALL', 'CASE', 'CHECKPOINT', 'CHKP', 'CHKP-STATUS', 'CLEAR',
+        'CLOSE', 'COL', 'COLOR', 'COMMIT', 'CONTROL', 'COPY', 'CURSOR', 'D',
+        'DECLARE', 'DEFAULT', 'DEFINE', 'DELETE', 'DENWA', 'DISPLAY', 'DLI',
+        'DO', 'DUPLICATE', 'E', 'ELSE', 'ELSE-IF', 'END', 'END-CASE',
+        'END-DO', 'END-IF', 'END-PROC', 'ENDPAGE', 'ENDTABLE', 'ENTER', 'EOF',
+        'EQ', 'ERROR', 'EXIT', 'EXTERNAL', 'EZLIB', 'F1', 'F10', 'F11', 'F12',
+        'F13', 'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F2', 'F20', 'F21',
+        'F22', 'F23', 'F24', 'F25', 'F26', 'F27', 'F28', 'F29', 'F3', 'F30',
+        'F31', 'F32', 'F33', 'F34', 'F35', 'F36', 'F4', 'F5', 'F6', 'F7',
+        'F8', 'F9', 'FETCH', 'FILE-STATUS', 'FILL', 'FINAL', 'FIRST',
+        'FIRST-DUP', 'FOR', 'GE', 'GET', 'GO', 'GOTO', 'GQ', 'GR', 'GT',
+        'HEADING', 'HEX', 'HIGH-VALUES', 'IDD', 'IDMS', 'IF', 'IN', 'INSERT',
+        'JUSTIFY', 'KANJI-DATE', 'KANJI-DATE-LONG', 'KANJI-TIME', 'KEY',
+        'KEY-PRESSED', 'KOKUGO', 'KUN', 'LAST-DUP', 'LE', 'LEVEL', 'LIKE',
+        'LINE', 'LINE-COUNT', 'LINE-NUMBER', 'LINK', 'LIST', 'LOW-VALUES',
+        'LQ', 'LS', 'LT', 'MACRO', 'MASK', 'MATCHED', 'MEND', 'MESSAGE',
+        'MOVE', 'MSTART', 'NE', 'NEWPAGE', 'NOMASK', 'NOPRINT', 'NOT',
+        'NOTE', 'NOVERIFY', 'NQ', 'NULL', 'OF', 'OR', 'OTHERWISE', 'PA1',
+        'PA2', 'PA3', 'PAGE-COUNT', 'PAGE-NUMBER', 'PARM-REGISTER',
+        'PATH-ID', 'PATTERN', 'PERFORM', 'POINT', 'POS', 'PRIMARY', 'PRINT',
+        'PROCEDURE', 'PROGRAM', 'PUT', 'READ', 'RECORD', 'RECORD-COUNT',
+        'RECORD-LENGTH', 'REFRESH', 'RELEASE', 'RENUM', 'REPEAT', 'REPORT',
+        'REPORT-INPUT', 'RESHOW', 'RESTART', 'RETRIEVE', 'RETURN-CODE',
+        'ROLLBACK', 'ROW', 'S', 'SCREEN', 'SEARCH', 'SECONDARY', 'SELECT',
+        'SEQUENCE', 'SIZE', 'SKIP', 'SOKAKU', 'SORT', 'SQL', 'STOP', 'SUM',
+        'SYSDATE', 'SYSDATE-LONG', 'SYSIN', 'SYSIPT', 'SYSLST', 'SYSPRINT',
+        'SYSSNAP', 'SYSTIME', 'TALLY', 'TERM-COLUMNS', 'TERM-NAME',
+        'TERM-ROWS', 'TERMINATION', 'TITLE', 'TO', 'TRANSFER', 'TRC',
+        'UNIQUE', 'UNTIL', 'UPDATE', 'UPPERCASE', 'USER', 'USERID', 'VALUE',
+        'VERIFY', 'W', 'WHEN', 'WHILE', 'WORK', 'WRITE', 'X', 'XDM', 'XRST'
+    ]
+
+    tokens = {
+        'root': [
+            (r'\*.*\n', Comment.Single),
+            (r'\n+', Whitespace),
+            # Macro argument
+            (r'&' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+\.', Name.Variable,
+             'after_macro_argument'),
+            # Macro call
+            (r'%' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Variable),
+            (r'(FILE|MACRO|REPORT)(\s+)',
+             bygroups(Keyword.Declaration, Whitespace), 'after_declaration'),
+            (r'(JOB|PARM)' + r'(' + _DELIMITER_PATTERN + r')',
+             bygroups(Keyword.Declaration, Operator)),
+            (words(_KEYWORDS, suffix=_DELIMITER_PATTERN_CAPTURE),
+             bygroups(Keyword.Reserved, Operator)),
+            (_OPERATORS_PATTERN, Operator),
+            # Procedure declaration
+            (r'(' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+)(\s*)(\.?)(\s*)(PROC)(\s*\n)',
+             bygroups(Name.Function, Whitespace, Operator, Whitespace,
+                      Keyword.Declaration, Whitespace)),
+            (r'[0-9]+\.[0-9]*', Number.Float),
+            (r'[0-9]+', Number.Integer),
+            (r"'(''|[^'])*'", String),
+            (r'\s+', Whitespace),
+            # Everything else just belongs to a name
+            (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
+         ],
+        'after_declaration': [
+            (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Function),
+            default('#pop'),
+        ],
+        'after_macro_argument': [
+            (r'\*.*\n', Comment.Single, '#pop'),
+            (r'\s+', Whitespace, '#pop'),
+            (_OPERATORS_PATTERN, Operator, '#pop'),
+            (r"'(''|[^'])*'", String, '#pop'),
+            # Everything else just belongs to a name
+            (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
+        ],
+    }
+    _COMMENT_LINE_REGEX = re.compile(r'^\s*\*')
+    _MACRO_HEADER_REGEX = re.compile(r'^\s*MACRO')
+
+    def analyse_text(text):
+        """
+        Perform a structural analysis for basic Easytrieve constructs.
+        """
+        result = 0.0
+        lines = text.split('\n')
+        hasEndProc = False
+        hasHeaderComment = False
+        hasFile = False
+        hasJob = False
+        hasProc = False
+        hasParm = False
+        hasReport = False
+
+        def isCommentLine(line):
+            return EasytrieveLexer._COMMENT_LINE_REGEX.match(lines[0]) is not None
+
+        def isEmptyLine(line):
+            return not bool(line.strip())
+
+        # Remove possible empty lines and header comments.
+        while lines and (isEmptyLine(lines[0]) or isCommentLine(lines[0])):
+            if not isEmptyLine(lines[0]):
+                hasHeaderComment = True
+            del lines[0]
+
+        if EasytrieveLexer._MACRO_HEADER_REGEX.match(lines[0]):
+            # Looks like an Easytrieve macro.
+            result = 0.4
+            if hasHeaderComment:
+                result += 0.4
+        else:
+            # Scan the source for lines starting with indicators.
+            for line in lines:
+                words = line.split()
+                if (len(words) >= 2):
+                    firstWord = words[0]
+                    if not hasReport:
+                        if not hasJob:
+                            if not hasFile:
+                                if not hasParm:
+                                    if firstWord == 'PARM':
+                                        hasParm = True
+                                if firstWord == 'FILE':
+                                    hasFile = True
+                            if firstWord == 'JOB':
+                                hasJob = True
+                        elif firstWord == 'PROC':
+                            hasProc = True
+                        elif firstWord == 'END-PROC':
+                            hasEndProc = True
+                        elif firstWord == 'REPORT':
+                            hasReport = True
+
+            # Weight the findings.
+            if hasJob and (hasProc == hasEndProc):
+                if hasHeaderComment:
+                    result += 0.1
+                if hasParm:
+                    if hasProc:
+                        # Found PARM, JOB and PROC/END-PROC:
+                        # pretty sure this is Easytrieve.
+                        result += 0.8
+                    else:
+                        # Found PARAM and  JOB: probably this is Easytrieve
+                        result += 0.5
+                else:
+                    # Found JOB and possibly other keywords: might be Easytrieve
+                    result += 0.11
+                    if hasParm:
+                        # Note: PARAM is not a proper English word, so this is
+                        # regarded a much better indicator for Easytrieve than
+                        # the other words.
+                        result += 0.2
+                    if hasFile:
+                        result += 0.01
+                    if hasReport:
+                        result += 0.01
+        assert 0.0 <= result <= 1.0
+        return result
+
+
+class JclLexer(RegexLexer):
+    """
+    Job Control Language (JCL)
+    is a scripting language used on mainframe platforms to instruct the system
+    on how to run a batch job or start a subsystem. It is somewhat
+    comparable to MS DOS batch and Unix shell scripts.
+    """
+    name = 'JCL'
+    aliases = ['jcl']
+    filenames = ['*.jcl']
+    mimetypes = ['text/x-jcl']
+    url = 'https://en.wikipedia.org/wiki/Job_Control_Language'
+    version_added = '2.1'
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r'//\*.*\n', Comment.Single),
+            (r'//', Keyword.Pseudo, 'statement'),
+            (r'/\*', Keyword.Pseudo, 'jes2_statement'),
+            # TODO: JES3 statement
+            (r'.*\n', Other)  # Input text or inline code in any language.
+        ],
+        'statement': [
+            (r'\s*\n', Whitespace, '#pop'),
+            (r'([a-z]\w*)(\s+)(exec|job)(\s*)',
+             bygroups(Name.Label, Whitespace, Keyword.Reserved, Whitespace),
+             'option'),
+            (r'[a-z]\w*', Name.Variable, 'statement_command'),
+            (r'\s+', Whitespace, 'statement_command'),
+        ],
+        'statement_command': [
+            (r'\s+(command|cntl|dd|endctl|endif|else|include|jcllib|'
+             r'output|pend|proc|set|then|xmit)\s+', Keyword.Reserved, 'option'),
+            include('option')
+        ],
+        'jes2_statement': [
+            (r'\s*\n', Whitespace, '#pop'),
+            (r'\$', Keyword, 'option'),
+            (r'\b(jobparam|message|netacct|notify|output|priority|route|'
+             r'setup|signoff|xeq|xmit)\b', Keyword, 'option'),
+        ],
+        'option': [
+            # (r'\n', Text, 'root'),
+            (r'\*', Name.Builtin),
+            (r'[\[\](){}<>;,]', Punctuation),
+            (r'[-+*/=&%]', Operator),
+            (r'[a-z_]\w*', Name),
+            (r'\d+\.\d*', Number.Float),
+            (r'\.\d+', Number.Float),
+            (r'\d+', Number.Integer),
+            (r"'", String, 'option_string'),
+            (r'[ \t]+', Whitespace, 'option_comment'),
+            (r'\.', Punctuation),
+        ],
+        'option_string': [
+            (r"(\n)(//)", bygroups(Text, Keyword.Pseudo)),
+            (r"''", String),
+            (r"[^']", String),
+            (r"'", String, '#pop'),
+        ],
+        'option_comment': [
+            # (r'\n', Text, 'root'),
+            (r'.+', Comment.Single),
+        ]
+    }
+
+    _JOB_HEADER_PATTERN = re.compile(r'^//[a-z#$@][a-z0-9#$@]{0,7}\s+job(\s+.*)?$',
+                                     re.IGNORECASE)
+
+    def analyse_text(text):
+        """
+        Recognize JCL job by header.
+        """
+        result = 0.0
+        lines = text.split('\n')
+        if len(lines) > 0:
+            if JclLexer._JOB_HEADER_PATTERN.match(lines[0]):
+                result = 1.0
+        assert 0.0 <= result <= 1.0
+        return result
+
+
+class MiniScriptLexer(RegexLexer):
+    """
+    For MiniScript source code.
+    """
+
+    name = 'MiniScript'
+    url = 'https://miniscript.org'
+    aliases = ['miniscript', 'ms']
+    filenames = ['*.ms']
+    mimetypes = ['text/x-minicript', 'application/x-miniscript']
+    version_added = '2.6'
+
+    tokens = {
+        'root': [
+            (r'#!(.*?)$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            ('//.*$', Comment.Single),
+            (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number),
+            (r'(?i)\d+e[+-]?\d+', Number),
+            (r'\d+', Number),
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'"', String, 'string_double'),
+            (r'(==|!=|<=|>=|[=+\-*/%^<>.:])', Operator),
+            (r'[;,\[\]{}()]', Punctuation),
+            (words((
+                'break', 'continue', 'else', 'end', 'for', 'function', 'if',
+                'in', 'isa', 'then', 'repeat', 'return', 'while'), suffix=r'\b'),
+             Keyword),
+            (words((
+                'abs', 'acos', 'asin', 'atan', 'ceil', 'char', 'cos', 'floor',
+                'log', 'round', 'rnd', 'pi', 'sign', 'sin', 'sqrt', 'str', 'tan',
+                'hasIndex', 'indexOf', 'len', 'val', 'code', 'remove', 'lower',
+                'upper', 'replace', 'split', 'indexes', 'values', 'join', 'sum',
+                'sort', 'shuffle', 'push', 'pop', 'pull', 'range',
+                'print', 'input', 'time', 'wait', 'locals', 'globals', 'outer',
+                'yield'), suffix=r'\b'),
+             Name.Builtin),
+            (r'(true|false|null)\b', Keyword.Constant),
+            (r'(and|or|not|new)\b', Operator.Word),
+            (r'(self|super|__isa)\b', Name.Builtin.Pseudo),
+            (r'[a-zA-Z_]\w*', Name.Variable)
+        ],
+        'string_double': [
+            (r'[^"\n]+', String),
+            (r'""', String),
+            (r'"', String, '#pop'),
+            (r'\n', Text, '#pop'),  # Stray linefeed also terminates strings.
+        ]
+    }
diff --git a/lib/pygments/lexers/sgf.py b/lib/pygments/lexers/sgf.py
new file mode 100644
index 0000000..f0e56cb
--- /dev/null
+++ b/lib/pygments/lexers/sgf.py
@@ -0,0 +1,59 @@
+"""
+    pygments.lexers.sgf
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Smart Game Format (sgf) file format.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Name, Literal, String, Punctuation, Whitespace
+
+__all__ = ["SmartGameFormatLexer"]
+
+
+class SmartGameFormatLexer(RegexLexer):
+    """
+    Lexer for Smart Game Format (sgf) file format.
+
+    The format is used to store game records of board games for two players
+    (mainly Go game).
+    """
+    name = 'SmartGameFormat'
+    url = 'https://www.red-bean.com/sgf/'
+    aliases = ['sgf']
+    filenames = ['*.sgf']
+    version_added = '2.4'
+
+    tokens = {
+        'root': [
+            (r'[():;]+', Punctuation),
+            # tokens:
+            (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|'
+             r'DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|'
+             r'KO|LB|LN|LT|L|MA|MN|M|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|'
+             r'RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|VW|'
+             r'V|[BW]|C)',
+             Name.Builtin),
+            # number:
+            (r'(\[)([0-9.]+)(\])',
+             bygroups(Punctuation, Literal.Number, Punctuation)),
+            # date:
+            (r'(\[)([0-9]{4}-[0-9]{2}-[0-9]{2})(\])',
+             bygroups(Punctuation, Literal.Date, Punctuation)),
+            # point:
+            (r'(\[)([a-z]{2})(\])',
+             bygroups(Punctuation, String, Punctuation)),
+            # double points:
+            (r'(\[)([a-z]{2})(:)([a-z]{2})(\])',
+             bygroups(Punctuation, String, Punctuation, String, Punctuation)),
+
+            (r'(\[)([\w\s#()+,\-.:?]+)(\])',
+             bygroups(Punctuation, String, Punctuation)),
+            (r'(\[)(\s.*)(\])',
+             bygroups(Punctuation, Whitespace, Punctuation)),
+            (r'\s+', Whitespace)
+        ],
+    }
diff --git a/lib/pygments/lexers/shell.py b/lib/pygments/lexers/shell.py
new file mode 100644
index 0000000..744767a
--- /dev/null
+++ b/lib/pygments/lexers/shell.py
@@ -0,0 +1,902 @@
+"""
+    pygments.lexers.shell
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for various shells.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, \
+    include, default, this, using, words, line_re
+from pygments.token import Punctuation, Whitespace, \
+    Text, Comment, Operator, Keyword, Name, String, Number, Generic
+from pygments.util import shebang_matches
+
+__all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer',
+           'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer',
+           'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer',
+           'ExeclineLexer']
+
+
+class BashLexer(RegexLexer):
+    """
+    Lexer for (ba|k|z|)sh shell scripts.
+    """
+
+    name = 'Bash'
+    aliases = ['bash', 'sh', 'ksh', 'zsh', 'shell', 'openrc']
+    filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass',
+                 '*.exheres-0', '*.exlib', '*.zsh',
+                 '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc',
+                 '.kshrc', 'kshrc',
+                 'PKGBUILD']
+    mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript']
+    url = 'https://en.wikipedia.org/wiki/Unix_shell'
+    version_added = '0.6'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            (r'`', String.Backtick, 'backticks'),
+            include('data'),
+            include('interp'),
+        ],
+        'interp': [
+            (r'\$\(\(', Keyword, 'math'),
+            (r'\$\(', Keyword, 'paren'),
+            (r'\$\{#?', String.Interpol, 'curly'),
+            (r'\$[a-zA-Z_]\w*', Name.Variable),  # user variable
+            (r'\$(?:\d+|[#$?!_*@-])', Name.Variable),      # builtin
+            (r'\$', Text),
+        ],
+        'basic': [
+            (r'\b(if|fi|else|while|in|do|done|for|then|return|function|case|'
+             r'select|break|continue|until|esac|elif)(\s*)\b',
+             bygroups(Keyword, Whitespace)),
+            (r'\b(alias|bg|bind|builtin|caller|cd|command|compgen|'
+             r'complete|declare|dirs|disown|echo|enable|eval|exec|exit|'
+             r'export|false|fc|fg|getopts|hash|help|history|jobs|kill|let|'
+             r'local|logout|popd|printf|pushd|pwd|read|readonly|set|shift|'
+             r'shopt|source|suspend|test|time|times|trap|true|type|typeset|'
+             r'ulimit|umask|unalias|unset|wait)(?=[\s)`])',
+             Name.Builtin),
+            (r'\A#!.+\n', Comment.Hashbang),
+            (r'#.*\n', Comment.Single),
+            (r'\\[\w\W]', String.Escape),
+            (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Whitespace, Operator)),
+            (r'[\[\]{}()=]', Operator),
+            (r'<<<', Operator),  # here-string
+            (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+            (r'&&|\|\|', Operator),
+        ],
+        'data': [
+            (r'(?s)\$?"(\\.|[^"\\$])*"', String.Double),
+            (r'"', String.Double, 'string'),
+            (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r"(?s)'.*?'", String.Single),
+            (r';', Punctuation),
+            (r'&', Punctuation),
+            (r'\|', Punctuation),
+            (r'\s+', Whitespace),
+            (r'\d+\b', Number),
+            (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
+            (r'<', Text),
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double),
+            include('interp'),
+        ],
+        'curly': [
+            (r'\}', String.Interpol, '#pop'),
+            (r':-', Keyword),
+            (r'\w+', Name.Variable),
+            (r'[^}:"\'`$\\]+', Punctuation),
+            (r':', Punctuation),
+            include('root'),
+        ],
+        'paren': [
+            (r'\)', Keyword, '#pop'),
+            include('root'),
+        ],
+        'math': [
+            (r'\)\)', Keyword, '#pop'),
+            (r'\*\*|\|\||<<|>>|[-+*/%^|&<>]', Operator),
+            (r'\d+#[\da-zA-Z]+', Number),
+            (r'\d+#(?! )', Number),
+            (r'0[xX][\da-fA-F]+', Number),
+            (r'\d+', Number),
+            (r'[a-zA-Z_]\w*', Name.Variable),  # user variable
+            include('root'),
+        ],
+        'backticks': [
+            (r'`', String.Backtick, '#pop'),
+            include('root'),
+        ],
+    }
+
+    def analyse_text(text):
+        if shebang_matches(text, r'(ba|z|)sh'):
+            return 1
+        if text.startswith('$ '):
+            return 0.2
+
+
+class SlurmBashLexer(BashLexer):
+    """
+    Lexer for (ba|k|z|)sh Slurm scripts.
+    """
+
+    name = 'Slurm'
+    aliases = ['slurm', 'sbatch']
+    filenames = ['*.sl']
+    mimetypes = []
+    version_added = '2.4'
+    EXTRA_KEYWORDS = {'srun'}
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in BashLexer.get_tokens_unprocessed(self, text):
+            if token is Text and value in self.EXTRA_KEYWORDS:
+                yield index, Name.Builtin, value
+            elif token is Comment.Single and 'SBATCH' in value:
+                yield index, Keyword.Pseudo, value
+            else:
+                yield index, token, value
+
+
+class ShellSessionBaseLexer(Lexer):
+    """
+    Base lexer for shell sessions.
+
+    .. versionadded:: 2.1
+    """
+
+    _bare_continuation = False
+    _venv = re.compile(r'^(\([^)]*\))(\s*)')
+
+    def get_tokens_unprocessed(self, text):
+        innerlexer = self._innerLexerCls(**self.options)
+
+        pos = 0
+        curcode = ''
+        insertions = []
+        backslash_continuation = False
+
+        for match in line_re.finditer(text):
+            line = match.group()
+
+            venv_match = self._venv.match(line)
+            if venv_match:
+                venv = venv_match.group(1)
+                venv_whitespace = venv_match.group(2)
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt.VirtualEnv, venv)]))
+                if venv_whitespace:
+                    insertions.append((len(curcode),
+                                       [(0, Text, venv_whitespace)]))
+                line = line[venv_match.end():]
+
+            m = self._ps1rgx.match(line)
+            if m:
+                # To support output lexers (say diff output), the output
+                # needs to be broken by prompts whenever the output lexer
+                # changes.
+                if not insertions:
+                    pos = match.start()
+
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, m.group(1))]))
+                curcode += m.group(2)
+                backslash_continuation = curcode.endswith('\\\n')
+            elif backslash_continuation:
+                if line.startswith(self._ps2):
+                    insertions.append((len(curcode),
+                                       [(0, Generic.Prompt,
+                                         line[:len(self._ps2)])]))
+                    curcode += line[len(self._ps2):]
+                else:
+                    curcode += line
+                backslash_continuation = curcode.endswith('\\\n')
+            elif self._bare_continuation and line.startswith(self._ps2):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt,
+                                     line[:len(self._ps2)])]))
+                curcode += line[len(self._ps2):]
+            else:
+                if insertions:
+                    toks = innerlexer.get_tokens_unprocessed(curcode)
+                    for i, t, v in do_insertions(insertions, toks):
+                        yield pos+i, t, v
+                yield match.start(), Generic.Output, line
+                insertions = []
+                curcode = ''
+        if insertions:
+            for i, t, v in do_insertions(insertions,
+                                         innerlexer.get_tokens_unprocessed(curcode)):
+                yield pos+i, t, v
+
+
+class BashSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for Bash shell sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'Bash Session'
+    aliases = ['console', 'shell-session']
+    filenames = ['*.sh-session', '*.shell-session']
+    mimetypes = ['application/x-shell-session', 'application/x-sh-session']
+    url = 'https://en.wikipedia.org/wiki/Unix_shell'
+    version_added = '1.1'
+    _example = "console/example.sh-session"
+
+    _innerLexerCls = BashLexer
+    _ps1rgx = re.compile(
+        r'^((?:(?:\[.*?\])|(?:\(\S+\))?(?:| |sh\S*?|\w+\S+[@:]\S+(?:\s+\S+)' \
+        r'?|\[\S+[@:][^\n]+\].+))\s*[$#%]\s*)(.*\n?)')
+    _ps2 = '> '
+
+
+class BatchLexer(RegexLexer):
+    """
+    Lexer for the DOS/Windows Batch file format.
+    """
+    name = 'Batchfile'
+    aliases = ['batch', 'bat', 'dosbatch', 'winbatch']
+    filenames = ['*.bat', '*.cmd']
+    mimetypes = ['application/x-dos-batch']
+    url = 'https://en.wikipedia.org/wiki/Batch_file'
+    version_added = '0.7'
+
+    flags = re.MULTILINE | re.IGNORECASE
+
+    _nl = r'\n\x1a'
+    _punct = r'&<>|'
+    _ws = r'\t\v\f\r ,;=\xa0'
+    _nlws = r'\s\x1a\xa0,;='
+    _space = rf'(?:(?:(?:\^[{_nl}])?[{_ws}])+)'
+    _keyword_terminator = (rf'(?=(?:\^[{_nl}]?)?[{_ws}+./:[\\\]]|[{_nl}{_punct}(])')
+    _token_terminator = rf'(?=\^?[{_ws}]|[{_punct}{_nl}])'
+    _start_label = rf'((?:(?<=^[^:])|^[^:]?)[{_ws}]*)(:)'
+    _label = rf'(?:(?:[^{_nlws}{_punct}+:^]|\^[{_nl}]?[\w\W])*)'
+    _label_compound = rf'(?:(?:[^{_nlws}{_punct}+:^)]|\^[{_nl}]?[^)])*)'
+    _number = rf'(?:-?(?:0[0-7]+|0x[\da-f]+|\d+){_token_terminator})'
+    _opword = r'(?:equ|geq|gtr|leq|lss|neq)'
+    _string = rf'(?:"[^{_nl}"]*(?:"|(?=[{_nl}])))'
+    _variable = (r'(?:(?:%(?:\*|(?:~[a-z]*(?:\$[^:]+:)?)?\d|'
+                 rf'[^%:{_nl}]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:[^%{_nl}^]|'
+                 rf'\^[^%{_nl}])[^={_nl}]*=(?:[^%{_nl}^]|\^[^%{_nl}])*)?)?%))|'
+                 rf'(?:\^?![^!:{_nl}]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:'
+                 rf'[^!{_nl}^]|\^[^!{_nl}])[^={_nl}]*=(?:[^!{_nl}^]|\^[^!{_nl}])*)?)?\^?!))')
+    _core_token = rf'(?:(?:(?:\^[{_nl}]?)?[^"{_nlws}{_punct}])+)'
+    _core_token_compound = rf'(?:(?:(?:\^[{_nl}]?)?[^"{_nlws}{_punct})])+)'
+    _token = rf'(?:[{_punct}]+|{_core_token})'
+    _token_compound = rf'(?:[{_punct}]+|{_core_token_compound})'
+    _stoken = (rf'(?:[{_punct}]+|(?:{_string}|{_variable}|{_core_token})+)')
+
+    def _make_begin_state(compound, _core_token=_core_token,
+                          _core_token_compound=_core_token_compound,
+                          _keyword_terminator=_keyword_terminator,
+                          _nl=_nl, _punct=_punct, _string=_string,
+                          _space=_space, _start_label=_start_label,
+                          _stoken=_stoken, _token_terminator=_token_terminator,
+                          _variable=_variable, _ws=_ws):
+        rest = '(?:{}|{}|[^"%{}{}{}])*'.format(_string, _variable, _nl, _punct,
+                                            ')' if compound else '')
+        rest_of_line = rf'(?:(?:[^{_nl}^]|\^[{_nl}]?[\w\W])*)'
+        rest_of_line_compound = rf'(?:(?:[^{_nl}^)]|\^[{_nl}]?[^)])*)'
+        set_space = rf'((?:(?:\^[{_nl}]?)?[^\S\n])*)'
+        suffix = ''
+        if compound:
+            _keyword_terminator = rf'(?:(?=\))|{_keyword_terminator})'
+            _token_terminator = rf'(?:(?=\))|{_token_terminator})'
+            suffix = '/compound'
+        return [
+            ((r'\)', Punctuation, '#pop') if compound else
+             (rf'\)((?=\()|{_token_terminator}){rest_of_line}',
+              Comment.Single)),
+            (rf'(?={_start_label})', Text, f'follow{suffix}'),
+            (_space, using(this, state='text')),
+            include(f'redirect{suffix}'),
+            (rf'[{_nl}]+', Text),
+            (r'\(', Punctuation, 'root/compound'),
+            (r'@+', Punctuation),
+            (rf'((?:for|if|rem)(?:(?=(?:\^[{_nl}]?)?/)|(?:(?!\^)|'
+             rf'(?<=m))(?:(?=\()|{_token_terminator})))({_space}?{_core_token_compound if compound else _core_token}?(?:\^[{_nl}]?)?/(?:\^[{_nl}]?)?\?)',
+             bygroups(Keyword, using(this, state='text')),
+             f'follow{suffix}'),
+            (rf'(goto{_keyword_terminator})({rest}(?:\^[{_nl}]?)?/(?:\^[{_nl}]?)?\?{rest})',
+             bygroups(Keyword, using(this, state='text')),
+             f'follow{suffix}'),
+            (words(('assoc', 'break', 'cd', 'chdir', 'cls', 'color', 'copy',
+                    'date', 'del', 'dir', 'dpath', 'echo', 'endlocal', 'erase',
+                    'exit', 'ftype', 'keys', 'md', 'mkdir', 'mklink', 'move',
+                    'path', 'pause', 'popd', 'prompt', 'pushd', 'rd', 'ren',
+                    'rename', 'rmdir', 'setlocal', 'shift', 'start', 'time',
+                    'title', 'type', 'ver', 'verify', 'vol'),
+                   suffix=_keyword_terminator), Keyword, f'follow{suffix}'),
+            (rf'(call)({_space}?)(:)',
+             bygroups(Keyword, using(this, state='text'), Punctuation),
+             f'call{suffix}'),
+            (rf'call{_keyword_terminator}', Keyword),
+            (rf'(for{_token_terminator}(?!\^))({_space})(/f{_token_terminator})',
+             bygroups(Keyword, using(this, state='text'), Keyword),
+             ('for/f', 'for')),
+            (rf'(for{_token_terminator}(?!\^))({_space})(/l{_token_terminator})',
+             bygroups(Keyword, using(this, state='text'), Keyword),
+             ('for/l', 'for')),
+            (rf'for{_token_terminator}(?!\^)', Keyword, ('for2', 'for')),
+            (rf'(goto{_keyword_terminator})({_space}?)(:?)',
+             bygroups(Keyword, using(this, state='text'), Punctuation),
+             f'label{suffix}'),
+            (rf'(if(?:(?=\()|{_token_terminator})(?!\^))({_space}?)((?:/i{_token_terminator})?)({_space}?)((?:not{_token_terminator})?)({_space}?)',
+             bygroups(Keyword, using(this, state='text'), Keyword,
+                      using(this, state='text'), Keyword,
+                      using(this, state='text')), ('(?', 'if')),
+            (rf'rem(((?=\()|{_token_terminator}){_space}?{_stoken}?.*|{_keyword_terminator}{rest_of_line_compound if compound else rest_of_line})',
+             Comment.Single, f'follow{suffix}'),
+            (rf'(set{_keyword_terminator}){set_space}(/a)',
+             bygroups(Keyword, using(this, state='text'), Keyword),
+             f'arithmetic{suffix}'),
+            (r'(set{}){}((?:/p)?){}((?:(?:(?:\^[{}]?)?[^"{}{}^={}]|'
+             r'\^[{}]?[^"=])+)?)((?:(?:\^[{}]?)?=)?)'.format(_keyword_terminator, set_space, set_space, _nl, _nl, _punct,
+              ')' if compound else '', _nl, _nl),
+             bygroups(Keyword, using(this, state='text'), Keyword,
+                      using(this, state='text'), using(this, state='variable'),
+                      Punctuation),
+             f'follow{suffix}'),
+            default(f'follow{suffix}')
+        ]
+
+    def _make_follow_state(compound, _label=_label,
+                           _label_compound=_label_compound, _nl=_nl,
+                           _space=_space, _start_label=_start_label,
+                           _token=_token, _token_compound=_token_compound,
+                           _ws=_ws):
+        suffix = '/compound' if compound else ''
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state += [
+            (rf'{_start_label}([{_ws}]*)({_label_compound if compound else _label})(.*)',
+             bygroups(Text, Punctuation, Text, Name.Label, Comment.Single)),
+            include(f'redirect{suffix}'),
+            (rf'(?=[{_nl}])', Text, '#pop'),
+            (r'\|\|?|&&?', Punctuation, '#pop'),
+            include('text')
+        ]
+        return state
+
+    def _make_arithmetic_state(compound, _nl=_nl, _punct=_punct,
+                               _string=_string, _variable=_variable,
+                               _ws=_ws, _nlws=_nlws):
+        op = r'=+\-*/!~'
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state += [
+            (r'0[0-7]+', Number.Oct),
+            (r'0x[\da-f]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'[(),]+', Punctuation),
+            (rf'([{op}]|%|\^\^)+', Operator),
+            (r'({}|{}|(\^[{}]?)?[^(){}%\^"{}{}]|\^[{}]?{})+'.format(_string, _variable, _nl, op, _nlws, _punct, _nlws,
+              r'[^)]' if compound else r'[\w\W]'),
+             using(this, state='variable')),
+            (r'(?=[\x00|&])', Text, '#pop'),
+            include('follow')
+        ]
+        return state
+
+    def _make_call_state(compound, _label=_label,
+                         _label_compound=_label_compound):
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state.append((r'(:?)(%s)' % (_label_compound if compound else _label),
+                      bygroups(Punctuation, Name.Label), '#pop'))
+        return state
+
+    def _make_label_state(compound, _label=_label,
+                          _label_compound=_label_compound, _nl=_nl,
+                          _punct=_punct, _string=_string, _variable=_variable):
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state.append((r'({}?)((?:{}|{}|\^[{}]?{}|[^"%^{}{}{}])*)'.format(_label_compound if compound else _label, _string,
+                       _variable, _nl, r'[^)]' if compound else r'[\w\W]', _nl,
+                       _punct, r')' if compound else ''),
+                      bygroups(Name.Label, Comment.Single), '#pop'))
+        return state
+
+    def _make_redirect_state(compound,
+                             _core_token_compound=_core_token_compound,
+                             _nl=_nl, _punct=_punct, _stoken=_stoken,
+                             _string=_string, _space=_space,
+                             _variable=_variable, _nlws=_nlws):
+        stoken_compound = (rf'(?:[{_punct}]+|(?:{_string}|{_variable}|{_core_token_compound})+)')
+        return [
+            (rf'((?:(?<=[{_nlws}])\d)?)(>>?&|<&)([{_nlws}]*)(\d)',
+             bygroups(Number.Integer, Punctuation, Text, Number.Integer)),
+            (rf'((?:(?<=[{_nlws}])(?>?|<)({_space}?{stoken_compound if compound else _stoken})',
+             bygroups(Number.Integer, Punctuation, using(this, state='text')))
+        ]
+
+    tokens = {
+        'root': _make_begin_state(False),
+        'follow': _make_follow_state(False),
+        'arithmetic': _make_arithmetic_state(False),
+        'call': _make_call_state(False),
+        'label': _make_label_state(False),
+        'redirect': _make_redirect_state(False),
+        'root/compound': _make_begin_state(True),
+        'follow/compound': _make_follow_state(True),
+        'arithmetic/compound': _make_arithmetic_state(True),
+        'call/compound': _make_call_state(True),
+        'label/compound': _make_label_state(True),
+        'redirect/compound': _make_redirect_state(True),
+        'variable-or-escape': [
+            (_variable, Name.Variable),
+            (rf'%%|\^[{_nl}]?(\^!|[\w\W])', String.Escape)
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (_variable, Name.Variable),
+            (r'\^!|%%', String.Escape),
+            (rf'[^"%^{_nl}]+|[%^]', String.Double),
+            default('#pop')
+        ],
+        'sqstring': [
+            include('variable-or-escape'),
+            (r'[^%]+|%', String.Single)
+        ],
+        'bqstring': [
+            include('variable-or-escape'),
+            (r'[^%]+|%', String.Backtick)
+        ],
+        'text': [
+            (r'"', String.Double, 'string'),
+            include('variable-or-escape'),
+            (rf'[^"%^{_nlws}{_punct}\d)]+|.', Text)
+        ],
+        'variable': [
+            (r'"', String.Double, 'string'),
+            include('variable-or-escape'),
+            (rf'[^"%^{_nl}]+|.', Name.Variable)
+        ],
+        'for': [
+            (rf'({_space})(in)({_space})(\()',
+             bygroups(using(this, state='text'), Keyword,
+                      using(this, state='text'), Punctuation), '#pop'),
+            include('follow')
+        ],
+        'for2': [
+            (r'\)', Punctuation),
+            (rf'({_space})(do{_token_terminator})',
+             bygroups(using(this, state='text'), Keyword), '#pop'),
+            (rf'[{_nl}]+', Text),
+            include('follow')
+        ],
+        'for/f': [
+            (rf'(")((?:{_variable}|[^"])*?")([{_nlws}]*)(\))',
+             bygroups(String.Double, using(this, state='string'), Text,
+                      Punctuation)),
+            (r'"', String.Double, ('#pop', 'for2', 'string')),
+            (rf"('(?:%%|{_variable}|[\w\W])*?')([{_nlws}]*)(\))",
+             bygroups(using(this, state='sqstring'), Text, Punctuation)),
+            (rf'(`(?:%%|{_variable}|[\w\W])*?`)([{_nlws}]*)(\))',
+             bygroups(using(this, state='bqstring'), Text, Punctuation)),
+            include('for2')
+        ],
+        'for/l': [
+            (r'-?\d+', Number.Integer),
+            include('for2')
+        ],
+        'if': [
+            (rf'((?:cmdextversion|errorlevel){_token_terminator})({_space})(\d+)',
+             bygroups(Keyword, using(this, state='text'),
+                      Number.Integer), '#pop'),
+            (rf'(defined{_token_terminator})({_space})({_stoken})',
+             bygroups(Keyword, using(this, state='text'),
+                      using(this, state='variable')), '#pop'),
+            (rf'(exist{_token_terminator})({_space}{_stoken})',
+             bygroups(Keyword, using(this, state='text')), '#pop'),
+            (rf'({_number}{_space})({_opword})({_space}{_number})',
+             bygroups(using(this, state='arithmetic'), Operator.Word,
+                      using(this, state='arithmetic')), '#pop'),
+            (_stoken, using(this, state='text'), ('#pop', 'if2')),
+        ],
+        'if2': [
+            (rf'({_space}?)(==)({_space}?{_stoken})',
+             bygroups(using(this, state='text'), Operator,
+                      using(this, state='text')), '#pop'),
+            (rf'({_space})({_opword})({_space}{_stoken})',
+             bygroups(using(this, state='text'), Operator.Word,
+                      using(this, state='text')), '#pop')
+        ],
+        '(?': [
+            (_space, using(this, state='text')),
+            (r'\(', Punctuation, ('#pop', 'else?', 'root/compound')),
+            default('#pop')
+        ],
+        'else?': [
+            (_space, using(this, state='text')),
+            (rf'else{_token_terminator}', Keyword, '#pop'),
+            default('#pop')
+        ]
+    }
+
+
+class MSDOSSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for MS DOS shell sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'MSDOS Session'
+    aliases = ['doscon']
+    filenames = []
+    mimetypes = []
+    url = 'https://en.wikipedia.org/wiki/MS-DOS'
+    version_added = '2.1'
+    _example = "doscon/session"
+
+    _innerLexerCls = BatchLexer
+    _ps1rgx = re.compile(r'^([^>]*>)(.*\n?)')
+    _ps2 = 'More? '
+
+
+class TcshLexer(RegexLexer):
+    """
+    Lexer for tcsh scripts.
+    """
+
+    name = 'Tcsh'
+    aliases = ['tcsh', 'csh']
+    filenames = ['*.tcsh', '*.csh']
+    mimetypes = ['application/x-csh']
+    url = 'https://www.tcsh.org'
+    version_added = '0.10'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            (r'\$\(', Keyword, 'paren'),
+            (r'\$\{#?', Keyword, 'curly'),
+            (r'`', String.Backtick, 'backticks'),
+            include('data'),
+        ],
+        'basic': [
+            (r'\b(if|endif|else|while|then|foreach|case|default|'
+             r'break|continue|goto|breaksw|end|switch|endsw)\s*\b',
+             Keyword),
+            (r'\b(alias|alloc|bg|bindkey|builtins|bye|caller|cd|chdir|'
+             r'complete|dirs|echo|echotc|eval|exec|exit|fg|filetest|getxvers|'
+             r'glob|getspath|hashstat|history|hup|inlib|jobs|kill|'
+             r'limit|log|login|logout|ls-F|migrate|newgrp|nice|nohup|notify|'
+             r'onintr|popd|printenv|pushd|rehash|repeat|rootnode|popd|pushd|'
+             r'set|shift|sched|setenv|setpath|settc|setty|setxvers|shift|'
+             r'source|stop|suspend|source|suspend|telltc|time|'
+             r'umask|unalias|uncomplete|unhash|universe|unlimit|unset|unsetenv|'
+             r'ver|wait|warp|watchlog|where|which)\s*\b',
+             Name.Builtin),
+            (r'#.*', Comment),
+            (r'\\[\w\W]', String.Escape),
+            (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
+            (r'[\[\]{}()=]+', Operator),
+            (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+            (r';', Punctuation),
+        ],
+        'data': [
+            (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+            (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r'\s+', Text),
+            (r'[^=\s\[\]{}()$"\'`\\;#]+', Text),
+            (r'\d+(?= |\Z)', Number),
+            (r'\$#?(\w+|.)', Name.Variable),
+        ],
+        'curly': [
+            (r'\}', Keyword, '#pop'),
+            (r':-', Keyword),
+            (r'\w+', Name.Variable),
+            (r'[^}:"\'`$]+', Punctuation),
+            (r':', Punctuation),
+            include('root'),
+        ],
+        'paren': [
+            (r'\)', Keyword, '#pop'),
+            include('root'),
+        ],
+        'backticks': [
+            (r'`', String.Backtick, '#pop'),
+            include('root'),
+        ],
+    }
+
+
+class TcshSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for Tcsh sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'Tcsh Session'
+    aliases = ['tcshcon']
+    filenames = []
+    mimetypes = []
+    url = 'https://www.tcsh.org'
+    version_added = '2.1'
+    _example = "tcshcon/session"
+
+    _innerLexerCls = TcshLexer
+    _ps1rgx = re.compile(r'^([^>]+>)(.*\n?)')
+    _ps2 = '? '
+
+
+class PowerShellLexer(RegexLexer):
+    """
+    For Windows PowerShell code.
+    """
+    name = 'PowerShell'
+    aliases = ['powershell', 'pwsh', 'posh', 'ps1', 'psm1']
+    filenames = ['*.ps1', '*.psm1']
+    mimetypes = ['text/x-powershell']
+    url = 'https://learn.microsoft.com/en-us/powershell'
+    version_added = '1.5'
+
+    flags = re.DOTALL | re.IGNORECASE | re.MULTILINE
+
+    keywords = (
+        'while validateset validaterange validatepattern validatelength '
+        'validatecount until trap switch return ref process param parameter in '
+        'if global: local: function foreach for finally filter end elseif else '
+        'dynamicparam do default continue cmdletbinding break begin alias \\? '
+        '% #script #private #local #global mandatory parametersetname position '
+        'valuefrompipeline valuefrompipelinebypropertyname '
+        'valuefromremainingarguments helpmessage try catch throw').split()
+
+    operators = (
+        'and as band bnot bor bxor casesensitive ccontains ceq cge cgt cle '
+        'clike clt cmatch cne cnotcontains cnotlike cnotmatch contains '
+        'creplace eq exact f file ge gt icontains ieq ige igt ile ilike ilt '
+        'imatch ine inotcontains inotlike inotmatch ireplace is isnot le like '
+        'lt match ne not notcontains notlike notmatch or regex replace '
+        'wildcard').split()
+
+    verbs = (
+        'write where watch wait use update unregister unpublish unprotect '
+        'unlock uninstall undo unblock trace test tee take sync switch '
+        'suspend submit stop step start split sort skip show set send select '
+        'search scroll save revoke resume restore restart resolve resize '
+        'reset request repair rename remove register redo receive read push '
+        'publish protect pop ping out optimize open new move mount merge '
+        'measure lock limit join invoke install initialize import hide group '
+        'grant get format foreach find export expand exit enter enable edit '
+        'dismount disconnect disable deny debug cxnew copy convertto '
+        'convertfrom convert connect confirm compress complete compare close '
+        'clear checkpoint block backup assert approve aggregate add').split()
+
+    aliases_ = (
+        'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn '
+        'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal '
+        'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm '
+        'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi '
+        'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp '
+        'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv '
+        'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo '
+        'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select '
+        'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee '
+        'trcm type wget where wjb write').split()
+
+    commenthelp = (
+        'component description example externalhelp forwardhelpcategory '
+        'forwardhelptargetname functionality inputs link '
+        'notes outputs parameter remotehelprunspace role synopsis').split()
+
+    tokens = {
+        'root': [
+            # we need to count pairs of parentheses for correct highlight
+            # of '$(...)' blocks in strings
+            (r'\(', Punctuation, 'child'),
+            (r'\s+', Text),
+            (r'^(\s*#[#\s]*)(\.(?:{}))([^\n]*$)'.format('|'.join(commenthelp)),
+             bygroups(Comment, String.Doc, Comment)),
+            (r'#[^\n]*?$', Comment),
+            (r'(<|<)#', Comment.Multiline, 'multline'),
+            (r'@"\n', String.Heredoc, 'heredoc-double'),
+            (r"@'\n.*?\n'@", String.Heredoc),
+            # escaped syntax
+            (r'`[\'"$@-]', Punctuation),
+            (r'"', String.Double, 'string'),
+            (r"'([^']|'')*'", String.Single),
+            (r'(\$|@@|@)((global|script|private|env):)?\w+',
+             Name.Variable),
+            (r'({})\b'.format('|'.join(keywords)), Keyword),
+            (r'-({})\b'.format('|'.join(operators)), Operator),
+            (r'({})-[a-z_]\w*\b'.format('|'.join(verbs)), Name.Builtin),
+            (r'({})\s'.format('|'.join(aliases_)), Name.Builtin),
+            (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant),  # .net [type]s
+            (r'-[a-z_]\w*', Name),
+            (r'\w+', Name),
+            (r'[.,;:@{}\[\]$()=+*/\\&%!~?^`|<>-]', Punctuation),
+        ],
+        'child': [
+            (r'\)', Punctuation, '#pop'),
+            include('root'),
+        ],
+        'multline': [
+            (r'[^#&.]+', Comment.Multiline),
+            (r'#(>|>)', Comment.Multiline, '#pop'),
+            (r'\.({})'.format('|'.join(commenthelp)), String.Doc),
+            (r'[#&.]', Comment.Multiline),
+        ],
+        'string': [
+            (r"`[0abfnrtv'\"$`]", String.Escape),
+            (r'[^$`"]+', String.Double),
+            (r'\$\(', Punctuation, 'child'),
+            (r'""', String.Double),
+            (r'[`$]', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'heredoc-double': [
+            (r'\n"@', String.Heredoc, '#pop'),
+            (r'\$\(', Punctuation, 'child'),
+            (r'[^@\n]+"]', String.Heredoc),
+            (r".", String.Heredoc),
+        ]
+    }
+
+
+class PowerShellSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for PowerShell sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'PowerShell Session'
+    aliases = ['pwsh-session', 'ps1con']
+    filenames = []
+    mimetypes = []
+    url = 'https://learn.microsoft.com/en-us/powershell'
+    version_added = '2.1'
+    _example = "pwsh-session/session"
+
+    _innerLexerCls = PowerShellLexer
+    _bare_continuation = True
+    _ps1rgx = re.compile(r'^((?:\[[^]]+\]: )?PS[^>]*> ?)(.*\n?)')
+    _ps2 = '> '
+
+
+class FishShellLexer(RegexLexer):
+    """
+    Lexer for Fish shell scripts.
+    """
+
+    name = 'Fish'
+    aliases = ['fish', 'fishshell']
+    filenames = ['*.fish', '*.load']
+    mimetypes = ['application/x-fish']
+    url = 'https://fishshell.com'
+    version_added = '2.1'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            include('data'),
+            include('interp'),
+        ],
+        'interp': [
+            (r'\$\(\(', Keyword, 'math'),
+            (r'\(', Keyword, 'paren'),
+            (r'\$#?(\w+|.)', Name.Variable),
+        ],
+        'basic': [
+            (r'\b(begin|end|if|else|while|break|for|in|return|function|block|'
+             r'case|continue|switch|not|and|or|set|echo|exit|pwd|true|false|'
+             r'cd|count|test)(\s*)\b',
+             bygroups(Keyword, Text)),
+            (r'\b(alias|bg|bind|breakpoint|builtin|command|commandline|'
+             r'complete|contains|dirh|dirs|emit|eval|exec|fg|fish|fish_config|'
+             r'fish_indent|fish_pager|fish_prompt|fish_right_prompt|'
+             r'fish_update_completions|fishd|funced|funcsave|functions|help|'
+             r'history|isatty|jobs|math|mimedb|nextd|open|popd|prevd|psub|'
+             r'pushd|random|read|set_color|source|status|trap|type|ulimit|'
+             r'umask|vared|fc|getopts|hash|kill|printf|time|wait)\s*\b(?!\.)',
+             Name.Builtin),
+            (r'#.*\n', Comment),
+            (r'\\[\w\W]', String.Escape),
+            (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Whitespace, Operator)),
+            (r'[\[\]()=]', Operator),
+            (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+        ],
+        'data': [
+            (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double),
+            (r'"', String.Double, 'string'),
+            (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r"(?s)'.*?'", String.Single),
+            (r';', Punctuation),
+            (r'&|\||\^|<|>', Operator),
+            (r'\s+', Text),
+            (r'\d+(?= |\Z)', Number),
+            (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double),
+            include('interp'),
+        ],
+        'paren': [
+            (r'\)', Keyword, '#pop'),
+            include('root'),
+        ],
+        'math': [
+            (r'\)\)', Keyword, '#pop'),
+            (r'[-+*/%^|&]|\*\*|\|\|', Operator),
+            (r'\d+#\d+', Number),
+            (r'\d+#(?! )', Number),
+            (r'\d+', Number),
+            include('root'),
+        ],
+    }
+
+class ExeclineLexer(RegexLexer):
+    """
+    Lexer for Laurent Bercot's execline language.
+    """
+
+    name = 'execline'
+    aliases = ['execline']
+    filenames = ['*.exec']
+    url = 'https://skarnet.org/software/execline'
+    version_added = '2.7'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            include('data'),
+            include('interp')
+        ],
+        'interp': [
+            (r'\$\{', String.Interpol, 'curly'),
+            (r'\$[\w@#]+', Name.Variable),  # user variable
+            (r'\$', Text),
+        ],
+        'basic': [
+            (r'\b(background|backtick|cd|define|dollarat|elgetopt|'
+             r'elgetpositionals|elglob|emptyenv|envfile|exec|execlineb|'
+             r'exit|export|fdblock|fdclose|fdmove|fdreserve|fdswap|'
+             r'forbacktickx|foreground|forstdin|forx|getcwd|getpid|heredoc|'
+             r'homeof|if|ifelse|ifte|ifthenelse|importas|loopwhilex|'
+             r'multidefine|multisubstitute|pipeline|piperw|posix-cd|'
+             r'redirfd|runblock|shift|trap|tryexec|umask|unexport|wait|'
+             r'withstdinas)\b', Name.Builtin),
+            (r'\A#!.+\n', Comment.Hashbang),
+            (r'#.*\n', Comment.Single),
+            (r'[{}]', Operator)
+        ],
+        'data': [
+            (r'(?s)"(\\.|[^"\\$])*"', String.Double),
+            (r'"', String.Double, 'string'),
+            (r'\s+', Text),
+            (r'[^\s{}$"\\]+', Text)
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'(?s)(\\\\|\\.|[^"\\$])+', String.Double),
+            include('interp'),
+        ],
+        'curly': [
+            (r'\}', String.Interpol, '#pop'),
+            (r'[\w#@]+', Name.Variable),
+            include('root')
+        ]
+
+    }
+
+    def analyse_text(text):
+        if shebang_matches(text, r'execlineb'):
+            return 1
diff --git a/lib/pygments/lexers/sieve.py b/lib/pygments/lexers/sieve.py
new file mode 100644
index 0000000..fc48980
--- /dev/null
+++ b/lib/pygments/lexers/sieve.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.sieve
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Sieve file format.
+
+    https://tools.ietf.org/html/rfc5228
+    https://tools.ietf.org/html/rfc5173
+    https://tools.ietf.org/html/rfc5229
+    https://tools.ietf.org/html/rfc5230
+    https://tools.ietf.org/html/rfc5232
+    https://tools.ietf.org/html/rfc5235
+    https://tools.ietf.org/html/rfc5429
+    https://tools.ietf.org/html/rfc8580
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Comment, Name, Literal, String, Text, Punctuation, \
+    Keyword
+
+__all__ = ["SieveLexer"]
+
+
+class SieveLexer(RegexLexer):
+    """
+    Lexer for sieve format.
+    """
+    name = 'Sieve'
+    filenames = ['*.siv', '*.sieve']
+    aliases = ['sieve']
+    url = 'https://en.wikipedia.org/wiki/Sieve_(mail_filtering_language)'
+    version_added = '2.6'
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'[();,{}\[\]]', Punctuation),
+            # import:
+            (r'(?i)require',
+             Keyword.Namespace),
+            # tags:
+            (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|'
+             r'count|days|detail|domain|fcc|flags|from|handle|importance|is|'
+             r'localpart|length|lowerfirst|lower|matches|message|mime|options|'
+             r'over|percent|quotewildcard|raw|regex|specialuse|subject|text|'
+             r'under|upperfirst|upper|value)',
+             bygroups(Name.Tag, Name.Tag)),
+            # tokens:
+            (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|'
+             r'ereject|exists|false|fileinto|if|hasflag|header|keep|'
+             r'notify_method_capability|notify|not|redirect|reject|removeflag|'
+             r'setflag|size|spamtest|stop|string|true|vacation|virustest)',
+             Name.Builtin),
+            (r'(?i)set',
+             Keyword.Declaration),
+            # number:
+            (r'([0-9.]+)([kmgKMG])?',
+             bygroups(Literal.Number, Literal.Number)),
+            # comment:
+            (r'#.*$',
+             Comment.Single),
+            (r'/\*.*\*/',
+             Comment.Multiline),
+            # string:
+            (r'"[^"]*?"',
+             String),
+            # text block:
+            (r'text:',
+             Name.Tag, 'text'),
+        ],
+        'text': [
+            (r'[^.].*?\n', String),
+            (r'^\.', Punctuation, "#pop"),
+        ]
+    }
diff --git a/lib/pygments/lexers/slash.py b/lib/pygments/lexers/slash.py
new file mode 100644
index 0000000..1c439d0
--- /dev/null
+++ b/lib/pygments/lexers/slash.py
@@ -0,0 +1,183 @@
+"""
+    pygments.lexers.slash
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Slash programming language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer
+from pygments.token import Name, Number, String, Comment, Punctuation, \
+    Other, Keyword, Operator, Whitespace
+
+__all__ = ['SlashLexer']
+
+
+class SlashLanguageLexer(ExtendedRegexLexer):
+    _nkw = r'(?=[^a-zA-Z_0-9])'
+
+    def move_state(new_state):
+        return ("#pop", new_state)
+
+    def right_angle_bracket(lexer, match, ctx):
+        if len(ctx.stack) > 1 and ctx.stack[-2] == "string":
+            ctx.stack.pop()
+        yield match.start(), String.Interpol, '}'
+        ctx.pos = match.end()
+        pass
+
+    tokens = {
+        "root": [
+            (r"<%=",        Comment.Preproc,    move_state("slash")),
+            (r"<%!!",       Comment.Preproc,    move_state("slash")),
+            (r"<%#.*?%>",   Comment.Multiline),
+            (r"<%",         Comment.Preproc,    move_state("slash")),
+            (r".|\n",       Other),
+        ],
+        "string": [
+            (r"\\",         String.Escape,      move_state("string_e")),
+            (r"\"",         String,             move_state("slash")),
+            (r"#\{",        String.Interpol,    "slash"),
+            (r'.|\n',       String),
+        ],
+        "string_e": [
+            (r'n',                  String.Escape,      move_state("string")),
+            (r't',                  String.Escape,      move_state("string")),
+            (r'r',                  String.Escape,      move_state("string")),
+            (r'e',                  String.Escape,      move_state("string")),
+            (r'x[a-fA-F0-9]{2}',    String.Escape,      move_state("string")),
+            (r'.',                  String.Escape,      move_state("string")),
+        ],
+        "regexp": [
+            (r'}[a-z]*',            String.Regex,       move_state("slash")),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+            (r'.|\n',               String.Regex),
+        ],
+        "regexp_r": [
+            (r'}[a-z]*',            String.Regex,       "#pop"),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+        ],
+        "slash": [
+            (r"%>",                     Comment.Preproc,    move_state("root")),
+            (r"\"",                     String,             move_state("string")),
+            (r"'[a-zA-Z0-9_]+",         String),
+            (r'%r{',                    String.Regex,       move_state("regexp")),
+            (r'/\*.*?\*/',              Comment.Multiline),
+            (r"(#|//).*?\n",            Comment.Single),
+            (r'-?[0-9]+e[+-]?[0-9]+',   Number.Float),
+            (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),
+            (r'-?[0-9]+',               Number.Integer),
+            (r'nil'+_nkw,               Name.Builtin),
+            (r'true'+_nkw,              Name.Builtin),
+            (r'false'+_nkw,             Name.Builtin),
+            (r'self'+_nkw,              Name.Builtin),
+            (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)',
+                bygroups(Keyword, Whitespace, Name.Class)),
+            (r'class'+_nkw,             Keyword),
+            (r'extends'+_nkw,           Keyword),
+            (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)),
+            (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Function)),
+            (r'def'+_nkw,               Keyword),
+            (r'if'+_nkw,                Keyword),
+            (r'elsif'+_nkw,             Keyword),
+            (r'else'+_nkw,              Keyword),
+            (r'unless'+_nkw,            Keyword),
+            (r'for'+_nkw,               Keyword),
+            (r'in'+_nkw,                Keyword),
+            (r'while'+_nkw,             Keyword),
+            (r'until'+_nkw,             Keyword),
+            (r'and'+_nkw,               Keyword),
+            (r'or'+_nkw,                Keyword),
+            (r'not'+_nkw,               Keyword),
+            (r'lambda'+_nkw,            Keyword),
+            (r'try'+_nkw,               Keyword),
+            (r'catch'+_nkw,             Keyword),
+            (r'return'+_nkw,            Keyword),
+            (r'next'+_nkw,              Keyword),
+            (r'last'+_nkw,              Keyword),
+            (r'throw'+_nkw,             Keyword),
+            (r'use'+_nkw,               Keyword),
+            (r'switch'+_nkw,            Keyword),
+            (r'\\',                     Keyword),
+            (r'λ',                      Keyword),
+            (r'__FILE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'__LINE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant),
+            (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name),
+            (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance),
+            (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class),
+            (r'\(',                     Punctuation),
+            (r'\)',                     Punctuation),
+            (r'\[',                     Punctuation),
+            (r'\]',                     Punctuation),
+            (r'\{',                     Punctuation),
+            (r'\}',                     right_angle_bracket),
+            (r';',                      Punctuation),
+            (r',',                      Punctuation),
+            (r'<<=',                    Operator),
+            (r'>>=',                    Operator),
+            (r'<<',                     Operator),
+            (r'>>',                     Operator),
+            (r'==',                     Operator),
+            (r'!=',                     Operator),
+            (r'=>',                     Operator),
+            (r'=',                      Operator),
+            (r'<=>',                    Operator),
+            (r'<=',                     Operator),
+            (r'>=',                     Operator),
+            (r'<',                      Operator),
+            (r'>',                      Operator),
+            (r'\+\+',                   Operator),
+            (r'\+=',                    Operator),
+            (r'-=',                     Operator),
+            (r'\*\*=',                  Operator),
+            (r'\*=',                    Operator),
+            (r'\*\*',                   Operator),
+            (r'\*',                     Operator),
+            (r'/=',                     Operator),
+            (r'\+',                     Operator),
+            (r'-',                      Operator),
+            (r'/',                      Operator),
+            (r'%=',                     Operator),
+            (r'%',                      Operator),
+            (r'^=',                     Operator),
+            (r'&&=',                    Operator),
+            (r'&=',                     Operator),
+            (r'&&',                     Operator),
+            (r'&',                      Operator),
+            (r'\|\|=',                  Operator),
+            (r'\|=',                    Operator),
+            (r'\|\|',                   Operator),
+            (r'\|',                     Operator),
+            (r'!',                      Operator),
+            (r'\.\.\.',                 Operator),
+            (r'\.\.',                   Operator),
+            (r'\.',                     Operator),
+            (r'::',                     Operator),
+            (r':',                      Operator),
+            (r'(\s|\n)+',               Whitespace),
+            (r'[a-z_][a-zA-Z0-9_\']*',  Name.Variable),
+        ],
+    }
+
+
+class SlashLexer(DelegatingLexer):
+    """
+    Lexer for the Slash programming language.
+    """
+
+    name = 'Slash'
+    aliases = ['slash']
+    filenames = ['*.sla']
+    url = 'https://github.com/arturadib/Slash-A'
+    version_added = '2.4'
+
+    def __init__(self, **options):
+        from pygments.lexers.web import HtmlLexer
+        super().__init__(HtmlLexer, SlashLanguageLexer, **options)
diff --git a/lib/pygments/lexers/smalltalk.py b/lib/pygments/lexers/smalltalk.py
new file mode 100644
index 0000000..674b7b4
--- /dev/null
+++ b/lib/pygments/lexers/smalltalk.py
@@ -0,0 +1,194 @@
+"""
+    pygments.lexers.smalltalk
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Smalltalk and related languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['SmalltalkLexer', 'NewspeakLexer']
+
+
+class SmalltalkLexer(RegexLexer):
+    """
+    For Smalltalk syntax.
+    Contributed by Stefan Matthias Aust.
+    Rewritten by Nils Winter.
+    """
+    name = 'Smalltalk'
+    url = 'http://www.smalltalk.org/'
+    filenames = ['*.st']
+    aliases = ['smalltalk', 'squeak', 'st']
+    mimetypes = ['text/x-smalltalk']
+    version_added = '0.10'
+
+    tokens = {
+        'root': [
+            (r'(<)(\w+:)(.*?)(>)', bygroups(Text, Keyword, Text, Text)),
+            include('squeak fileout'),
+            include('whitespaces'),
+            include('method definition'),
+            (r'(\|)([\w\s]*)(\|)', bygroups(Operator, Name.Variable, Operator)),
+            include('objects'),
+            (r'\^|\:=|\_', Operator),
+            # temporaries
+            (r'[\]({}.;!]', Text),
+        ],
+        'method definition': [
+            # Not perfect can't allow whitespaces at the beginning and the
+            # without breaking everything
+            (r'([a-zA-Z]+\w*:)(\s*)(\w+)',
+             bygroups(Name.Function, Text, Name.Variable)),
+            (r'^(\b[a-zA-Z]+\w*\b)(\s*)$', bygroups(Name.Function, Text)),
+            (r'^([-+*/\\~<>=|&!?,@%]+)(\s*)(\w+)(\s*)$',
+             bygroups(Name.Function, Text, Name.Variable, Text)),
+        ],
+        'blockvariables': [
+            include('whitespaces'),
+            (r'(:)(\s*)(\w+)',
+             bygroups(Operator, Text, Name.Variable)),
+            (r'\|', Operator, '#pop'),
+            default('#pop'),  # else pop
+        ],
+        'literals': [
+            (r"'(''|[^'])*'", String, 'afterobject'),
+            (r'\$.', String.Char, 'afterobject'),
+            (r'#\(', String.Symbol, 'parenth'),
+            (r'\)', Text, 'afterobject'),
+            (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number, 'afterobject'),
+        ],
+        '_parenth_helper': [
+            include('whitespaces'),
+            (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number),
+            (r'[-+*/\\~<>=|&#!?,@%\w:]+', String.Symbol),
+            # literals
+            (r"'(''|[^'])*'", String),
+            (r'\$.', String.Char),
+            (r'#*\(', String.Symbol, 'inner_parenth'),
+        ],
+        'parenth': [
+            # This state is a bit tricky since
+            # we can't just pop this state
+            (r'\)', String.Symbol, ('root', 'afterobject')),
+            include('_parenth_helper'),
+        ],
+        'inner_parenth': [
+            (r'\)', String.Symbol, '#pop'),
+            include('_parenth_helper'),
+        ],
+        'whitespaces': [
+            # skip whitespace and comments
+            (r'\s+', Text),
+            (r'"(""|[^"])*"', Comment),
+        ],
+        'objects': [
+            (r'\[', Text, 'blockvariables'),
+            (r'\]', Text, 'afterobject'),
+            (r'\b(self|super|true|false|nil|thisContext)\b',
+             Name.Builtin.Pseudo, 'afterobject'),
+            (r'\b[A-Z]\w*(?!:)\b', Name.Class, 'afterobject'),
+            (r'\b[a-z]\w*(?!:)\b', Name.Variable, 'afterobject'),
+            (r'#("(""|[^"])*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)',
+             String.Symbol, 'afterobject'),
+            include('literals'),
+        ],
+        'afterobject': [
+            (r'! !$', Keyword, '#pop'),  # squeak chunk delimiter
+            include('whitespaces'),
+            (r'\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)',
+             Name.Builtin, '#pop'),
+            (r'\b(new\b(?!:))', Name.Builtin),
+            (r'\:=|\_', Operator, '#pop'),
+            (r'\b[a-zA-Z]+\w*:', Name.Function, '#pop'),
+            (r'\b[a-zA-Z]+\w*', Name.Function),
+            (r'\w+:?|[-+*/\\~<>=|&!?,@%]+', Name.Function, '#pop'),
+            (r'\.', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'[\])}]', Text),
+            (r'[\[({]', Text, '#pop'),
+        ],
+        'squeak fileout': [
+            # Squeak fileout format (optional)
+            (r'^"(""|[^"])*"!', Keyword),
+            (r"^'(''|[^'])*'!", Keyword),
+            (r'^(!)(\w+)( commentStamp: )(.*?)( prior: .*?!\n)(.*?)(!)',
+                bygroups(Keyword, Name.Class, Keyword, String, Keyword, Text, Keyword)),
+            (r"^(!)(\w+(?: class)?)( methodsFor: )('(?:''|[^'])*')(.*?!)",
+                bygroups(Keyword, Name.Class, Keyword, String, Keyword)),
+            (r'^(\w+)( subclass: )(#\w+)'
+             r'(\s+instanceVariableNames: )(.*?)'
+             r'(\s+classVariableNames: )(.*?)'
+             r'(\s+poolDictionaries: )(.*?)'
+             r'(\s+category: )(.*?)(!)',
+                bygroups(Name.Class, Keyword, String.Symbol, Keyword, String, Keyword,
+                         String, Keyword, String, Keyword, String, Keyword)),
+            (r'^(\w+(?: class)?)(\s+instanceVariableNames: )(.*?)(!)',
+                bygroups(Name.Class, Keyword, String, Keyword)),
+            (r'(!\n)(\].*)(! !)$', bygroups(Keyword, Text, Keyword)),
+            (r'! !$', Keyword),
+        ],
+    }
+
+
+class NewspeakLexer(RegexLexer):
+    """
+    For Newspeak syntax.
+    """
+    name = 'Newspeak'
+    url = 'http://newspeaklanguage.org/'
+    filenames = ['*.ns2']
+    aliases = ['newspeak', ]
+    mimetypes = ['text/x-newspeak']
+    version_added = '1.1'
+
+    tokens = {
+        'root': [
+            (r'\b(Newsqueak2)\b', Keyword.Declaration),
+            (r"'[^']*'", String),
+            (r'\b(class)(\s+)(\w+)(\s*)',
+             bygroups(Keyword.Declaration, Text, Name.Class, Text)),
+            (r'\b(mixin|self|super|private|public|protected|nil|true|false)\b',
+             Keyword),
+            (r'(\w+\:)(\s*)([a-zA-Z_]\w+)',
+             bygroups(Name.Function, Text, Name.Variable)),
+            (r'(\w+)(\s*)(=)',
+             bygroups(Name.Attribute, Text, Operator)),
+            (r'<\w+>', Comment.Special),
+            include('expressionstat'),
+            include('whitespace')
+        ],
+
+        'expressionstat': [
+            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+            (r'\d+', Number.Integer),
+            (r':\w+', Name.Variable),
+            (r'(\w+)(::)', bygroups(Name.Variable, Operator)),
+            (r'\w+:', Name.Function),
+            (r'\w+', Name.Variable),
+            (r'\(|\)', Punctuation),
+            (r'\[|\]', Punctuation),
+            (r'\{|\}', Punctuation),
+
+            (r'(\^|\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-|:)', Operator),
+            (r'\.|;', Punctuation),
+            include('whitespace'),
+            include('literals'),
+        ],
+        'literals': [
+            (r'\$.', String),
+            (r"'[^']*'", String),
+            (r"#'[^']*'", String.Symbol),
+            (r"#\w+:?", String.Symbol),
+            (r"#(\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-)+", String.Symbol)
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+            (r'"[^"]*"', Comment)
+        ],
+    }
diff --git a/lib/pygments/lexers/smithy.py b/lib/pygments/lexers/smithy.py
new file mode 100644
index 0000000..bd479ae
--- /dev/null
+++ b/lib/pygments/lexers/smithy.py
@@ -0,0 +1,77 @@
+"""
+    pygments.lexers.smithy
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Smithy IDL.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Keyword, Name, String, \
+    Number, Whitespace, Punctuation
+
+__all__ = ['SmithyLexer']
+
+
+class SmithyLexer(RegexLexer):
+    """
+    For Smithy IDL
+    """
+    name = 'Smithy'
+    url = 'https://awslabs.github.io/smithy/'
+    filenames = ['*.smithy']
+    aliases = ['smithy']
+    version_added = '2.10'
+
+    unquoted = r'[A-Za-z0-9_\.#$-]+'
+    identifier = r"[A-Za-z0-9_\.#$-]+"
+
+    simple_shapes = (
+        'use', 'byte', 'short', 'integer', 'long', 'float', 'document',
+        'double', 'bigInteger', 'bigDecimal', 'boolean', 'blob', 'string',
+        'timestamp',
+    )
+
+    aggregate_shapes = (
+       'apply', 'list', 'map', 'set', 'structure', 'union', 'resource',
+       'operation', 'service', 'trait'
+    )
+
+    tokens = {
+        'root': [
+            (r'///.*$', Comment.Multiline),
+            (r'//.*$', Comment),
+            (r'@[0-9a-zA-Z\.#-]*', Name.Decorator),
+            (r'(=)', Name.Decorator),
+            (r'^(\$version)(:)(.+)',
+                bygroups(Keyword.Declaration, Name.Decorator, Name.Class)),
+            (r'^(namespace)(\s+' + identifier + r')\b',
+                bygroups(Keyword.Declaration, Name.Class)),
+            (words(simple_shapes,
+                   prefix=r'^', suffix=r'(\s+' + identifier + r')\b'),
+                bygroups(Keyword.Declaration, Name.Class)),
+            (words(aggregate_shapes,
+                   prefix=r'^', suffix=r'(\s+' + identifier + r')'),
+                bygroups(Keyword.Declaration, Name.Class)),
+            (r'^(metadata)(\s+)((?:\S+)|(?:\"[^"]+\"))(\s*)(=)',
+                bygroups(Keyword.Declaration, Whitespace, Name.Class,
+                         Whitespace, Name.Decorator)),
+            (r"(true|false|null)", Keyword.Constant),
+            (r"(-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)", Number),
+            (identifier + ":", Name.Label),
+            (identifier, Name.Variable.Class),
+            (r'\[', Text, "#push"),
+            (r'\]', Text, "#pop"),
+            (r'\(', Text, "#push"),
+            (r'\)', Text, "#pop"),
+            (r'\{', Text, "#push"),
+            (r'\}', Text, "#pop"),
+            (r'"{3}(\\\\|\n|\\")*"{3}', String.Doc),
+            (r'"(\\\\|\n|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\n|\\'|[^'])*'", String.Single),
+            (r'[:,]+', Punctuation),
+            (r'\s+', Whitespace),
+        ]
+    }
diff --git a/lib/pygments/lexers/smv.py b/lib/pygments/lexers/smv.py
new file mode 100644
index 0000000..bf97b52
--- /dev/null
+++ b/lib/pygments/lexers/smv.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.smv
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the SMV languages.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, Text
+
+__all__ = ['NuSMVLexer']
+
+
+class NuSMVLexer(RegexLexer):
+    """
+    Lexer for the NuSMV language.
+    """
+
+    name = 'NuSMV'
+    aliases = ['nusmv']
+    filenames = ['*.smv']
+    mimetypes = []
+    url = 'https://nusmv.fbk.eu'
+    version_added = '2.2'
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'(?s)\/\-\-.*?\-\-/', Comment),
+            (r'--.*\n', Comment),
+
+            # Reserved
+            (words(('MODULE', 'DEFINE', 'MDEFINE', 'CONSTANTS', 'VAR', 'IVAR',
+                    'FROZENVAR', 'INIT', 'TRANS', 'INVAR', 'SPEC', 'CTLSPEC',
+                    'LTLSPEC', 'PSLSPEC', 'COMPUTE', 'NAME', 'INVARSPEC',
+                    'FAIRNESS', 'JUSTICE', 'COMPASSION', 'ISA', 'ASSIGN',
+                    'CONSTRAINT', 'SIMPWFF', 'CTLWFF', 'LTLWFF', 'PSLWFF',
+                    'COMPWFF', 'IN', 'MIN', 'MAX', 'MIRROR', 'PRED',
+                    'PREDICATES'), suffix=r'(?![\w$#-])'),
+             Keyword.Declaration),
+            (r'process(?![\w$#-])', Keyword),
+            (words(('array', 'of', 'boolean', 'integer', 'real', 'word'),
+                   suffix=r'(?![\w$#-])'), Keyword.Type),
+            (words(('case', 'esac'), suffix=r'(?![\w$#-])'), Keyword),
+            (words(('word1', 'bool', 'signed', 'unsigned', 'extend', 'resize',
+                    'sizeof', 'uwconst', 'swconst', 'init', 'self', 'count',
+                    'abs', 'max', 'min'), suffix=r'(?![\w$#-])'),
+             Name.Builtin),
+            (words(('EX', 'AX', 'EF', 'AF', 'EG', 'AG', 'E', 'F', 'O', 'G',
+                    'H', 'X', 'Y', 'Z', 'A', 'U', 'S', 'V', 'T', 'BU', 'EBF',
+                    'ABF', 'EBG', 'ABG', 'next', 'mod', 'union', 'in', 'xor',
+                    'xnor'), suffix=r'(?![\w$#-])'),
+                Operator.Word),
+            (words(('TRUE', 'FALSE'), suffix=r'(?![\w$#-])'), Keyword.Constant),
+
+            # Names
+            (r'[a-zA-Z_][\w$#-]*', Name.Variable),
+
+            # Operators
+            (r':=', Operator),
+            (r'[-&|+*/<>!=]', Operator),
+
+            # Literals
+            (r'\-?\d+\b', Number.Integer),
+            (r'0[su][bB]\d*_[01_]+', Number.Bin),
+            (r'0[su][oO]\d*_[0-7_]+', Number.Oct),
+            (r'0[su][dD]\d*_[\d_]+', Number.Decimal),
+            (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex),
+
+            # Whitespace, punctuation and the rest
+            (r'\s+', Text.Whitespace),
+            (r'[()\[\]{};?:.,]', Punctuation),
+        ],
+    }
diff --git a/lib/pygments/lexers/snobol.py b/lib/pygments/lexers/snobol.py
new file mode 100644
index 0000000..bab51e9
--- /dev/null
+++ b/lib/pygments/lexers/snobol.py
@@ -0,0 +1,82 @@
+"""
+    pygments.lexers.snobol
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the SNOBOL language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['SnobolLexer']
+
+
+class SnobolLexer(RegexLexer):
+    """
+    Lexer for the SNOBOL4 programming language.
+
+    Recognizes the common ASCII equivalents of the original SNOBOL4 operators.
+    Does not require spaces around binary operators.
+    """
+
+    name = "Snobol"
+    aliases = ["snobol"]
+    filenames = ['*.snobol']
+    mimetypes = ['text/x-snobol']
+    url = 'https://www.regressive.org/snobol4'
+    version_added = '1.5'
+
+    tokens = {
+        # root state, start of line
+        # comments, continuation lines, and directives start in column 1
+        # as do labels
+        'root': [
+            (r'\*.*\n', Comment),
+            (r'[+.] ', Punctuation, 'statement'),
+            (r'-.*\n', Comment),
+            (r'END\s*\n', Name.Label, 'heredoc'),
+            (r'[A-Za-z$][\w$]*', Name.Label, 'statement'),
+            (r'\s+', Text, 'statement'),
+        ],
+        # statement state, line after continuation or label
+        'statement': [
+            (r'\s*\n', Text, '#pop'),
+            (r'\s+', Text),
+            (r'(?<=[^\w.])(LT|LE|EQ|NE|GE|GT|INTEGER|IDENT|DIFFER|LGT|SIZE|'
+             r'REPLACE|TRIM|DUPL|REMDR|DATE|TIME|EVAL|APPLY|OPSYN|LOAD|UNLOAD|'
+             r'LEN|SPAN|BREAK|ANY|NOTANY|TAB|RTAB|REM|POS|RPOS|FAIL|FENCE|'
+             r'ABORT|ARB|ARBNO|BAL|SUCCEED|INPUT|OUTPUT|TERMINAL)(?=[^\w.])',
+             Name.Builtin),
+            (r'[A-Za-z][\w.]*', Name),
+            # ASCII equivalents of original operators
+            # | for the EBCDIC equivalent, ! likewise
+            # \ for EBCDIC negation
+            (r'\*\*|[?$.!%*/#+\-@|&\\=]', Operator),
+            (r'"[^"]*"', String),
+            (r"'[^']*'", String),
+            # Accept SPITBOL syntax for real numbers
+            # as well as Macro SNOBOL4
+            (r'[0-9]+(?=[^.EeDd])', Number.Integer),
+            (r'[0-9]+(\.[0-9]*)?([EDed][-+]?[0-9]+)?', Number.Float),
+            # Goto
+            (r':', Punctuation, 'goto'),
+            (r'[()<>,;]', Punctuation),
+        ],
+        # Goto block
+        'goto': [
+            (r'\s*\n', Text, "#pop:2"),
+            (r'\s+', Text),
+            (r'F|S', Keyword),
+            (r'(\()([A-Za-z][\w.]*)(\))',
+             bygroups(Punctuation, Name.Label, Punctuation))
+        ],
+        # everything after the END statement is basically one
+        # big heredoc.
+        'heredoc': [
+            (r'.*\n', String.Heredoc)
+        ]
+    }
diff --git a/lib/pygments/lexers/solidity.py b/lib/pygments/lexers/solidity.py
new file mode 100644
index 0000000..3182a14
--- /dev/null
+++ b/lib/pygments/lexers/solidity.py
@@ -0,0 +1,87 @@
+"""
+    pygments.lexers.solidity
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Solidity.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['SolidityLexer']
+
+
+class SolidityLexer(RegexLexer):
+    """
+    For Solidity source code.
+    """
+
+    name = 'Solidity'
+    aliases = ['solidity']
+    filenames = ['*.sol']
+    mimetypes = []
+    url = 'https://soliditylang.org'
+    version_added = '2.5'
+
+    datatype = (
+        r'\b(address|bool|(?:(?:bytes|hash|int|string|uint)(?:8|16|24|32|40|48|56|64'
+        r'|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208'
+        r'|216|224|232|240|248|256)?))\b'
+    )
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+            (r'\bpragma\s+solidity\b', Keyword, 'pragma'),
+            (r'\b(contract)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword, Whitespace, Name.Entity)),
+            (datatype + r'(\s+)((?:external|public|internal|private)\s+)?' +
+             r'([a-zA-Z_]\w*)',
+             bygroups(Keyword.Type, Whitespace, Keyword, Name.Variable)),
+            (r'\b(enum|event|function|struct)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword.Type, Whitespace, Name.Variable)),
+            (r'\b(msg|block|tx)\.([A-Za-z_][a-zA-Z0-9_]*)\b', Keyword),
+            (words((
+                'block', 'break', 'constant', 'constructor', 'continue',
+                'contract', 'do', 'else', 'external', 'false', 'for',
+                'function', 'if', 'import', 'inherited', 'internal', 'is',
+                'library', 'mapping', 'memory', 'modifier', 'msg', 'new',
+                'payable', 'private', 'public', 'require', 'return',
+                'returns', 'struct', 'suicide', 'throw', 'this', 'true',
+                'tx', 'var', 'while'), prefix=r'\b', suffix=r'\b'),
+             Keyword.Type),
+            (words(('keccak256',), prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            (datatype, Keyword.Type),
+            include('constants'),
+            (r'[a-zA-Z_]\w*', Text),
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+            (r'[.;{}(),\[\]]', Punctuation)
+        ],
+        'comments': [
+            (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
+            (r'/(\\\n)?[*][\w\W]*', Comment.Multiline)
+        ],
+        'constants': [
+            (r'("(\\"|.)*?")', String.Double),
+            (r"('(\\'|.)*?')", String.Single),
+            (r'\b0[xX][0-9a-fA-F]+\b', Number.Hex),
+            (r'\b\d+\b', Number.Decimal),
+        ],
+        'pragma': [
+            include('whitespace'),
+            include('comments'),
+            (r'(\^|>=|<)(\s*)(\d+\.\d+\.\d+)',
+             bygroups(Operator, Whitespace, Keyword)),
+            (r';', Punctuation, '#pop')
+        ],
+        'whitespace': [
+            (r'\s+', Whitespace),
+            (r'\n', Whitespace)
+        ]
+    }
diff --git a/lib/pygments/lexers/soong.py b/lib/pygments/lexers/soong.py
new file mode 100644
index 0000000..bbf204d
--- /dev/null
+++ b/lib/pygments/lexers/soong.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.soong
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Soong (Android.bp Blueprint) files.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Comment, Name, Number, Operator, Punctuation, \
+        String, Whitespace
+
+__all__ = ['SoongLexer']
+
+class SoongLexer(RegexLexer):
+    name = 'Soong'
+    version_added = '2.18'
+    url = 'https://source.android.com/docs/setup/reference/androidbp'
+    aliases = ['androidbp', 'bp', 'soong']
+    filenames = ['Android.bp']
+
+    tokens = {
+        'root': [
+            # A variable assignment
+            (r'(\w*)(\s*)(\+?=)(\s*)',
+             bygroups(Name.Variable, Whitespace, Operator, Whitespace),
+             'assign-rhs'),
+
+            # A top-level module
+            (r'(\w*)(\s*)(\{)',
+             bygroups(Name.Function, Whitespace, Punctuation),
+             'in-rule'),
+
+            # Everything else
+            include('comments'),
+            (r'\s+', Whitespace),  # newlines okay
+        ],
+        'assign-rhs': [
+            include('expr'),
+            (r'\n', Whitespace, '#pop'),
+        ],
+        'in-list': [
+            include('expr'),
+            include('comments'),
+            (r'\s+', Whitespace),  # newlines okay in a list
+            (r',', Punctuation),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'in-map': [
+            # A map key
+            (r'(\w+)(:)(\s*)', bygroups(Name, Punctuation, Whitespace)),
+
+            include('expr'),
+            include('comments'),
+            (r'\s+', Whitespace),  # newlines okay in a map
+            (r',', Punctuation),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'in-rule': [
+            # Just re-use map syntax
+            include('in-map'),
+        ],
+        'comments': [
+            (r'//.*', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+        ],
+        'expr': [
+            (r'(true|false)\b', Name.Builtin),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'".*?"', String),
+            (r'\{', Punctuation, 'in-map'),
+            (r'\[', Punctuation, 'in-list'),
+            (r'\w+', Name),
+        ],
+    }
diff --git a/lib/pygments/lexers/sophia.py b/lib/pygments/lexers/sophia.py
new file mode 100644
index 0000000..37fcec5
--- /dev/null
+++ b/lib/pygments/lexers/sophia.py
@@ -0,0 +1,102 @@
+"""
+    pygments.lexers.sophia
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Sophia.
+
+    Derived from pygments/lexers/reason.py.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, default, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+
+__all__ = ['SophiaLexer']
+
+class SophiaLexer(RegexLexer):
+    """
+    A Sophia lexer.
+    """
+
+    name = 'Sophia'
+    aliases = ['sophia']
+    filenames = ['*.aes']
+    mimetypes = []
+    url = 'https://docs.aeternity.com/aesophia'
+    version_added = '2.11'
+
+    keywords = (
+        'contract', 'include', 'let', 'switch', 'type', 'record', 'datatype',
+        'if', 'elif', 'else', 'function', 'stateful', 'payable', 'public',
+        'entrypoint', 'private', 'indexed', 'namespace', 'interface', 'main',
+        'using', 'as', 'for', 'hiding',
+    )
+
+    builtins = ('state', 'put', 'abort', 'require')
+
+    word_operators = ('mod', 'band', 'bor', 'bxor', 'bnot')
+
+    primitive_types = ('int', 'address', 'bool', 'bits', 'bytes', 'string',
+                       'list', 'option', 'char', 'unit', 'map', 'event',
+                       'hash', 'signature', 'oracle', 'oracle_query')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text.Whitespace),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Class, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Function),
+            (r'//.*?\n', Comment.Single),
+            (r'\/\*(?!/)', Comment.Multiline, 'comment'),
+
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'#[\da-fA-F][\da-fA-F_]*', Name.Label),
+            (r'\d[\d_]*', Number.Integer),
+
+            (words(keywords, suffix=r'\b'), Keyword),
+            (words(builtins, suffix=r'\b'), Name.Builtin),
+            (words(word_operators, prefix=r'\b', suffix=r'\b'), Operator.Word),
+            (words(primitive_types, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+
+            (r'[=!<>+\\*/:&|?~@^-]', Operator.Word),
+            (r'[.;:{}(),\[\]]', Punctuation),
+
+            (r"(ak_|ok_|oq_|ct_)[\w']*", Name.Label),
+            (r"[^\W\d][\w']*", Name),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'[a-z][\w]*", Name.Variable),
+
+            (r'"', String.Double, 'string')
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'\/\*', Comment.Multiline, '#push'),
+            (r'\*\/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Function),
+            (r'[A-Z][\w\']*', Name.Function, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
diff --git a/lib/pygments/lexers/special.py b/lib/pygments/lexers/special.py
new file mode 100644
index 0000000..524946f
--- /dev/null
+++ b/lib/pygments/lexers/special.py
@@ -0,0 +1,122 @@
+"""
+    pygments.lexers.special
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Special lexers.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import ast
+
+from pygments.lexer import Lexer, line_re
+from pygments.token import Token, Error, Text, Generic
+from pygments.util import get_choice_opt
+
+
+__all__ = ['TextLexer', 'OutputLexer', 'RawTokenLexer']
+
+
+class TextLexer(Lexer):
+    """
+    "Null" lexer, doesn't highlight anything.
+    """
+    name = 'Text only'
+    aliases = ['text']
+    filenames = ['*.txt']
+    mimetypes = ['text/plain']
+    url = ""
+    version_added = ''
+
+    priority = 0.01
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Text, text
+
+    def analyse_text(text):
+        return TextLexer.priority
+
+
+class OutputLexer(Lexer):
+    """
+    Simple lexer that highlights everything as ``Token.Generic.Output``.
+    """
+    name = 'Text output'
+    aliases = ['output']
+    url = ""
+    version_added = '2.10'
+    _example = "output/output"
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Generic.Output, text
+
+
+_ttype_cache = {}
+
+
+class RawTokenLexer(Lexer):
+    """
+    Recreate a token stream formatted with the `RawTokenFormatter`.
+
+    Additional options accepted:
+
+    `compress`
+        If set to ``"gz"`` or ``"bz2"``, decompress the token stream with
+        the given compression algorithm before lexing (default: ``""``).
+    """
+    name = 'Raw token data'
+    aliases = []
+    filenames = []
+    mimetypes = ['application/x-pygments-tokens']
+    url = 'https://pygments.org/docs/formatters/#RawTokenFormatter'
+    version_added = ''
+
+    def __init__(self, **options):
+        self.compress = get_choice_opt(options, 'compress',
+                                       ['', 'none', 'gz', 'bz2'], '')
+        Lexer.__init__(self, **options)
+
+    def get_tokens(self, text):
+        if self.compress:
+            if isinstance(text, str):
+                text = text.encode('latin1')
+            try:
+                if self.compress == 'gz':
+                    import gzip
+                    text = gzip.decompress(text)
+                elif self.compress == 'bz2':
+                    import bz2
+                    text = bz2.decompress(text)
+            except OSError:
+                yield Error, text.decode('latin1')
+        if isinstance(text, bytes):
+            text = text.decode('latin1')
+
+        # do not call Lexer.get_tokens() because stripping is not optional.
+        text = text.strip('\n') + '\n'
+        for i, t, v in self.get_tokens_unprocessed(text):
+            yield t, v
+
+    def get_tokens_unprocessed(self, text):
+        length = 0
+        for match in line_re.finditer(text):
+            try:
+                ttypestr, val = match.group().rstrip().split('\t', 1)
+                ttype = _ttype_cache.get(ttypestr)
+                if not ttype:
+                    ttype = Token
+                    ttypes = ttypestr.split('.')[1:]
+                    for ttype_ in ttypes:
+                        if not ttype_ or not ttype_[0].isupper():
+                            raise ValueError('malformed token name')
+                        ttype = getattr(ttype, ttype_)
+                    _ttype_cache[ttypestr] = ttype
+                val = ast.literal_eval(val)
+                if not isinstance(val, str):
+                    raise ValueError('expected str')
+            except (SyntaxError, ValueError):
+                val = match.group()
+                ttype = Error
+            yield length, ttype, val
+            length += len(val)
diff --git a/lib/pygments/lexers/spice.py b/lib/pygments/lexers/spice.py
new file mode 100644
index 0000000..9d2b1a1
--- /dev/null
+++ b/lib/pygments/lexers/spice.py
@@ -0,0 +1,70 @@
+"""
+    pygments.lexers.spice
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Spice programming language.
+
+    :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['SpiceLexer']
+
+
+class SpiceLexer(RegexLexer):
+    """
+    For Spice source.
+    """
+    name = 'Spice'
+    url = 'https://www.spicelang.com'
+    filenames = ['*.spice']
+    aliases = ['spice', 'spicelang']
+    mimetypes = ['text/x-spice']
+    version_added = '2.11'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'\\\n', Text),
+            # comments
+            (r'//(.*?)\n', Comment.Single),
+            (r'/(\\\n)?[*]{2}(.|\n)*?[*](\\\n)?/', String.Doc),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            # keywords
+            (r'(import|as)\b', Keyword.Namespace),
+            (r'(f|p|type|struct|interface|enum|alias|operator)\b', Keyword.Declaration),
+            (words(('if', 'else', 'switch', 'case', 'default', 'for', 'foreach', 'do',
+                    'while', 'break', 'continue', 'fallthrough', 'return', 'assert',
+                    'unsafe', 'ext'), suffix=r'\b'), Keyword),
+            (words(('const', 'signed', 'unsigned', 'inline', 'public', 'heap', 'compose'),
+                   suffix=r'\b'), Keyword.Pseudo),
+            (words(('new', 'yield', 'stash', 'pick', 'sync', 'class'), suffix=r'\b'),
+                   Keyword.Reserved),
+            (r'(true|false|nil)\b', Keyword.Constant),
+            (words(('double', 'int', 'short', 'long', 'byte', 'char', 'string',
+                    'bool', 'dyn'), suffix=r'\b'), Keyword.Type),
+            (words(('printf', 'sizeof', 'alignof', 'len', 'panic'), suffix=r'\b(\()'),
+             bygroups(Name.Builtin, Punctuation)),
+            # numeric literals
+            (r'[-]?[0-9]*[.][0-9]+([eE][+-]?[0-9]+)?', Number.Double),
+            (r'0[bB][01]+[slu]?', Number.Bin),
+            (r'0[oO][0-7]+[slu]?', Number.Oct),
+            (r'0[xXhH][0-9a-fA-F]+[slu]?', Number.Hex),
+            (r'(0[dD])?[0-9]+[slu]?', Number.Integer),
+            # string literal
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # char literal
+            (r'\'(\\\\|\\[^\\]|[^\'\\])\'', String.Char),
+            # tokens
+            (r'<<=|>>=|<<|>>|<=|>=|\+=|-=|\*=|/=|\%=|\|=|&=|\^=|&&|\|\||&|\||'
+             r'\+\+|--|\%|\^|\~|==|!=|->|::|[.]{3}|#!|#|[+\-*/&]', Operator),
+            (r'[|<>=!()\[\]{}.,;:\?]', Punctuation),
+            # identifiers
+            (r'[^\W\d]\w*', Name.Other),
+        ]
+    }
diff --git a/lib/pygments/lexers/sql.py b/lib/pygments/lexers/sql.py
new file mode 100644
index 0000000..d3e6f17
--- /dev/null
+++ b/lib/pygments/lexers/sql.py
@@ -0,0 +1,1109 @@
+"""
+    pygments.lexers.sql
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for various SQL dialects and related interactive sessions.
+
+    Postgres specific lexers:
+
+    `PostgresLexer`
+        A SQL lexer for the PostgreSQL dialect. Differences w.r.t. the SQL
+        lexer are:
+
+        - keywords and data types list parsed from the PG docs (run the
+          `_postgres_builtins` module to update them);
+        - Content of $-strings parsed using a specific lexer, e.g. the content
+          of a PL/Python function is parsed using the Python lexer;
+        - parse PG specific constructs: E-strings, $-strings, U&-strings,
+          different operators and punctuation.
+
+    `PlPgsqlLexer`
+        A lexer for the PL/pgSQL language. Adds a few specific construct on
+        top of the PG SQL lexer (such as <{text}' if rule else text
+                    append(text)
+            else:
+                styles: Dict[str, int] = {}
+                for text, style, _ in Segment.filter_control(
+                    Segment.simplify(self._record_buffer)
+                ):
+                    text = escape(text)
+                    if style:
+                        rule = style.get_html_style(_theme)
+                        style_number = styles.setdefault(rule, len(styles) + 1)
+                        if style.link:
+                            text = f'{text}'
+                        else:
+                            text = f'{text}'
+                    append(text)
+                stylesheet_rules: List[str] = []
+                stylesheet_append = stylesheet_rules.append
+                for style_rule, style_number in styles.items():
+                    if style_rule:
+                        stylesheet_append(f".r{style_number} {{{style_rule}}}")
+                stylesheet = "\n".join(stylesheet_rules)
+
+            rendered_code = render_code_format.format(
+                code="".join(fragments),
+                stylesheet=stylesheet,
+                foreground=_theme.foreground_color.hex,
+                background=_theme.background_color.hex,
+            )
+            if clear:
+                del self._record_buffer[:]
+        return rendered_code
+
+    def save_html(
+        self,
+        path: str,
+        *,
+        theme: Optional[TerminalTheme] = None,
+        clear: bool = True,
+        code_format: str = CONSOLE_HTML_FORMAT,
+        inline_styles: bool = False,
+    ) -> None:
+        """Generate HTML from console contents and write to a file (requires record=True argument in constructor).
+
+        Args:
+            path (str): Path to write html file.
+            theme (TerminalTheme, optional): TerminalTheme object containing console colors.
+            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``.
+            code_format (str, optional): Format string to render HTML. In addition to '{foreground}',
+                '{background}', and '{code}', should contain '{stylesheet}' if inline_styles is ``False``.
+            inline_styles (bool, optional): If ``True`` styles will be inlined in to spans, which makes files
+                larger but easier to cut and paste markup. If ``False``, styles will be embedded in a style tag.
+                Defaults to False.
+
+        """
+        html = self.export_html(
+            theme=theme,
+            clear=clear,
+            code_format=code_format,
+            inline_styles=inline_styles,
+        )
+        with open(path, "w", encoding="utf-8") as write_file:
+            write_file.write(html)
+
+    def export_svg(
+        self,
+        *,
+        title: str = "Rich",
+        theme: Optional[TerminalTheme] = None,
+        clear: bool = True,
+        code_format: str = CONSOLE_SVG_FORMAT,
+        font_aspect_ratio: float = 0.61,
+        unique_id: Optional[str] = None,
+    ) -> str:
+        """
+        Generate an SVG from the console contents (requires record=True in Console constructor).
+
+        Args:
+            title (str, optional): The title of the tab in the output image
+            theme (TerminalTheme, optional): The ``TerminalTheme`` object to use to style the terminal
+            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``
+            code_format (str, optional): Format string used to generate the SVG. Rich will inject a number of variables
+                into the string in order to form the final SVG output. The default template used and the variables
+                injected by Rich can be found by inspecting the ``console.CONSOLE_SVG_FORMAT`` variable.
+            font_aspect_ratio (float, optional): The width to height ratio of the font used in the ``code_format``
+                string. Defaults to 0.61, which is the width to height ratio of Fira Code (the default font).
+                If you aren't specifying a different font inside ``code_format``, you probably don't need this.
+            unique_id (str, optional): unique id that is used as the prefix for various elements (CSS styles, node
+                ids). If not set, this defaults to a computed value based on the recorded content.
+        """
+
+        from rich.cells import cell_len
+
+        style_cache: Dict[Style, str] = {}
+
+        def get_svg_style(style: Style) -> str:
+            """Convert a Style to CSS rules for SVG."""
+            if style in style_cache:
+                return style_cache[style]
+            css_rules = []
+            color = (
+                _theme.foreground_color
+                if (style.color is None or style.color.is_default)
+                else style.color.get_truecolor(_theme)
+            )
+            bgcolor = (
+                _theme.background_color
+                if (style.bgcolor is None or style.bgcolor.is_default)
+                else style.bgcolor.get_truecolor(_theme)
+            )
+            if style.reverse:
+                color, bgcolor = bgcolor, color
+            if style.dim:
+                color = blend_rgb(color, bgcolor, 0.4)
+            css_rules.append(f"fill: {color.hex}")
+            if style.bold:
+                css_rules.append("font-weight: bold")
+            if style.italic:
+                css_rules.append("font-style: italic;")
+            if style.underline:
+                css_rules.append("text-decoration: underline;")
+            if style.strike:
+                css_rules.append("text-decoration: line-through;")
+
+            css = ";".join(css_rules)
+            style_cache[style] = css
+            return css
+
+        _theme = theme or SVG_EXPORT_THEME
+
+        width = self.width
+        char_height = 20
+        char_width = char_height * font_aspect_ratio
+        line_height = char_height * 1.22
+
+        margin_top = 1
+        margin_right = 1
+        margin_bottom = 1
+        margin_left = 1
+
+        padding_top = 40
+        padding_right = 8
+        padding_bottom = 8
+        padding_left = 8
+
+        padding_width = padding_left + padding_right
+        padding_height = padding_top + padding_bottom
+        margin_width = margin_left + margin_right
+        margin_height = margin_top + margin_bottom
+
+        text_backgrounds: List[str] = []
+        text_group: List[str] = []
+        classes: Dict[str, int] = {}
+        style_no = 1
+
+        def escape_text(text: str) -> str:
+            """HTML escape text and replace spaces with nbsp."""
+            return escape(text).replace(" ", " ")
+
+        def make_tag(
+            name: str, content: Optional[str] = None, **attribs: object
+        ) -> str:
+            """Make a tag from name, content, and attributes."""
+
+            def stringify(value: object) -> str:
+                if isinstance(value, (float)):
+                    return format(value, "g")
+                return str(value)
+
+            tag_attribs = " ".join(
+                f'{k.lstrip("_").replace("_", "-")}="{stringify(v)}"'
+                for k, v in attribs.items()
+            )
+            return (
+                f"<{name} {tag_attribs}>{content}"
+                if content
+                else f"<{name} {tag_attribs}/>"
+            )
+
+        with self._record_buffer_lock:
+            segments = list(Segment.filter_control(self._record_buffer))
+            if clear:
+                self._record_buffer.clear()
+
+        if unique_id is None:
+            unique_id = "terminal-" + str(
+                zlib.adler32(
+                    ("".join(repr(segment) for segment in segments)).encode(
+                        "utf-8",
+                        "ignore",
+                    )
+                    + title.encode("utf-8", "ignore")
+                )
+            )
+        y = 0
+        for y, line in enumerate(Segment.split_and_crop_lines(segments, length=width)):
+            x = 0
+            for text, style, _control in line:
+                style = style or Style()
+                rules = get_svg_style(style)
+                if rules not in classes:
+                    classes[rules] = style_no
+                    style_no += 1
+                class_name = f"r{classes[rules]}"
+
+                if style.reverse:
+                    has_background = True
+                    background = (
+                        _theme.foreground_color.hex
+                        if style.color is None
+                        else style.color.get_truecolor(_theme).hex
+                    )
+                else:
+                    bgcolor = style.bgcolor
+                    has_background = bgcolor is not None and not bgcolor.is_default
+                    background = (
+                        _theme.background_color.hex
+                        if style.bgcolor is None
+                        else style.bgcolor.get_truecolor(_theme).hex
+                    )
+
+                text_length = cell_len(text)
+                if has_background:
+                    text_backgrounds.append(
+                        make_tag(
+                            "rect",
+                            fill=background,
+                            x=x * char_width,
+                            y=y * line_height + 1.5,
+                            width=char_width * text_length,
+                            height=line_height + 0.25,
+                            shape_rendering="crispEdges",
+                        )
+                    )
+
+                if text != " " * len(text):
+                    text_group.append(
+                        make_tag(
+                            "text",
+                            escape_text(text),
+                            _class=f"{unique_id}-{class_name}",
+                            x=x * char_width,
+                            y=y * line_height + char_height,
+                            textLength=char_width * len(text),
+                            clip_path=f"url(#{unique_id}-line-{y})",
+                        )
+                    )
+                x += cell_len(text)
+
+        line_offsets = [line_no * line_height + 1.5 for line_no in range(y)]
+        lines = "\n".join(
+            f"""
+    {make_tag("rect", x=0, y=offset, width=char_width * width, height=line_height + 0.25)}
+            """
+            for line_no, offset in enumerate(line_offsets)
+        )
+
+        styles = "\n".join(
+            f".{unique_id}-r{rule_no} {{ {css} }}" for css, rule_no in classes.items()
+        )
+        backgrounds = "".join(text_backgrounds)
+        matrix = "".join(text_group)
+
+        terminal_width = ceil(width * char_width + padding_width)
+        terminal_height = (y + 1) * line_height + padding_height
+        chrome = make_tag(
+            "rect",
+            fill=_theme.background_color.hex,
+            stroke="rgba(255,255,255,0.35)",
+            stroke_width="1",
+            x=margin_left,
+            y=margin_top,
+            width=terminal_width,
+            height=terminal_height,
+            rx=8,
+        )
+
+        title_color = _theme.foreground_color.hex
+        if title:
+            chrome += make_tag(
+                "text",
+                escape_text(title),
+                _class=f"{unique_id}-title",
+                fill=title_color,
+                text_anchor="middle",
+                x=terminal_width // 2,
+                y=margin_top + char_height + 6,
+            )
+        chrome += f"""
+            
+            
+            
+            
+            
+        """
+
+        svg = code_format.format(
+            unique_id=unique_id,
+            char_width=char_width,
+            char_height=char_height,
+            line_height=line_height,
+            terminal_width=char_width * width - 1,
+            terminal_height=(y + 1) * line_height - 1,
+            width=terminal_width + margin_width,
+            height=terminal_height + margin_height,
+            terminal_x=margin_left + padding_left,
+            terminal_y=margin_top + padding_top,
+            styles=styles,
+            chrome=chrome,
+            backgrounds=backgrounds,
+            matrix=matrix,
+            lines=lines,
+        )
+        return svg
+
+    def save_svg(
+        self,
+        path: str,
+        *,
+        title: str = "Rich",
+        theme: Optional[TerminalTheme] = None,
+        clear: bool = True,
+        code_format: str = CONSOLE_SVG_FORMAT,
+        font_aspect_ratio: float = 0.61,
+        unique_id: Optional[str] = None,
+    ) -> None:
+        """Generate an SVG file from the console contents (requires record=True in Console constructor).
+
+        Args:
+            path (str): The path to write the SVG to.
+            title (str, optional): The title of the tab in the output image
+            theme (TerminalTheme, optional): The ``TerminalTheme`` object to use to style the terminal
+            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``
+            code_format (str, optional): Format string used to generate the SVG. Rich will inject a number of variables
+                into the string in order to form the final SVG output. The default template used and the variables
+                injected by Rich can be found by inspecting the ``console.CONSOLE_SVG_FORMAT`` variable.
+            font_aspect_ratio (float, optional): The width to height ratio of the font used in the ``code_format``
+                string. Defaults to 0.61, which is the width to height ratio of Fira Code (the default font).
+                If you aren't specifying a different font inside ``code_format``, you probably don't need this.
+            unique_id (str, optional): unique id that is used as the prefix for various elements (CSS styles, node
+                ids). If not set, this defaults to a computed value based on the recorded content.
+        """
+        svg = self.export_svg(
+            title=title,
+            theme=theme,
+            clear=clear,
+            code_format=code_format,
+            font_aspect_ratio=font_aspect_ratio,
+            unique_id=unique_id,
+        )
+        with open(path, "w", encoding="utf-8") as write_file:
+            write_file.write(svg)
+
+
+def _svg_hash(svg_main_code: str) -> str:
+    """Returns a unique hash for the given SVG main code.
+
+    Args:
+        svg_main_code (str): The content we're going to inject in the SVG envelope.
+
+    Returns:
+        str: a hash of the given content
+    """
+    return str(zlib.adler32(svg_main_code.encode()))
+
+
+if __name__ == "__main__":  # pragma: no cover
+    console = Console(record=True)
+
+    console.log(
+        "JSONRPC [i]request[/i]",
+        5,
+        1.3,
+        True,
+        False,
+        None,
+        {
+            "jsonrpc": "2.0",
+            "method": "subtract",
+            "params": {"minuend": 42, "subtrahend": 23},
+            "id": 3,
+        },
+    )
+
+    console.log("Hello, World!", "{'a': 1}", repr(console))
+
+    console.print(
+        {
+            "name": None,
+            "empty": [],
+            "quiz": {
+                "sport": {
+                    "answered": True,
+                    "q1": {
+                        "question": "Which one is correct team name in NBA?",
+                        "options": [
+                            "New York Bulls",
+                            "Los Angeles Kings",
+                            "Golden State Warriors",
+                            "Huston Rocket",
+                        ],
+                        "answer": "Huston Rocket",
+                    },
+                },
+                "maths": {
+                    "answered": False,
+                    "q1": {
+                        "question": "5 + 7 = ?",
+                        "options": [10, 11, 12, 13],
+                        "answer": 12,
+                    },
+                    "q2": {
+                        "question": "12 - 8 = ?",
+                        "options": [1, 2, 3, 4],
+                        "answer": 4,
+                    },
+                },
+            },
+        }
+    )
diff --git a/lib/rich/constrain.py b/lib/rich/constrain.py
new file mode 100644
index 0000000..65fdf56
--- /dev/null
+++ b/lib/rich/constrain.py
@@ -0,0 +1,37 @@
+from typing import Optional, TYPE_CHECKING
+
+from .jupyter import JupyterMixin
+from .measure import Measurement
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderableType, RenderResult
+
+
+class Constrain(JupyterMixin):
+    """Constrain the width of a renderable to a given number of characters.
+
+    Args:
+        renderable (RenderableType): A renderable object.
+        width (int, optional): The maximum width (in characters) to render. Defaults to 80.
+    """
+
+    def __init__(self, renderable: "RenderableType", width: Optional[int] = 80) -> None:
+        self.renderable = renderable
+        self.width = width
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        if self.width is None:
+            yield self.renderable
+        else:
+            child_options = options.update_width(min(self.width, options.max_width))
+            yield from console.render(self.renderable, child_options)
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        if self.width is not None:
+            options = options.update_width(self.width)
+        measurement = Measurement.get(console, options, self.renderable)
+        return measurement
diff --git a/lib/rich/containers.py b/lib/rich/containers.py
new file mode 100644
index 0000000..901ff8b
--- /dev/null
+++ b/lib/rich/containers.py
@@ -0,0 +1,167 @@
+from itertools import zip_longest
+from typing import (
+    TYPE_CHECKING,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    TypeVar,
+    Union,
+    overload,
+)
+
+if TYPE_CHECKING:
+    from .console import (
+        Console,
+        ConsoleOptions,
+        JustifyMethod,
+        OverflowMethod,
+        RenderResult,
+        RenderableType,
+    )
+    from .text import Text
+
+from .cells import cell_len
+from .measure import Measurement
+
+T = TypeVar("T")
+
+
+class Renderables:
+    """A list subclass which renders its contents to the console."""
+
+    def __init__(
+        self, renderables: Optional[Iterable["RenderableType"]] = None
+    ) -> None:
+        self._renderables: List["RenderableType"] = (
+            list(renderables) if renderables is not None else []
+        )
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        """Console render method to insert line-breaks."""
+        yield from self._renderables
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        dimensions = [
+            Measurement.get(console, options, renderable)
+            for renderable in self._renderables
+        ]
+        if not dimensions:
+            return Measurement(1, 1)
+        _min = max(dimension.minimum for dimension in dimensions)
+        _max = max(dimension.maximum for dimension in dimensions)
+        return Measurement(_min, _max)
+
+    def append(self, renderable: "RenderableType") -> None:
+        self._renderables.append(renderable)
+
+    def __iter__(self) -> Iterable["RenderableType"]:
+        return iter(self._renderables)
+
+
+class Lines:
+    """A list subclass which can render to the console."""
+
+    def __init__(self, lines: Iterable["Text"] = ()) -> None:
+        self._lines: List["Text"] = list(lines)
+
+    def __repr__(self) -> str:
+        return f"Lines({self._lines!r})"
+
+    def __iter__(self) -> Iterator["Text"]:
+        return iter(self._lines)
+
+    @overload
+    def __getitem__(self, index: int) -> "Text":
+        ...
+
+    @overload
+    def __getitem__(self, index: slice) -> List["Text"]:
+        ...
+
+    def __getitem__(self, index: Union[slice, int]) -> Union["Text", List["Text"]]:
+        return self._lines[index]
+
+    def __setitem__(self, index: int, value: "Text") -> "Lines":
+        self._lines[index] = value
+        return self
+
+    def __len__(self) -> int:
+        return self._lines.__len__()
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        """Console render method to insert line-breaks."""
+        yield from self._lines
+
+    def append(self, line: "Text") -> None:
+        self._lines.append(line)
+
+    def extend(self, lines: Iterable["Text"]) -> None:
+        self._lines.extend(lines)
+
+    def pop(self, index: int = -1) -> "Text":
+        return self._lines.pop(index)
+
+    def justify(
+        self,
+        console: "Console",
+        width: int,
+        justify: "JustifyMethod" = "left",
+        overflow: "OverflowMethod" = "fold",
+    ) -> None:
+        """Justify and overflow text to a given width.
+
+        Args:
+            console (Console): Console instance.
+            width (int): Number of cells available per line.
+            justify (str, optional): Default justify method for text: "left", "center", "full" or "right". Defaults to "left".
+            overflow (str, optional): Default overflow for text: "crop", "fold", or "ellipsis". Defaults to "fold".
+
+        """
+        from .text import Text
+
+        if justify == "left":
+            for line in self._lines:
+                line.truncate(width, overflow=overflow, pad=True)
+        elif justify == "center":
+            for line in self._lines:
+                line.rstrip()
+                line.truncate(width, overflow=overflow)
+                line.pad_left((width - cell_len(line.plain)) // 2)
+                line.pad_right(width - cell_len(line.plain))
+        elif justify == "right":
+            for line in self._lines:
+                line.rstrip()
+                line.truncate(width, overflow=overflow)
+                line.pad_left(width - cell_len(line.plain))
+        elif justify == "full":
+            for line_index, line in enumerate(self._lines):
+                if line_index == len(self._lines) - 1:
+                    break
+                words = line.split(" ")
+                words_size = sum(cell_len(word.plain) for word in words)
+                num_spaces = len(words) - 1
+                spaces = [1 for _ in range(num_spaces)]
+                index = 0
+                if spaces:
+                    while words_size + num_spaces < width:
+                        spaces[len(spaces) - index - 1] += 1
+                        num_spaces += 1
+                        index = (index + 1) % len(spaces)
+                tokens: List[Text] = []
+                for index, (word, next_word) in enumerate(
+                    zip_longest(words, words[1:])
+                ):
+                    tokens.append(word)
+                    if index < len(spaces):
+                        style = word.get_style_at_offset(console, -1)
+                        next_style = next_word.get_style_at_offset(console, 0)
+                        space_style = style if style == next_style else line.style
+                        tokens.append(Text(" " * spaces[index], style=space_style))
+                self[line_index] = Text("").join(tokens)
diff --git a/lib/rich/control.py b/lib/rich/control.py
new file mode 100644
index 0000000..248b0f5
--- /dev/null
+++ b/lib/rich/control.py
@@ -0,0 +1,219 @@
+import time
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Union, Final
+
+from .segment import ControlCode, ControlType, Segment
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderResult
+
+STRIP_CONTROL_CODES: Final = [
+    7,  # Bell
+    8,  # Backspace
+    11,  # Vertical tab
+    12,  # Form feed
+    13,  # Carriage return
+]
+_CONTROL_STRIP_TRANSLATE: Final = {
+    _codepoint: None for _codepoint in STRIP_CONTROL_CODES
+}
+
+CONTROL_ESCAPE: Final = {
+    7: "\\a",
+    8: "\\b",
+    11: "\\v",
+    12: "\\f",
+    13: "\\r",
+}
+
+CONTROL_CODES_FORMAT: Dict[int, Callable[..., str]] = {
+    ControlType.BELL: lambda: "\x07",
+    ControlType.CARRIAGE_RETURN: lambda: "\r",
+    ControlType.HOME: lambda: "\x1b[H",
+    ControlType.CLEAR: lambda: "\x1b[2J",
+    ControlType.ENABLE_ALT_SCREEN: lambda: "\x1b[?1049h",
+    ControlType.DISABLE_ALT_SCREEN: lambda: "\x1b[?1049l",
+    ControlType.SHOW_CURSOR: lambda: "\x1b[?25h",
+    ControlType.HIDE_CURSOR: lambda: "\x1b[?25l",
+    ControlType.CURSOR_UP: lambda param: f"\x1b[{param}A",
+    ControlType.CURSOR_DOWN: lambda param: f"\x1b[{param}B",
+    ControlType.CURSOR_FORWARD: lambda param: f"\x1b[{param}C",
+    ControlType.CURSOR_BACKWARD: lambda param: f"\x1b[{param}D",
+    ControlType.CURSOR_MOVE_TO_COLUMN: lambda param: f"\x1b[{param+1}G",
+    ControlType.ERASE_IN_LINE: lambda param: f"\x1b[{param}K",
+    ControlType.CURSOR_MOVE_TO: lambda x, y: f"\x1b[{y+1};{x+1}H",
+    ControlType.SET_WINDOW_TITLE: lambda title: f"\x1b]0;{title}\x07",
+}
+
+
+class Control:
+    """A renderable that inserts a control code (non printable but may move cursor).
+
+    Args:
+        *codes (str): Positional arguments are either a :class:`~rich.segment.ControlType` enum or a
+            tuple of ControlType and an integer parameter
+    """
+
+    __slots__ = ["segment"]
+
+    def __init__(self, *codes: Union[ControlType, ControlCode]) -> None:
+        control_codes: List[ControlCode] = [
+            (code,) if isinstance(code, ControlType) else code for code in codes
+        ]
+        _format_map = CONTROL_CODES_FORMAT
+        rendered_codes = "".join(
+            _format_map[code](*parameters) for code, *parameters in control_codes
+        )
+        self.segment = Segment(rendered_codes, None, control_codes)
+
+    @classmethod
+    def bell(cls) -> "Control":
+        """Ring the 'bell'."""
+        return cls(ControlType.BELL)
+
+    @classmethod
+    def home(cls) -> "Control":
+        """Move cursor to 'home' position."""
+        return cls(ControlType.HOME)
+
+    @classmethod
+    def move(cls, x: int = 0, y: int = 0) -> "Control":
+        """Move cursor relative to current position.
+
+        Args:
+            x (int): X offset.
+            y (int): Y offset.
+
+        Returns:
+            ~Control: Control object.
+
+        """
+
+        def get_codes() -> Iterable[ControlCode]:
+            control = ControlType
+            if x:
+                yield (
+                    control.CURSOR_FORWARD if x > 0 else control.CURSOR_BACKWARD,
+                    abs(x),
+                )
+            if y:
+                yield (
+                    control.CURSOR_DOWN if y > 0 else control.CURSOR_UP,
+                    abs(y),
+                )
+
+        control = cls(*get_codes())
+        return control
+
+    @classmethod
+    def move_to_column(cls, x: int, y: int = 0) -> "Control":
+        """Move to the given column, optionally add offset to row.
+
+        Returns:
+            x (int): absolute x (column)
+            y (int): optional y offset (row)
+
+        Returns:
+            ~Control: Control object.
+        """
+
+        return (
+            cls(
+                (ControlType.CURSOR_MOVE_TO_COLUMN, x),
+                (
+                    ControlType.CURSOR_DOWN if y > 0 else ControlType.CURSOR_UP,
+                    abs(y),
+                ),
+            )
+            if y
+            else cls((ControlType.CURSOR_MOVE_TO_COLUMN, x))
+        )
+
+    @classmethod
+    def move_to(cls, x: int, y: int) -> "Control":
+        """Move cursor to absolute position.
+
+        Args:
+            x (int): x offset (column)
+            y (int): y offset (row)
+
+        Returns:
+            ~Control: Control object.
+        """
+        return cls((ControlType.CURSOR_MOVE_TO, x, y))
+
+    @classmethod
+    def clear(cls) -> "Control":
+        """Clear the screen."""
+        return cls(ControlType.CLEAR)
+
+    @classmethod
+    def show_cursor(cls, show: bool) -> "Control":
+        """Show or hide the cursor."""
+        return cls(ControlType.SHOW_CURSOR if show else ControlType.HIDE_CURSOR)
+
+    @classmethod
+    def alt_screen(cls, enable: bool) -> "Control":
+        """Enable or disable alt screen."""
+        if enable:
+            return cls(ControlType.ENABLE_ALT_SCREEN, ControlType.HOME)
+        else:
+            return cls(ControlType.DISABLE_ALT_SCREEN)
+
+    @classmethod
+    def title(cls, title: str) -> "Control":
+        """Set the terminal window title
+
+        Args:
+            title (str): The new terminal window title
+        """
+        return cls((ControlType.SET_WINDOW_TITLE, title))
+
+    def __str__(self) -> str:
+        return self.segment.text
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        if self.segment.text:
+            yield self.segment
+
+
+def strip_control_codes(
+    text: str, _translate_table: Dict[int, None] = _CONTROL_STRIP_TRANSLATE
+) -> str:
+    """Remove control codes from text.
+
+    Args:
+        text (str): A string possibly contain control codes.
+
+    Returns:
+        str: String with control codes removed.
+    """
+    return text.translate(_translate_table)
+
+
+def escape_control_codes(
+    text: str,
+    _translate_table: Dict[int, str] = CONTROL_ESCAPE,
+) -> str:
+    """Replace control codes with their "escaped" equivalent in the given text.
+    (e.g. "\b" becomes "\\b")
+
+    Args:
+        text (str): A string possibly containing control codes.
+
+    Returns:
+        str: String with control codes replaced with their escaped version.
+    """
+    return text.translate(_translate_table)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich.console import Console
+
+    console = Console()
+    console.print("Look at the title of your terminal window ^")
+    # console.print(Control((ControlType.SET_WINDOW_TITLE, "Hello, world!")))
+    for i in range(10):
+        console.set_window_title("🚀 Loading" + "." * i)
+        time.sleep(0.5)
diff --git a/lib/rich/default_styles.py b/lib/rich/default_styles.py
new file mode 100644
index 0000000..c18b609
--- /dev/null
+++ b/lib/rich/default_styles.py
@@ -0,0 +1,195 @@
+from typing import Dict
+
+from .style import Style
+
+DEFAULT_STYLES: Dict[str, Style] = {
+    "none": Style.null(),
+    "reset": Style(
+        color="default",
+        bgcolor="default",
+        dim=False,
+        bold=False,
+        italic=False,
+        underline=False,
+        blink=False,
+        blink2=False,
+        reverse=False,
+        conceal=False,
+        strike=False,
+    ),
+    "dim": Style(dim=True),
+    "bright": Style(dim=False),
+    "bold": Style(bold=True),
+    "strong": Style(bold=True),
+    "code": Style(reverse=True, bold=True),
+    "italic": Style(italic=True),
+    "emphasize": Style(italic=True),
+    "underline": Style(underline=True),
+    "blink": Style(blink=True),
+    "blink2": Style(blink2=True),
+    "reverse": Style(reverse=True),
+    "strike": Style(strike=True),
+    "black": Style(color="black"),
+    "red": Style(color="red"),
+    "green": Style(color="green"),
+    "yellow": Style(color="yellow"),
+    "magenta": Style(color="magenta"),
+    "cyan": Style(color="cyan"),
+    "white": Style(color="white"),
+    "inspect.attr": Style(color="yellow", italic=True),
+    "inspect.attr.dunder": Style(color="yellow", italic=True, dim=True),
+    "inspect.callable": Style(bold=True, color="red"),
+    "inspect.async_def": Style(italic=True, color="bright_cyan"),
+    "inspect.def": Style(italic=True, color="bright_cyan"),
+    "inspect.class": Style(italic=True, color="bright_cyan"),
+    "inspect.error": Style(bold=True, color="red"),
+    "inspect.equals": Style(),
+    "inspect.help": Style(color="cyan"),
+    "inspect.doc": Style(dim=True),
+    "inspect.value.border": Style(color="green"),
+    "live.ellipsis": Style(bold=True, color="red"),
+    "layout.tree.row": Style(dim=False, color="red"),
+    "layout.tree.column": Style(dim=False, color="blue"),
+    "logging.keyword": Style(bold=True, color="yellow"),
+    "logging.level.notset": Style(dim=True),
+    "logging.level.debug": Style(color="green"),
+    "logging.level.info": Style(color="blue"),
+    "logging.level.warning": Style(color="yellow"),
+    "logging.level.error": Style(color="red", bold=True),
+    "logging.level.critical": Style(color="red", bold=True, reverse=True),
+    "log.level": Style.null(),
+    "log.time": Style(color="cyan", dim=True),
+    "log.message": Style.null(),
+    "log.path": Style(dim=True),
+    "repr.ellipsis": Style(color="yellow"),
+    "repr.indent": Style(color="green", dim=True),
+    "repr.error": Style(color="red", bold=True),
+    "repr.str": Style(color="green", italic=False, bold=False),
+    "repr.brace": Style(bold=True),
+    "repr.comma": Style(bold=True),
+    "repr.ipv4": Style(bold=True, color="bright_green"),
+    "repr.ipv6": Style(bold=True, color="bright_green"),
+    "repr.eui48": Style(bold=True, color="bright_green"),
+    "repr.eui64": Style(bold=True, color="bright_green"),
+    "repr.tag_start": Style(bold=True),
+    "repr.tag_name": Style(color="bright_magenta", bold=True),
+    "repr.tag_contents": Style(color="default"),
+    "repr.tag_end": Style(bold=True),
+    "repr.attrib_name": Style(color="yellow", italic=False),
+    "repr.attrib_equal": Style(bold=True),
+    "repr.attrib_value": Style(color="magenta", italic=False),
+    "repr.number": Style(color="cyan", bold=True, italic=False),
+    "repr.number_complex": Style(color="cyan", bold=True, italic=False),  # same
+    "repr.bool_true": Style(color="bright_green", italic=True),
+    "repr.bool_false": Style(color="bright_red", italic=True),
+    "repr.none": Style(color="magenta", italic=True),
+    "repr.url": Style(underline=True, color="bright_blue", italic=False, bold=False),
+    "repr.uuid": Style(color="bright_yellow", bold=False),
+    "repr.call": Style(color="magenta", bold=True),
+    "repr.path": Style(color="magenta"),
+    "repr.filename": Style(color="bright_magenta"),
+    "rule.line": Style(color="bright_green"),
+    "rule.text": Style.null(),
+    "json.brace": Style(bold=True),
+    "json.bool_true": Style(color="bright_green", italic=True),
+    "json.bool_false": Style(color="bright_red", italic=True),
+    "json.null": Style(color="magenta", italic=True),
+    "json.number": Style(color="cyan", bold=True, italic=False),
+    "json.str": Style(color="green", italic=False, bold=False),
+    "json.key": Style(color="blue", bold=True),
+    "prompt": Style.null(),
+    "prompt.choices": Style(color="magenta", bold=True),
+    "prompt.default": Style(color="cyan", bold=True),
+    "prompt.invalid": Style(color="red"),
+    "prompt.invalid.choice": Style(color="red"),
+    "pretty": Style.null(),
+    "scope.border": Style(color="blue"),
+    "scope.key": Style(color="yellow", italic=True),
+    "scope.key.special": Style(color="yellow", italic=True, dim=True),
+    "scope.equals": Style(color="red"),
+    "table.header": Style(bold=True),
+    "table.footer": Style(bold=True),
+    "table.cell": Style.null(),
+    "table.title": Style(italic=True),
+    "table.caption": Style(italic=True, dim=True),
+    "traceback.error": Style(color="red", italic=True),
+    "traceback.border.syntax_error": Style(color="bright_red"),
+    "traceback.border": Style(color="red"),
+    "traceback.text": Style.null(),
+    "traceback.title": Style(color="red", bold=True),
+    "traceback.exc_type": Style(color="bright_red", bold=True),
+    "traceback.exc_value": Style.null(),
+    "traceback.offset": Style(color="bright_red", bold=True),
+    "traceback.error_range": Style(underline=True, bold=True),
+    "traceback.note": Style(color="green", bold=True),
+    "traceback.group.border": Style(color="magenta"),
+    "bar.back": Style(color="grey23"),
+    "bar.complete": Style(color="rgb(249,38,114)"),
+    "bar.finished": Style(color="rgb(114,156,31)"),
+    "bar.pulse": Style(color="rgb(249,38,114)"),
+    "progress.description": Style.null(),
+    "progress.filesize": Style(color="green"),
+    "progress.filesize.total": Style(color="green"),
+    "progress.download": Style(color="green"),
+    "progress.elapsed": Style(color="yellow"),
+    "progress.percentage": Style(color="magenta"),
+    "progress.remaining": Style(color="cyan"),
+    "progress.data.speed": Style(color="red"),
+    "progress.spinner": Style(color="green"),
+    "status.spinner": Style(color="green"),
+    "tree": Style(),
+    "tree.line": Style(),
+    "markdown.paragraph": Style(),
+    "markdown.text": Style(),
+    "markdown.em": Style(italic=True),
+    "markdown.emph": Style(italic=True),  # For commonmark backwards compatibility
+    "markdown.strong": Style(bold=True),
+    "markdown.code": Style(bold=True, color="cyan", bgcolor="black"),
+    "markdown.code_block": Style(color="cyan", bgcolor="black"),
+    "markdown.block_quote": Style(color="magenta"),
+    "markdown.list": Style(color="cyan"),
+    "markdown.item": Style(),
+    "markdown.item.bullet": Style(bold=True),
+    "markdown.item.number": Style(color="cyan"),
+    "markdown.hr": Style(dim=True),
+    "markdown.h1.border": Style(),
+    "markdown.h1": Style(bold=True, underline=True),
+    "markdown.h2": Style(color="magenta", underline=True),
+    "markdown.h3": Style(color="magenta", bold=True),
+    "markdown.h4": Style(color="magenta", italic=True),
+    "markdown.h5": Style(italic=True),
+    "markdown.h6": Style(dim=True),
+    "markdown.h7": Style(italic=True, dim=True),
+    "markdown.link": Style(color="bright_blue"),
+    "markdown.link_url": Style(color="blue", underline=True),
+    "markdown.s": Style(strike=True),
+    "markdown.table.border": Style(color="cyan"),
+    "markdown.table.header": Style(color="cyan", bold=False),
+    "iso8601.date": Style(color="blue"),
+    "iso8601.time": Style(color="magenta"),
+    "iso8601.timezone": Style(color="yellow"),
+}
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import argparse
+    import io
+
+    from rich.console import Console
+    from rich.table import Table
+    from rich.text import Text
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--html", action="store_true", help="Export as HTML table")
+    args = parser.parse_args()
+    html: bool = args.html
+    console = Console(record=True, width=70, file=io.StringIO()) if html else Console()
+
+    table = Table("Name", "Styling")
+
+    for style_name, style in DEFAULT_STYLES.items():
+        table.add_row(Text(style_name, style=style), str(style))
+
+    console.print(table)
+    if html:
+        print(console.export_html(inline_styles=True))
diff --git a/lib/rich/diagnose.py b/lib/rich/diagnose.py
new file mode 100644
index 0000000..9d5ff3e
--- /dev/null
+++ b/lib/rich/diagnose.py
@@ -0,0 +1,39 @@
+import os
+import platform
+
+from rich import inspect
+from rich.console import Console, get_windows_console_features
+from rich.panel import Panel
+from rich.pretty import Pretty
+
+
+def report() -> None:  # pragma: no cover
+    """Print a report to the terminal with debugging information"""
+    console = Console()
+    inspect(console)
+    features = get_windows_console_features()
+    inspect(features)
+
+    env_names = (
+        "CLICOLOR",
+        "COLORTERM",
+        "COLUMNS",
+        "JPY_PARENT_PID",
+        "JUPYTER_COLUMNS",
+        "JUPYTER_LINES",
+        "LINES",
+        "NO_COLOR",
+        "TERM_PROGRAM",
+        "TERM",
+        "TTY_COMPATIBLE",
+        "TTY_INTERACTIVE",
+        "VSCODE_VERBOSE_LOGGING",
+    )
+    env = {name: os.getenv(name) for name in env_names}
+    console.print(Panel.fit((Pretty(env)), title="[b]Environment Variables"))
+
+    console.print(f'platform="{platform.system()}"')
+
+
+if __name__ == "__main__":  # pragma: no cover
+    report()
diff --git a/lib/rich/emoji.py b/lib/rich/emoji.py
new file mode 100644
index 0000000..9433e6f
--- /dev/null
+++ b/lib/rich/emoji.py
@@ -0,0 +1,91 @@
+import sys
+from typing import TYPE_CHECKING, Optional, Union, Literal
+
+from .jupyter import JupyterMixin
+from .segment import Segment
+from .style import Style
+from ._emoji_codes import EMOJI
+from ._emoji_replace import _emoji_replace
+
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderResult
+
+
+EmojiVariant = Literal["emoji", "text"]
+
+
+class NoEmoji(Exception):
+    """No emoji by that name."""
+
+
+class Emoji(JupyterMixin):
+    __slots__ = ["name", "style", "_char", "variant"]
+
+    VARIANTS = {"text": "\uFE0E", "emoji": "\uFE0F"}
+
+    def __init__(
+        self,
+        name: str,
+        style: Union[str, Style] = "none",
+        variant: Optional[EmojiVariant] = None,
+    ) -> None:
+        """A single emoji character.
+
+        Args:
+            name (str): Name of emoji.
+            style (Union[str, Style], optional): Optional style. Defaults to None.
+
+        Raises:
+            NoEmoji: If the emoji doesn't exist.
+        """
+        self.name = name
+        self.style = style
+        self.variant = variant
+        try:
+            self._char = EMOJI[name]
+        except KeyError:
+            raise NoEmoji(f"No emoji called {name!r}")
+        if variant is not None:
+            self._char += self.VARIANTS.get(variant, "")
+
+    @classmethod
+    def replace(cls, text: str) -> str:
+        """Replace emoji markup with corresponding unicode characters.
+
+        Args:
+            text (str): A string with emojis codes, e.g. "Hello :smiley:!"
+
+        Returns:
+            str: A string with emoji codes replaces with actual emoji.
+        """
+        return _emoji_replace(text)
+
+    def __repr__(self) -> str:
+        return f""
+
+    def __str__(self) -> str:
+        return self._char
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        yield Segment(self._char, console.get_style(self.style))
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import sys
+
+    from rich.columns import Columns
+    from rich.console import Console
+
+    console = Console(record=True)
+
+    columns = Columns(
+        (f":{name}: {name}" for name in sorted(EMOJI.keys()) if "\u200D" not in name),
+        column_first=True,
+    )
+
+    console.print(columns)
+    if len(sys.argv) > 1:
+        console.save_html(sys.argv[1])
diff --git a/lib/rich/errors.py b/lib/rich/errors.py
new file mode 100644
index 0000000..0bcbe53
--- /dev/null
+++ b/lib/rich/errors.py
@@ -0,0 +1,34 @@
+class ConsoleError(Exception):
+    """An error in console operation."""
+
+
+class StyleError(Exception):
+    """An error in styles."""
+
+
+class StyleSyntaxError(ConsoleError):
+    """Style was badly formatted."""
+
+
+class MissingStyle(StyleError):
+    """No such style."""
+
+
+class StyleStackError(ConsoleError):
+    """Style stack is invalid."""
+
+
+class NotRenderableError(ConsoleError):
+    """Object is not renderable."""
+
+
+class MarkupError(ConsoleError):
+    """Markup was badly formatted."""
+
+
+class LiveError(ConsoleError):
+    """Error related to Live display."""
+
+
+class NoAltScreen(ConsoleError):
+    """Alt screen mode was required."""
diff --git a/lib/rich/file_proxy.py b/lib/rich/file_proxy.py
new file mode 100644
index 0000000..4b0b0da
--- /dev/null
+++ b/lib/rich/file_proxy.py
@@ -0,0 +1,57 @@
+import io
+from typing import IO, TYPE_CHECKING, Any, List
+
+from .ansi import AnsiDecoder
+from .text import Text
+
+if TYPE_CHECKING:
+    from .console import Console
+
+
+class FileProxy(io.TextIOBase):
+    """Wraps a file (e.g. sys.stdout) and redirects writes to a console."""
+
+    def __init__(self, console: "Console", file: IO[str]) -> None:
+        self.__console = console
+        self.__file = file
+        self.__buffer: List[str] = []
+        self.__ansi_decoder = AnsiDecoder()
+
+    @property
+    def rich_proxied_file(self) -> IO[str]:
+        """Get proxied file."""
+        return self.__file
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self.__file, name)
+
+    def write(self, text: str) -> int:
+        if not isinstance(text, str):
+            raise TypeError(f"write() argument must be str, not {type(text).__name__}")
+        buffer = self.__buffer
+        lines: List[str] = []
+        while text:
+            line, new_line, text = text.partition("\n")
+            if new_line:
+                lines.append("".join(buffer) + line)
+                buffer.clear()
+            else:
+                buffer.append(line)
+                break
+        if lines:
+            console = self.__console
+            with console:
+                output = Text("\n").join(
+                    self.__ansi_decoder.decode_line(line) for line in lines
+                )
+                console.print(output)
+        return len(text)
+
+    def flush(self) -> None:
+        output = "".join(self.__buffer)
+        if output:
+            self.__console.print(output)
+        del self.__buffer[:]
+
+    def fileno(self) -> int:
+        return self.__file.fileno()
diff --git a/lib/rich/filesize.py b/lib/rich/filesize.py
new file mode 100644
index 0000000..83bc911
--- /dev/null
+++ b/lib/rich/filesize.py
@@ -0,0 +1,88 @@
+"""Functions for reporting filesizes. Borrowed from https://github.com/PyFilesystem/pyfilesystem2
+
+The functions declared in this module should cover the different
+use cases needed to generate a string representation of a file size
+using several different units. Since there are many standards regarding
+file size units, three different functions have been implemented.
+
+See Also:
+    * `Wikipedia: Binary prefix `_
+
+"""
+
+__all__ = ["decimal"]
+
+from typing import Iterable, List, Optional, Tuple
+
+
+def _to_str(
+    size: int,
+    suffixes: Iterable[str],
+    base: int,
+    *,
+    precision: Optional[int] = 1,
+    separator: Optional[str] = " ",
+) -> str:
+    if size == 1:
+        return "1 byte"
+    elif size < base:
+        return f"{size:,} bytes"
+
+    for i, suffix in enumerate(suffixes, 2):  # noqa: B007
+        unit = base**i
+        if size < unit:
+            break
+    return "{:,.{precision}f}{separator}{}".format(
+        (base * size / unit),
+        suffix,
+        precision=precision,
+        separator=separator,
+    )
+
+
+def pick_unit_and_suffix(size: int, suffixes: List[str], base: int) -> Tuple[int, str]:
+    """Pick a suffix and base for the given size."""
+    for i, suffix in enumerate(suffixes):
+        unit = base**i
+        if size < unit * base:
+            break
+    return unit, suffix
+
+
+def decimal(
+    size: int,
+    *,
+    precision: Optional[int] = 1,
+    separator: Optional[str] = " ",
+) -> str:
+    """Convert a filesize in to a string (powers of 1000, SI prefixes).
+
+    In this convention, ``1000 B = 1 kB``.
+
+    This is typically the format used to advertise the storage
+    capacity of USB flash drives and the like (*256 MB* meaning
+    actually a storage capacity of more than *256 000 000 B*),
+    or used by **Mac OS X** since v10.6 to report file sizes.
+
+    Arguments:
+        int (size): A file size.
+        int (precision): The number of decimal places to include (default = 1).
+        str (separator): The string to separate the value from the units (default = " ").
+
+    Returns:
+        `str`: A string containing a abbreviated file size and units.
+
+    Example:
+        >>> filesize.decimal(30000)
+        '30.0 kB'
+        >>> filesize.decimal(30000, precision=2, separator="")
+        '30.00kB'
+
+    """
+    return _to_str(
+        size,
+        ("kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"),
+        1000,
+        precision=precision,
+        separator=separator,
+    )
diff --git a/lib/rich/highlighter.py b/lib/rich/highlighter.py
new file mode 100644
index 0000000..df28048
--- /dev/null
+++ b/lib/rich/highlighter.py
@@ -0,0 +1,232 @@
+import re
+from abc import ABC, abstractmethod
+from typing import ClassVar, Sequence, Union
+
+from .text import Span, Text
+
+
+def _combine_regex(*regexes: str) -> str:
+    """Combine a number of regexes in to a single regex.
+
+    Returns:
+        str: New regex with all regexes ORed together.
+    """
+    return "|".join(regexes)
+
+
+class Highlighter(ABC):
+    """Abstract base class for highlighters."""
+
+    def __call__(self, text: Union[str, Text]) -> Text:
+        """Highlight a str or Text instance.
+
+        Args:
+            text (Union[str, ~Text]): Text to highlight.
+
+        Raises:
+            TypeError: If not called with text or str.
+
+        Returns:
+            Text: A test instance with highlighting applied.
+        """
+        if isinstance(text, str):
+            highlight_text = Text(text)
+        elif isinstance(text, Text):
+            highlight_text = text.copy()
+        else:
+            raise TypeError(f"str or Text instance required, not {text!r}")
+        self.highlight(highlight_text)
+        return highlight_text
+
+    @abstractmethod
+    def highlight(self, text: Text) -> None:
+        """Apply highlighting in place to text.
+
+        Args:
+            text (~Text): A text object highlight.
+        """
+
+
+class NullHighlighter(Highlighter):
+    """A highlighter object that doesn't highlight.
+
+    May be used to disable highlighting entirely.
+
+    """
+
+    def highlight(self, text: Text) -> None:
+        """Nothing to do"""
+
+
+class RegexHighlighter(Highlighter):
+    """Applies highlighting from a list of regular expressions."""
+
+    highlights: ClassVar[Sequence[str]] = []
+    base_style: ClassVar[str] = ""
+
+    def highlight(self, text: Text) -> None:
+        """Highlight :class:`rich.text.Text` using regular expressions.
+
+        Args:
+            text (~Text): Text to highlighted.
+
+        """
+
+        highlight_regex = text.highlight_regex
+        for re_highlight in self.highlights:
+            highlight_regex(re_highlight, style_prefix=self.base_style)
+
+
+class ReprHighlighter(RegexHighlighter):
+    """Highlights the text typically produced from ``__repr__`` methods."""
+
+    base_style = "repr."
+    highlights: ClassVar[Sequence[str]] = [
+        r"(?P<)(?P[-\w.:|]*)(?P[\w\W]*)(?P>)",
+        r'(?P[\w_]{1,50})=(?P"?[\w_]+"?)?',
+        r"(?P[][{}()])",
+        _combine_regex(
+            r"(?P[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})",
+            r"(?P([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})",
+            r"(?P(?:[0-9A-Fa-f]{1,2}-){7}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{1,2}:){7}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{4}\.){3}[0-9A-Fa-f]{4})",
+            r"(?P(?:[0-9A-Fa-f]{1,2}-){5}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{1,2}:){5}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})",
+            r"(?P[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})",
+            r"(?P[\w.]*?)\(",
+            r"\b(?PTrue)\b|\b(?PFalse)\b|\b(?PNone)\b",
+            r"(?P\.\.\.)",
+            r"(?P(?(?\B(/[-\w._+]+)*\/)(?P[-\w._+]*)?",
+            r"(?b?'''.*?(?(file|https|http|ws|wss)://[-0-9a-zA-Z$_+!`(),.?/;:&=%#~@]*)",
+        ),
+    ]
+
+
+class JSONHighlighter(RegexHighlighter):
+    """Highlights JSON"""
+
+    # Captures the start and end of JSON strings, handling escaped quotes
+    JSON_STR = r"(?b?\".*?(?[\{\[\(\)\]\}])",
+            r"\b(?Ptrue)\b|\b(?Pfalse)\b|\b(?Pnull)\b",
+            r"(?P(? None:
+        super().highlight(text)
+
+        # Additional work to handle highlighting JSON keys
+        plain = text.plain
+        append = text.spans.append
+        whitespace = self.JSON_WHITESPACE
+        for match in re.finditer(self.JSON_STR, plain):
+            start, end = match.span()
+            cursor = end
+            while cursor < len(plain):
+                char = plain[cursor]
+                cursor += 1
+                if char == ":":
+                    append(Span(start, end, "json.key"))
+                elif char in whitespace:
+                    continue
+                break
+
+
+class ISO8601Highlighter(RegexHighlighter):
+    """Highlights the ISO8601 date time strings.
+    Regex reference: https://www.oreilly.com/library/view/regular-expressions-cookbook/9781449327453/ch04s07.html
+    """
+
+    base_style: ClassVar[str] = "iso8601."
+    highlights: ClassVar[Sequence[str]] = [
+        #
+        # Dates
+        #
+        # Calendar month (e.g. 2008-08). The hyphen is required
+        r"^(?P[0-9]{4})-(?P1[0-2]|0[1-9])$",
+        # Calendar date w/o hyphens (e.g. 20080830)
+        r"^(?P(?P[0-9]{4})(?P1[0-2]|0[1-9])(?P3[01]|0[1-9]|[12][0-9]))$",
+        # Ordinal date (e.g. 2008-243). The hyphen is optional
+        r"^(?P(?P[0-9]{4})-?(?P36[0-6]|3[0-5][0-9]|[12][0-9]{2}|0[1-9][0-9]|00[1-9]))$",
+        #
+        # Weeks
+        #
+        # Week of the year (e.g., 2008-W35). The hyphen is optional
+        r"^(?P(?P[0-9]{4})-?W(?P5[0-3]|[1-4][0-9]|0[1-9]))$",
+        # Week date (e.g., 2008-W35-6). The hyphens are optional
+        r"^(?P(?P[0-9]{4})-?W(?P5[0-3]|[1-4][0-9]|0[1-9])-?(?P[1-7]))$",
+        #
+        # Times
+        #
+        # Hours and minutes (e.g., 17:21). The colon is optional
+        r"^(?P{text}'
+        append_fragment(text)
+
+    code = "".join(fragments)
+    html = JUPYTER_HTML_FORMAT.format(code=code)
+
+    return html
+
+
+def display(segments: Iterable[Segment], text: str) -> None:
+    """Render segments to Jupyter."""
+    html = _render_segments(segments)
+    jupyter_renderable = JupyterRenderable(html, text)
+    try:
+        from IPython.display import display as ipython_display
+
+        ipython_display(jupyter_renderable)
+    except ModuleNotFoundError:
+        # Handle the case where the Console has force_jupyter=True,
+        # but IPython is not installed.
+        pass
+
+
+def print(*args: Any, **kwargs: Any) -> None:
+    """Proxy for Console print."""
+    console = get_console()
+    return console.print(*args, **kwargs)
diff --git a/lib/rich/layout.py b/lib/rich/layout.py
new file mode 100644
index 0000000..7fa2852
--- /dev/null
+++ b/lib/rich/layout.py
@@ -0,0 +1,442 @@
+from abc import ABC, abstractmethod
+from itertools import islice
+from operator import itemgetter
+from threading import RLock
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
+
+from ._ratio import ratio_resolve
+from .align import Align
+from .console import Console, ConsoleOptions, RenderableType, RenderResult
+from .highlighter import ReprHighlighter
+from .panel import Panel
+from .pretty import Pretty
+from .region import Region
+from .repr import Result, rich_repr
+from .segment import Segment
+from .style import StyleType
+
+if TYPE_CHECKING:
+    from rich.tree import Tree
+
+
+class LayoutRender(NamedTuple):
+    """An individual layout render."""
+
+    region: Region
+    render: List[List[Segment]]
+
+
+RegionMap = Dict["Layout", Region]
+RenderMap = Dict["Layout", LayoutRender]
+
+
+class LayoutError(Exception):
+    """Layout related error."""
+
+
+class NoSplitter(LayoutError):
+    """Requested splitter does not exist."""
+
+
+class _Placeholder:
+    """An internal renderable used as a Layout placeholder."""
+
+    highlighter = ReprHighlighter()
+
+    def __init__(self, layout: "Layout", style: StyleType = "") -> None:
+        self.layout = layout
+        self.style = style
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        width = options.max_width
+        height = options.height or options.size.height
+        layout = self.layout
+        title = (
+            f"{layout.name!r} ({width} x {height})"
+            if layout.name
+            else f"({width} x {height})"
+        )
+        yield Panel(
+            Align.center(Pretty(layout), vertical="middle"),
+            style=self.style,
+            title=self.highlighter(title),
+            border_style="blue",
+            height=height,
+        )
+
+
+class Splitter(ABC):
+    """Base class for a splitter."""
+
+    name: str = ""
+
+    @abstractmethod
+    def get_tree_icon(self) -> str:
+        """Get the icon (emoji) used in layout.tree"""
+
+    @abstractmethod
+    def divide(
+        self, children: Sequence["Layout"], region: Region
+    ) -> Iterable[Tuple["Layout", Region]]:
+        """Divide a region amongst several child layouts.
+
+        Args:
+            children (Sequence(Layout)): A number of child layouts.
+            region (Region): A rectangular region to divide.
+        """
+
+
+class RowSplitter(Splitter):
+    """Split a layout region in to rows."""
+
+    name = "row"
+
+    def get_tree_icon(self) -> str:
+        return "[layout.tree.row]⬌"
+
+    def divide(
+        self, children: Sequence["Layout"], region: Region
+    ) -> Iterable[Tuple["Layout", Region]]:
+        x, y, width, height = region
+        render_widths = ratio_resolve(width, children)
+        offset = 0
+        _Region = Region
+        for child, child_width in zip(children, render_widths):
+            yield child, _Region(x + offset, y, child_width, height)
+            offset += child_width
+
+
+class ColumnSplitter(Splitter):
+    """Split a layout region in to columns."""
+
+    name = "column"
+
+    def get_tree_icon(self) -> str:
+        return "[layout.tree.column]⬍"
+
+    def divide(
+        self, children: Sequence["Layout"], region: Region
+    ) -> Iterable[Tuple["Layout", Region]]:
+        x, y, width, height = region
+        render_heights = ratio_resolve(height, children)
+        offset = 0
+        _Region = Region
+        for child, child_height in zip(children, render_heights):
+            yield child, _Region(x, y + offset, width, child_height)
+            offset += child_height
+
+
+@rich_repr
+class Layout:
+    """A renderable to divide a fixed height in to rows or columns.
+
+    Args:
+        renderable (RenderableType, optional): Renderable content, or None for placeholder. Defaults to None.
+        name (str, optional): Optional identifier for Layout. Defaults to None.
+        size (int, optional): Optional fixed size of layout. Defaults to None.
+        minimum_size (int, optional): Minimum size of layout. Defaults to 1.
+        ratio (int, optional): Optional ratio for flexible layout. Defaults to 1.
+        visible (bool, optional): Visibility of layout. Defaults to True.
+    """
+
+    splitters = {"row": RowSplitter, "column": ColumnSplitter}
+
+    def __init__(
+        self,
+        renderable: Optional[RenderableType] = None,
+        *,
+        name: Optional[str] = None,
+        size: Optional[int] = None,
+        minimum_size: int = 1,
+        ratio: int = 1,
+        visible: bool = True,
+    ) -> None:
+        self._renderable = renderable or _Placeholder(self)
+        self.size = size
+        self.minimum_size = minimum_size
+        self.ratio = ratio
+        self.name = name
+        self.visible = visible
+        self.splitter: Splitter = self.splitters["column"]()
+        self._children: List[Layout] = []
+        self._render_map: RenderMap = {}
+        self._lock = RLock()
+
+    def __rich_repr__(self) -> Result:
+        yield "name", self.name, None
+        yield "size", self.size, None
+        yield "minimum_size", self.minimum_size, 1
+        yield "ratio", self.ratio, 1
+
+    @property
+    def renderable(self) -> RenderableType:
+        """Layout renderable."""
+        return self if self._children else self._renderable
+
+    @property
+    def children(self) -> List["Layout"]:
+        """Gets (visible) layout children."""
+        return [child for child in self._children if child.visible]
+
+    @property
+    def map(self) -> RenderMap:
+        """Get a map of the last render."""
+        return self._render_map
+
+    def get(self, name: str) -> Optional["Layout"]:
+        """Get a named layout, or None if it doesn't exist.
+
+        Args:
+            name (str): Name of layout.
+
+        Returns:
+            Optional[Layout]: Layout instance or None if no layout was found.
+        """
+        if self.name == name:
+            return self
+        else:
+            for child in self._children:
+                named_layout = child.get(name)
+                if named_layout is not None:
+                    return named_layout
+        return None
+
+    def __getitem__(self, name: str) -> "Layout":
+        layout = self.get(name)
+        if layout is None:
+            raise KeyError(f"No layout with name {name!r}")
+        return layout
+
+    @property
+    def tree(self) -> "Tree":
+        """Get a tree renderable to show layout structure."""
+        from rich.styled import Styled
+        from rich.table import Table
+        from rich.tree import Tree
+
+        def summary(layout: "Layout") -> Table:
+            icon = layout.splitter.get_tree_icon()
+
+            table = Table.grid(padding=(0, 1, 0, 0))
+
+            text: RenderableType = (
+                Pretty(layout) if layout.visible else Styled(Pretty(layout), "dim")
+            )
+            table.add_row(icon, text)
+            _summary = table
+            return _summary
+
+        layout = self
+        tree = Tree(
+            summary(layout),
+            guide_style=f"layout.tree.{layout.splitter.name}",
+            highlight=True,
+        )
+
+        def recurse(tree: "Tree", layout: "Layout") -> None:
+            for child in layout._children:
+                recurse(
+                    tree.add(
+                        summary(child),
+                        guide_style=f"layout.tree.{child.splitter.name}",
+                    ),
+                    child,
+                )
+
+        recurse(tree, self)
+        return tree
+
+    def split(
+        self,
+        *layouts: Union["Layout", RenderableType],
+        splitter: Union[Splitter, str] = "column",
+    ) -> None:
+        """Split the layout in to multiple sub-layouts.
+
+        Args:
+            *layouts (Layout): Positional arguments should be (sub) Layout instances.
+            splitter (Union[Splitter, str]): Splitter instance or name of splitter.
+        """
+        _layouts = [
+            layout if isinstance(layout, Layout) else Layout(layout)
+            for layout in layouts
+        ]
+        try:
+            self.splitter = (
+                splitter
+                if isinstance(splitter, Splitter)
+                else self.splitters[splitter]()
+            )
+        except KeyError:
+            raise NoSplitter(f"No splitter called {splitter!r}")
+        self._children[:] = _layouts
+
+    def add_split(self, *layouts: Union["Layout", RenderableType]) -> None:
+        """Add a new layout(s) to existing split.
+
+        Args:
+            *layouts (Union[Layout, RenderableType]): Positional arguments should be renderables or (sub) Layout instances.
+
+        """
+        _layouts = (
+            layout if isinstance(layout, Layout) else Layout(layout)
+            for layout in layouts
+        )
+        self._children.extend(_layouts)
+
+    def split_row(self, *layouts: Union["Layout", RenderableType]) -> None:
+        """Split the layout in to a row (layouts side by side).
+
+        Args:
+            *layouts (Layout): Positional arguments should be (sub) Layout instances.
+        """
+        self.split(*layouts, splitter="row")
+
+    def split_column(self, *layouts: Union["Layout", RenderableType]) -> None:
+        """Split the layout in to a column (layouts stacked on top of each other).
+
+        Args:
+            *layouts (Layout): Positional arguments should be (sub) Layout instances.
+        """
+        self.split(*layouts, splitter="column")
+
+    def unsplit(self) -> None:
+        """Reset splits to initial state."""
+        del self._children[:]
+
+    def update(self, renderable: RenderableType) -> None:
+        """Update renderable.
+
+        Args:
+            renderable (RenderableType): New renderable object.
+        """
+        with self._lock:
+            self._renderable = renderable
+
+    def refresh_screen(self, console: "Console", layout_name: str) -> None:
+        """Refresh a sub-layout.
+
+        Args:
+            console (Console): Console instance where Layout is to be rendered.
+            layout_name (str): Name of layout.
+        """
+        with self._lock:
+            layout = self[layout_name]
+            region, _lines = self._render_map[layout]
+            (x, y, width, height) = region
+            lines = console.render_lines(
+                layout, console.options.update_dimensions(width, height)
+            )
+            self._render_map[layout] = LayoutRender(region, lines)
+            console.update_screen_lines(lines, x, y)
+
+    def _make_region_map(self, width: int, height: int) -> RegionMap:
+        """Create a dict that maps layout on to Region."""
+        stack: List[Tuple[Layout, Region]] = [(self, Region(0, 0, width, height))]
+        push = stack.append
+        pop = stack.pop
+        layout_regions: List[Tuple[Layout, Region]] = []
+        append_layout_region = layout_regions.append
+        while stack:
+            append_layout_region(pop())
+            layout, region = layout_regions[-1]
+            children = layout.children
+            if children:
+                for child_and_region in layout.splitter.divide(children, region):
+                    push(child_and_region)
+
+        region_map = {
+            layout: region
+            for layout, region in sorted(layout_regions, key=itemgetter(1))
+        }
+        return region_map
+
+    def render(self, console: Console, options: ConsoleOptions) -> RenderMap:
+        """Render the sub_layouts.
+
+        Args:
+            console (Console): Console instance.
+            options (ConsoleOptions): Console options.
+
+        Returns:
+            RenderMap: A dict that maps Layout on to a tuple of Region, lines
+        """
+        render_width = options.max_width
+        render_height = options.height or console.height
+        region_map = self._make_region_map(render_width, render_height)
+        layout_regions = [
+            (layout, region)
+            for layout, region in region_map.items()
+            if not layout.children
+        ]
+        render_map: Dict["Layout", "LayoutRender"] = {}
+        render_lines = console.render_lines
+        update_dimensions = options.update_dimensions
+
+        for layout, region in layout_regions:
+            lines = render_lines(
+                layout.renderable, update_dimensions(region.width, region.height)
+            )
+            render_map[layout] = LayoutRender(region, lines)
+        return render_map
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        with self._lock:
+            width = options.max_width or console.width
+            height = options.height or console.height
+            render_map = self.render(console, options.update_dimensions(width, height))
+            self._render_map = render_map
+            layout_lines: List[List[Segment]] = [[] for _ in range(height)]
+            _islice = islice
+            for region, lines in render_map.values():
+                _x, y, _layout_width, layout_height = region
+                for row, line in zip(
+                    _islice(layout_lines, y, y + layout_height), lines
+                ):
+                    row.extend(line)
+
+            new_line = Segment.line()
+            for layout_row in layout_lines:
+                yield from layout_row
+                yield new_line
+
+
+if __name__ == "__main__":
+    from rich.console import Console
+
+    console = Console()
+    layout = Layout()
+
+    layout.split_column(
+        Layout(name="header", size=3),
+        Layout(ratio=1, name="main"),
+        Layout(size=10, name="footer"),
+    )
+
+    layout["main"].split_row(Layout(name="side"), Layout(name="body", ratio=2))
+
+    layout["body"].split_row(Layout(name="content", ratio=2), Layout(name="s2"))
+
+    layout["s2"].split_column(
+        Layout(name="top"), Layout(name="middle"), Layout(name="bottom")
+    )
+
+    layout["side"].split_column(Layout(layout.tree, name="left1"), Layout(name="left2"))
+
+    layout["content"].update("foo")
+
+    console.print(layout)
diff --git a/lib/rich/live.py b/lib/rich/live.py
new file mode 100644
index 0000000..2fd893b
--- /dev/null
+++ b/lib/rich/live.py
@@ -0,0 +1,404 @@
+from __future__ import annotations
+
+import sys
+from threading import Event, RLock, Thread
+from types import TracebackType
+from typing import IO, TYPE_CHECKING, Any, Callable, List, Optional, TextIO, Type, cast
+
+from . import get_console
+from .console import Console, ConsoleRenderable, Group, RenderableType, RenderHook
+from .control import Control
+from .file_proxy import FileProxy
+from .jupyter import JupyterMixin
+from .live_render import LiveRender, VerticalOverflowMethod
+from .screen import Screen
+from .text import Text
+
+if TYPE_CHECKING:
+    # Can be replaced with `from typing import Self` in Python 3.11+
+    from typing_extensions import Self  # pragma: no cover
+
+
+class _RefreshThread(Thread):
+    """A thread that calls refresh() at regular intervals."""
+
+    def __init__(self, live: "Live", refresh_per_second: float) -> None:
+        self.live = live
+        self.refresh_per_second = refresh_per_second
+        self.done = Event()
+        super().__init__(daemon=True)
+
+    def stop(self) -> None:
+        self.done.set()
+
+    def run(self) -> None:
+        while not self.done.wait(1 / self.refresh_per_second):
+            with self.live._lock:
+                if not self.done.is_set():
+                    self.live.refresh()
+
+
+class Live(JupyterMixin, RenderHook):
+    """Renders an auto-updating live display of any given renderable.
+
+    Args:
+        renderable (RenderableType, optional): The renderable to live display. Defaults to displaying nothing.
+        console (Console, optional): Optional Console instance. Defaults to an internal Console instance writing to stdout.
+        screen (bool, optional): Enable alternate screen mode. Defaults to False.
+        auto_refresh (bool, optional): Enable auto refresh. If disabled, you will need to call `refresh()` or `update()` with refresh flag. Defaults to True
+        refresh_per_second (float, optional): Number of times per second to refresh the live display. Defaults to 4.
+        transient (bool, optional): Clear the renderable on exit (has no effect when screen=True). Defaults to False.
+        redirect_stdout (bool, optional): Enable redirection of stdout, so ``print`` may be used. Defaults to True.
+        redirect_stderr (bool, optional): Enable redirection of stderr. Defaults to True.
+        vertical_overflow (VerticalOverflowMethod, optional): How to handle renderable when it is too tall for the console. Defaults to "ellipsis".
+        get_renderable (Callable[[], RenderableType], optional): Optional callable to get renderable. Defaults to None.
+    """
+
+    def __init__(
+        self,
+        renderable: Optional[RenderableType] = None,
+        *,
+        console: Optional[Console] = None,
+        screen: bool = False,
+        auto_refresh: bool = True,
+        refresh_per_second: float = 4,
+        transient: bool = False,
+        redirect_stdout: bool = True,
+        redirect_stderr: bool = True,
+        vertical_overflow: VerticalOverflowMethod = "ellipsis",
+        get_renderable: Optional[Callable[[], RenderableType]] = None,
+    ) -> None:
+        assert refresh_per_second > 0, "refresh_per_second must be > 0"
+        self._renderable = renderable
+        self.console = console if console is not None else get_console()
+        self._screen = screen
+        self._alt_screen = False
+
+        self._redirect_stdout = redirect_stdout
+        self._redirect_stderr = redirect_stderr
+        self._restore_stdout: Optional[IO[str]] = None
+        self._restore_stderr: Optional[IO[str]] = None
+
+        self._lock = RLock()
+        self.ipy_widget: Optional[Any] = None
+        self.auto_refresh = auto_refresh
+        self._started: bool = False
+        self.transient = True if screen else transient
+
+        self._refresh_thread: Optional[_RefreshThread] = None
+        self.refresh_per_second = refresh_per_second
+
+        self.vertical_overflow = vertical_overflow
+        self._get_renderable = get_renderable
+        self._live_render = LiveRender(
+            self.get_renderable(), vertical_overflow=vertical_overflow
+        )
+        self._nested = False
+
+    @property
+    def is_started(self) -> bool:
+        """Check if live display has been started."""
+        return self._started
+
+    def get_renderable(self) -> RenderableType:
+        renderable = (
+            self._get_renderable()
+            if self._get_renderable is not None
+            else self._renderable
+        )
+        return renderable or ""
+
+    def start(self, refresh: bool = False) -> None:
+        """Start live rendering display.
+
+        Args:
+            refresh (bool, optional): Also refresh. Defaults to False.
+        """
+        with self._lock:
+            if self._started:
+                return
+            self._started = True
+
+            if not self.console.set_live(self):
+                self._nested = True
+                return
+
+            if self._screen:
+                self._alt_screen = self.console.set_alt_screen(True)
+            self.console.show_cursor(False)
+            self._enable_redirect_io()
+            self.console.push_render_hook(self)
+            if refresh:
+                try:
+                    self.refresh()
+                except Exception:
+                    # If refresh fails, we want to stop the redirection of sys.stderr,
+                    # so the error stacktrace is properly displayed in the terminal.
+                    # (or, if the code that calls Rich captures the exception and wants to display something,
+                    # let this be displayed in the terminal).
+                    self.stop()
+                    raise
+            if self.auto_refresh:
+                self._refresh_thread = _RefreshThread(self, self.refresh_per_second)
+                self._refresh_thread.start()
+
+    def stop(self) -> None:
+        """Stop live rendering display."""
+        with self._lock:
+            if not self._started:
+                return
+            self._started = False
+            self.console.clear_live()
+            if self._nested:
+                if not self.transient:
+                    self.console.print(self.renderable)
+                return
+
+            if self.auto_refresh and self._refresh_thread is not None:
+                self._refresh_thread.stop()
+                self._refresh_thread = None
+            # allow it to fully render on the last even if overflow
+            self.vertical_overflow = "visible"
+            with self.console:
+                try:
+                    if not self._alt_screen and not self.console.is_jupyter:
+                        self.refresh()
+                finally:
+                    self._disable_redirect_io()
+                    self.console.pop_render_hook()
+                    if (
+                        not self._alt_screen
+                        and self.console.is_terminal
+                        and self._live_render.last_render_height
+                    ):
+                        self.console.line()
+                    self.console.show_cursor(True)
+                    if self._alt_screen:
+                        self.console.set_alt_screen(False)
+                    if self.transient and not self._alt_screen:
+                        self.console.control(self._live_render.restore_cursor())
+                    if self.ipy_widget is not None and self.transient:
+                        self.ipy_widget.close()  # pragma: no cover
+
+    def __enter__(self) -> Self:
+        self.start(refresh=self._renderable is not None)
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.stop()
+
+    def _enable_redirect_io(self) -> None:
+        """Enable redirecting of stdout / stderr."""
+        if self.console.is_terminal or self.console.is_jupyter:
+            if self._redirect_stdout and not isinstance(sys.stdout, FileProxy):
+                self._restore_stdout = sys.stdout
+                sys.stdout = cast("TextIO", FileProxy(self.console, sys.stdout))
+            if self._redirect_stderr and not isinstance(sys.stderr, FileProxy):
+                self._restore_stderr = sys.stderr
+                sys.stderr = cast("TextIO", FileProxy(self.console, sys.stderr))
+
+    def _disable_redirect_io(self) -> None:
+        """Disable redirecting of stdout / stderr."""
+        if self._restore_stdout:
+            sys.stdout = cast("TextIO", self._restore_stdout)
+            self._restore_stdout = None
+        if self._restore_stderr:
+            sys.stderr = cast("TextIO", self._restore_stderr)
+            self._restore_stderr = None
+
+    @property
+    def renderable(self) -> RenderableType:
+        """Get the renderable that is being displayed
+
+        Returns:
+            RenderableType: Displayed renderable.
+        """
+        live_stack = self.console._live_stack
+        renderable: RenderableType
+        if live_stack and self is live_stack[0]:
+            # The first Live instance will render everything in the Live stack
+            renderable = Group(*[live.get_renderable() for live in live_stack])
+        else:
+            renderable = self.get_renderable()
+        return Screen(renderable) if self._alt_screen else renderable
+
+    def update(self, renderable: RenderableType, *, refresh: bool = False) -> None:
+        """Update the renderable that is being displayed
+
+        Args:
+            renderable (RenderableType): New renderable to use.
+            refresh (bool, optional): Refresh the display. Defaults to False.
+        """
+        if isinstance(renderable, str):
+            renderable = self.console.render_str(renderable)
+        with self._lock:
+            self._renderable = renderable
+            if refresh:
+                self.refresh()
+
+    def refresh(self) -> None:
+        """Update the display of the Live Render."""
+        with self._lock:
+            self._live_render.set_renderable(self.renderable)
+            if self._nested:
+                if self.console._live_stack:
+                    self.console._live_stack[0].refresh()
+                return
+
+            if self.console.is_jupyter:  # pragma: no cover
+                try:
+                    from IPython.display import display
+                    from ipywidgets import Output
+                except ImportError:
+                    import warnings
+
+                    warnings.warn('install "ipywidgets" for Jupyter support')
+                else:
+                    if self.ipy_widget is None:
+                        self.ipy_widget = Output()
+                        display(self.ipy_widget)
+
+                    with self.ipy_widget:
+                        self.ipy_widget.clear_output(wait=True)
+                        self.console.print(self._live_render.renderable)
+            elif self.console.is_terminal and not self.console.is_dumb_terminal:
+                with self.console:
+                    self.console.print(Control())
+            elif (
+                not self._started and not self.transient
+            ):  # if it is finished allow files or dumb-terminals to see final result
+                with self.console:
+                    self.console.print(Control())
+
+    def process_renderables(
+        self, renderables: List[ConsoleRenderable]
+    ) -> List[ConsoleRenderable]:
+        """Process renderables to restore cursor and display progress."""
+        self._live_render.vertical_overflow = self.vertical_overflow
+        if self.console.is_interactive:
+            # lock needs acquiring as user can modify live_render renderable at any time unlike in Progress.
+            with self._lock:
+                reset = (
+                    Control.home()
+                    if self._alt_screen
+                    else self._live_render.position_cursor()
+                )
+                renderables = [reset, *renderables, self._live_render]
+        elif (
+            not self._started and not self.transient
+        ):  # if it is finished render the final output for files or dumb_terminals
+            renderables = [*renderables, self._live_render]
+
+        return renderables
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import random
+    import time
+    from itertools import cycle
+    from typing import Dict, List, Tuple
+
+    from .align import Align
+    from .console import Console
+    from .live import Live as Live
+    from .panel import Panel
+    from .rule import Rule
+    from .syntax import Syntax
+    from .table import Table
+
+    console = Console()
+
+    syntax = Syntax(
+        '''def loop_last(values: Iterable[T]) -> Iterable[Tuple[bool, T]]:
+    """Iterate and generate a tuple with a flag for last value."""
+    iter_values = iter(values)
+    try:
+        previous_value = next(iter_values)
+    except StopIteration:
+        return
+    for value in iter_values:
+        yield False, previous_value
+        previous_value = value
+    yield True, previous_value''',
+        "python",
+        line_numbers=True,
+    )
+
+    table = Table("foo", "bar", "baz")
+    table.add_row("1", "2", "3")
+
+    progress_renderables = [
+        "You can make the terminal shorter and taller to see the live table hide"
+        "Text may be printed while the progress bars are rendering.",
+        Panel("In fact, [i]any[/i] renderable will work"),
+        "Such as [magenta]tables[/]...",
+        table,
+        "Pretty printed structures...",
+        {"type": "example", "text": "Pretty printed"},
+        "Syntax...",
+        syntax,
+        Rule("Give it a try!"),
+    ]
+
+    examples = cycle(progress_renderables)
+
+    exchanges = [
+        "SGD",
+        "MYR",
+        "EUR",
+        "USD",
+        "AUD",
+        "JPY",
+        "CNH",
+        "HKD",
+        "CAD",
+        "INR",
+        "DKK",
+        "GBP",
+        "RUB",
+        "NZD",
+        "MXN",
+        "IDR",
+        "TWD",
+        "THB",
+        "VND",
+    ]
+    with Live(console=console) as live_table:
+        exchange_rate_dict: Dict[Tuple[str, str], float] = {}
+
+        for index in range(100):
+            select_exchange = exchanges[index % len(exchanges)]
+
+            for exchange in exchanges:
+                if exchange == select_exchange:
+                    continue
+                time.sleep(0.4)
+                if random.randint(0, 10) < 1:
+                    console.log(next(examples))
+                exchange_rate_dict[(select_exchange, exchange)] = 200 / (
+                    (random.random() * 320) + 1
+                )
+                if len(exchange_rate_dict) > len(exchanges) - 1:
+                    exchange_rate_dict.pop(list(exchange_rate_dict.keys())[0])
+                table = Table(title="Exchange Rates")
+
+                table.add_column("Source Currency")
+                table.add_column("Destination Currency")
+                table.add_column("Exchange Rate")
+
+                for (source, dest), exchange_rate in exchange_rate_dict.items():
+                    table.add_row(
+                        source,
+                        dest,
+                        Text(
+                            f"{exchange_rate:.4f}",
+                            style="red" if exchange_rate < 1.0 else "green",
+                        ),
+                    )
+
+                live_table.update(Align.center(table))
diff --git a/lib/rich/live_render.py b/lib/rich/live_render.py
new file mode 100644
index 0000000..e7ec970
--- /dev/null
+++ b/lib/rich/live_render.py
@@ -0,0 +1,116 @@
+from typing import Literal, Optional, Tuple
+
+from ._loop import loop_last
+from .console import Console, ConsoleOptions, RenderableType, RenderResult
+from .control import Control
+from .segment import ControlType, Segment
+from .style import StyleType
+from .text import Text
+
+VerticalOverflowMethod = Literal["crop", "ellipsis", "visible"]
+
+
+class LiveRender:
+    """Creates a renderable that may be updated.
+
+    Args:
+        renderable (RenderableType): Any renderable object.
+        style (StyleType, optional): An optional style to apply to the renderable. Defaults to "".
+    """
+
+    def __init__(
+        self,
+        renderable: RenderableType,
+        style: StyleType = "",
+        vertical_overflow: VerticalOverflowMethod = "ellipsis",
+    ) -> None:
+        self.renderable = renderable
+        self.style = style
+        self.vertical_overflow = vertical_overflow
+        self._shape: Optional[Tuple[int, int]] = None
+
+    @property
+    def last_render_height(self) -> int:
+        """The number of lines in the last render (may be 0 if nothing was rendered).
+
+        Returns:
+            Height in lines
+        """
+        if self._shape is None:
+            return 0
+        return self._shape[1]
+
+    def set_renderable(self, renderable: RenderableType) -> None:
+        """Set a new renderable.
+
+        Args:
+            renderable (RenderableType): Any renderable object, including str.
+        """
+        self.renderable = renderable
+
+    def position_cursor(self) -> Control:
+        """Get control codes to move cursor to beginning of live render.
+
+        Returns:
+            Control: A control instance that may be printed.
+        """
+        if self._shape is not None:
+            _, height = self._shape
+            return Control(
+                ControlType.CARRIAGE_RETURN,
+                (ControlType.ERASE_IN_LINE, 2),
+                *(
+                    (
+                        (ControlType.CURSOR_UP, 1),
+                        (ControlType.ERASE_IN_LINE, 2),
+                    )
+                    * (height - 1)
+                )
+            )
+        return Control()
+
+    def restore_cursor(self) -> Control:
+        """Get control codes to clear the render and restore the cursor to its previous position.
+
+        Returns:
+            Control: A Control instance that may be printed.
+        """
+        if self._shape is not None:
+            _, height = self._shape
+            return Control(
+                ControlType.CARRIAGE_RETURN,
+                *((ControlType.CURSOR_UP, 1), (ControlType.ERASE_IN_LINE, 2)) * height
+            )
+        return Control()
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        renderable = self.renderable
+        style = console.get_style(self.style)
+        lines = console.render_lines(renderable, options, style=style, pad=False)
+        shape = Segment.get_shape(lines)
+
+        _, height = shape
+        if height > options.size.height:
+            if self.vertical_overflow == "crop":
+                lines = lines[: options.size.height]
+                shape = Segment.get_shape(lines)
+            elif self.vertical_overflow == "ellipsis":
+                lines = lines[: (options.size.height - 1)]
+                overflow_text = Text(
+                    "...",
+                    overflow="crop",
+                    justify="center",
+                    end="",
+                    style="live.ellipsis",
+                )
+                lines.append(list(console.render(overflow_text)))
+                shape = Segment.get_shape(lines)
+        self._shape = shape
+
+        new_line = Segment.line()
+        for last, line in loop_last(lines):
+            yield from line
+            if not last:
+                yield new_line
diff --git a/lib/rich/logging.py b/lib/rich/logging.py
new file mode 100644
index 0000000..c3e7a5f
--- /dev/null
+++ b/lib/rich/logging.py
@@ -0,0 +1,297 @@
+import logging
+from datetime import datetime
+from logging import Handler, LogRecord
+from pathlib import Path
+from types import ModuleType
+from typing import ClassVar, Iterable, List, Optional, Type, Union
+
+from rich._null_file import NullFile
+
+from . import get_console
+from ._log_render import FormatTimeCallable, LogRender
+from .console import Console, ConsoleRenderable
+from .highlighter import Highlighter, ReprHighlighter
+from .text import Text
+from .traceback import Traceback
+
+
+class RichHandler(Handler):
+    """A logging handler that renders output with Rich. The time / level / message and file are displayed in columns.
+    The level is color coded, and the message is syntax highlighted.
+
+    Note:
+        Be careful when enabling console markup in log messages if you have configured logging for libraries not
+        under your control. If a dependency writes messages containing square brackets, it may not produce the intended output.
+
+    Args:
+        level (Union[int, str], optional): Log level. Defaults to logging.NOTSET.
+        console (:class:`~rich.console.Console`, optional): Optional console instance to write logs.
+            Default will use a global console instance writing to stdout.
+        show_time (bool, optional): Show a column for the time. Defaults to True.
+        omit_repeated_times (bool, optional): Omit repetition of the same time. Defaults to True.
+        show_level (bool, optional): Show a column for the level. Defaults to True.
+        show_path (bool, optional): Show the path to the original log call. Defaults to True.
+        enable_link_path (bool, optional): Enable terminal link of path column to file. Defaults to True.
+        highlighter (Highlighter, optional): Highlighter to style log messages, or None to use ReprHighlighter. Defaults to None.
+        markup (bool, optional): Enable console markup in log messages. Defaults to False.
+        rich_tracebacks (bool, optional): Enable rich tracebacks with syntax highlighting and formatting. Defaults to False.
+        tracebacks_width (Optional[int], optional): Number of characters used to render tracebacks, or None for full width. Defaults to None.
+        tracebacks_code_width (int, optional): Number of code characters used to render tracebacks, or None for full width. Defaults to 88.
+        tracebacks_extra_lines (int, optional): Additional lines of code to render tracebacks, or None for full width. Defaults to None.
+        tracebacks_theme (str, optional): Override pygments theme used in traceback.
+        tracebacks_word_wrap (bool, optional): Enable word wrapping of long tracebacks lines. Defaults to True.
+        tracebacks_show_locals (bool, optional): Enable display of locals in tracebacks. Defaults to False.
+        tracebacks_suppress (Sequence[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback.
+        tracebacks_max_frames (int, optional): Optional maximum number of frames returned by traceback.
+        locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to 10.
+        locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80.
+        log_time_format (Union[str, TimeFormatterCallable], optional): If ``log_time`` is enabled, either string for strftime or callable that formats the time. Defaults to "[%x %X] ".
+        keywords (List[str], optional): List of words to highlight instead of ``RichHandler.KEYWORDS``.
+    """
+
+    KEYWORDS: ClassVar[Optional[List[str]]] = [
+        "GET",
+        "POST",
+        "HEAD",
+        "PUT",
+        "DELETE",
+        "OPTIONS",
+        "TRACE",
+        "PATCH",
+    ]
+    HIGHLIGHTER_CLASS: ClassVar[Type[Highlighter]] = ReprHighlighter
+
+    def __init__(
+        self,
+        level: Union[int, str] = logging.NOTSET,
+        console: Optional[Console] = None,
+        *,
+        show_time: bool = True,
+        omit_repeated_times: bool = True,
+        show_level: bool = True,
+        show_path: bool = True,
+        enable_link_path: bool = True,
+        highlighter: Optional[Highlighter] = None,
+        markup: bool = False,
+        rich_tracebacks: bool = False,
+        tracebacks_width: Optional[int] = None,
+        tracebacks_code_width: Optional[int] = 88,
+        tracebacks_extra_lines: int = 3,
+        tracebacks_theme: Optional[str] = None,
+        tracebacks_word_wrap: bool = True,
+        tracebacks_show_locals: bool = False,
+        tracebacks_suppress: Iterable[Union[str, ModuleType]] = (),
+        tracebacks_max_frames: int = 100,
+        locals_max_length: int = 10,
+        locals_max_string: int = 80,
+        log_time_format: Union[str, FormatTimeCallable] = "[%x %X]",
+        keywords: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__(level=level)
+        self.console = console or get_console()
+        self.highlighter = highlighter or self.HIGHLIGHTER_CLASS()
+        self._log_render = LogRender(
+            show_time=show_time,
+            show_level=show_level,
+            show_path=show_path,
+            time_format=log_time_format,
+            omit_repeated_times=omit_repeated_times,
+            level_width=None,
+        )
+        self.enable_link_path = enable_link_path
+        self.markup = markup
+        self.rich_tracebacks = rich_tracebacks
+        self.tracebacks_width = tracebacks_width
+        self.tracebacks_extra_lines = tracebacks_extra_lines
+        self.tracebacks_theme = tracebacks_theme
+        self.tracebacks_word_wrap = tracebacks_word_wrap
+        self.tracebacks_show_locals = tracebacks_show_locals
+        self.tracebacks_suppress = tracebacks_suppress
+        self.tracebacks_max_frames = tracebacks_max_frames
+        self.tracebacks_code_width = tracebacks_code_width
+        self.locals_max_length = locals_max_length
+        self.locals_max_string = locals_max_string
+        self.keywords = keywords
+
+    def get_level_text(self, record: LogRecord) -> Text:
+        """Get the level name from the record.
+
+        Args:
+            record (LogRecord): LogRecord instance.
+
+        Returns:
+            Text: A tuple of the style and level name.
+        """
+        level_name = record.levelname
+        level_text = Text.styled(
+            level_name.ljust(8), f"logging.level.{level_name.lower()}"
+        )
+        return level_text
+
+    def emit(self, record: LogRecord) -> None:
+        """Invoked by logging."""
+        message = self.format(record)
+        traceback = None
+        if (
+            self.rich_tracebacks
+            and record.exc_info
+            and record.exc_info != (None, None, None)
+        ):
+            exc_type, exc_value, exc_traceback = record.exc_info
+            assert exc_type is not None
+            assert exc_value is not None
+            traceback = Traceback.from_exception(
+                exc_type,
+                exc_value,
+                exc_traceback,
+                width=self.tracebacks_width,
+                code_width=self.tracebacks_code_width,
+                extra_lines=self.tracebacks_extra_lines,
+                theme=self.tracebacks_theme,
+                word_wrap=self.tracebacks_word_wrap,
+                show_locals=self.tracebacks_show_locals,
+                locals_max_length=self.locals_max_length,
+                locals_max_string=self.locals_max_string,
+                suppress=self.tracebacks_suppress,
+                max_frames=self.tracebacks_max_frames,
+            )
+            message = record.getMessage()
+            if self.formatter:
+                record.message = record.getMessage()
+                formatter = self.formatter
+                if hasattr(formatter, "usesTime") and formatter.usesTime():
+                    record.asctime = formatter.formatTime(record, formatter.datefmt)
+                message = formatter.formatMessage(record)
+
+        message_renderable = self.render_message(record, message)
+        log_renderable = self.render(
+            record=record, traceback=traceback, message_renderable=message_renderable
+        )
+        if isinstance(self.console.file, NullFile):
+            # Handles pythonw, where stdout/stderr are null, and we return NullFile
+            # instance from Console.file. In this case, we still want to make a log record
+            # even though we won't be writing anything to a file.
+            self.handleError(record)
+        else:
+            try:
+                self.console.print(log_renderable)
+            except Exception:
+                self.handleError(record)
+
+    def render_message(self, record: LogRecord, message: str) -> "ConsoleRenderable":
+        """Render message text in to Text.
+
+        Args:
+            record (LogRecord): logging Record.
+            message (str): String containing log message.
+
+        Returns:
+            ConsoleRenderable: Renderable to display log message.
+        """
+        use_markup = getattr(record, "markup", self.markup)
+        message_text = Text.from_markup(message) if use_markup else Text(message)
+
+        highlighter = getattr(record, "highlighter", self.highlighter)
+        if highlighter:
+            message_text = highlighter(message_text)
+
+        if self.keywords is None:
+            self.keywords = self.KEYWORDS
+
+        if self.keywords:
+            message_text.highlight_words(self.keywords, "logging.keyword")
+
+        return message_text
+
+    def render(
+        self,
+        *,
+        record: LogRecord,
+        traceback: Optional[Traceback],
+        message_renderable: "ConsoleRenderable",
+    ) -> "ConsoleRenderable":
+        """Render log for display.
+
+        Args:
+            record (LogRecord): logging Record.
+            traceback (Optional[Traceback]): Traceback instance or None for no Traceback.
+            message_renderable (ConsoleRenderable): Renderable (typically Text) containing log message contents.
+
+        Returns:
+            ConsoleRenderable: Renderable to display log.
+        """
+        path = Path(record.pathname).name
+        level = self.get_level_text(record)
+        time_format = None if self.formatter is None else self.formatter.datefmt
+        log_time = datetime.fromtimestamp(record.created)
+
+        log_renderable = self._log_render(
+            self.console,
+            [message_renderable] if not traceback else [message_renderable, traceback],
+            log_time=log_time,
+            time_format=time_format,
+            level=level,
+            path=path,
+            line_no=record.lineno,
+            link_path=record.pathname if self.enable_link_path else None,
+        )
+        return log_renderable
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from time import sleep
+
+    FORMAT = "%(message)s"
+    # FORMAT = "%(asctime)-15s - %(levelname)s - %(message)s"
+    logging.basicConfig(
+        level="NOTSET",
+        format=FORMAT,
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, tracebacks_show_locals=True)],
+    )
+    log = logging.getLogger("rich")
+
+    log.info("Server starting...")
+    log.info("Listening on http://127.0.0.1:8080")
+    sleep(1)
+
+    log.info("GET /index.html 200 1298")
+    log.info("GET /imgs/backgrounds/back1.jpg 200 54386")
+    log.info("GET /css/styles.css 200 54386")
+    log.warning("GET /favicon.ico 404 242")
+    sleep(1)
+
+    log.debug(
+        "JSONRPC request\n--> %r\n<-- %r",
+        {
+            "version": "1.1",
+            "method": "confirmFruitPurchase",
+            "params": [["apple", "orange", "mangoes", "pomelo"], 1.123],
+            "id": "194521489",
+        },
+        {"version": "1.1", "result": True, "error": None, "id": "194521489"},
+    )
+    log.debug(
+        "Loading configuration file /adasd/asdasd/qeqwe/qwrqwrqwr/sdgsdgsdg/werwerwer/dfgerert/ertertert/ertetert/werwerwer"
+    )
+    log.error("Unable to find 'pomelo' in database!")
+    log.info("POST /jsonrpc/ 200 65532")
+    log.info("POST /admin/ 401 42234")
+    log.warning("password was rejected for admin site.")
+
+    def divide() -> None:
+        number = 1
+        divisor = 0
+        foos = ["foo"] * 100
+        log.debug("in divide")
+        try:
+            number / divisor
+        except:
+            log.exception("An error of some kind occurred!")
+
+    divide()
+    sleep(1)
+    log.critical("Out of memory!")
+    log.info("Server exited with code=-1")
+    log.info("[bold]EXITING...[/bold]", extra=dict(markup=True))
diff --git a/lib/rich/markdown.py b/lib/rich/markdown.py
new file mode 100644
index 0000000..5db183b
--- /dev/null
+++ b/lib/rich/markdown.py
@@ -0,0 +1,793 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from typing import ClassVar, Iterable, get_args
+
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+
+from rich.table import Table
+
+from . import box
+from ._loop import loop_first
+from ._stack import Stack
+from .console import Console, ConsoleOptions, JustifyMethod, RenderResult
+from .containers import Renderables
+from .jupyter import JupyterMixin
+from .rule import Rule
+from .segment import Segment
+from .style import Style, StyleStack
+from .syntax import Syntax
+from .text import Text, TextType
+
+
+class MarkdownElement:
+    new_line: ClassVar[bool] = True
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> MarkdownElement:
+        """Factory to create markdown element,
+
+        Args:
+            markdown (Markdown): The parent Markdown object.
+            token (Token): A node from markdown-it.
+
+        Returns:
+            MarkdownElement: A new markdown element
+        """
+        return cls()
+
+    def on_enter(self, context: MarkdownContext) -> None:
+        """Called when the node is entered.
+
+        Args:
+            context (MarkdownContext): The markdown context.
+        """
+
+    def on_text(self, context: MarkdownContext, text: TextType) -> None:
+        """Called when text is parsed.
+
+        Args:
+            context (MarkdownContext): The markdown context.
+        """
+
+    def on_leave(self, context: MarkdownContext) -> None:
+        """Called when the parser leaves the element.
+
+        Args:
+            context (MarkdownContext): [description]
+        """
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        """Called when a child element is closed.
+
+        This method allows a parent element to take over rendering of its children.
+
+        Args:
+            context (MarkdownContext): The markdown context.
+            child (MarkdownElement): The child markdown element.
+
+        Returns:
+            bool: Return True to render the element, or False to not render the element.
+        """
+        return True
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        return ()
+
+
+class UnknownElement(MarkdownElement):
+    """An unknown element.
+
+    Hopefully there will be no unknown elements, and we will have a MarkdownElement for
+    everything in the document.
+
+    """
+
+
+class TextElement(MarkdownElement):
+    """Base class for elements that render text."""
+
+    style_name = "none"
+
+    def on_enter(self, context: MarkdownContext) -> None:
+        self.style = context.enter_style(self.style_name)
+        self.text = Text(justify="left")
+
+    def on_text(self, context: MarkdownContext, text: TextType) -> None:
+        self.text.append(text, context.current_style if isinstance(text, str) else None)
+
+    def on_leave(self, context: MarkdownContext) -> None:
+        context.leave_style()
+
+
+class Paragraph(TextElement):
+    """A Paragraph."""
+
+    style_name = "markdown.paragraph"
+    justify: JustifyMethod
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> Paragraph:
+        return cls(justify=markdown.justify or "left")
+
+    def __init__(self, justify: JustifyMethod) -> None:
+        self.justify = justify
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        self.text.justify = self.justify
+        yield self.text
+
+
+@dataclass
+class HeadingFormat:
+    justify: JustifyMethod = "left"
+    style: str = ""
+
+
+class Heading(TextElement):
+    """A heading."""
+
+    LEVEL_ALIGN: ClassVar[dict[str, JustifyMethod]] = {
+        "h1": "center",
+        "h2": "left",
+        "h3": "left",
+        "h4": "left",
+        "h5": "left",
+        "h6": "left",
+    }
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> Heading:
+        return cls(token.tag)
+
+    def on_enter(self, context: MarkdownContext) -> None:
+        self.text = Text()
+        context.enter_style(self.style_name)
+
+    def __init__(self, tag: str) -> None:
+        self.tag = tag
+        self.style_name = f"markdown.{tag}"
+        super().__init__()
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        text = self.text.copy()
+        heading_justify = self.LEVEL_ALIGN.get(self.tag, "left")
+        text.justify = heading_justify
+        yield text
+
+
+class CodeBlock(TextElement):
+    """A code block with syntax highlighting."""
+
+    style_name = "markdown.code_block"
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> CodeBlock:
+        node_info = token.info or ""
+        lexer_name = node_info.partition(" ")[0]
+        return cls(lexer_name or "text", markdown.code_theme)
+
+    def __init__(self, lexer_name: str, theme: str) -> None:
+        self.lexer_name = lexer_name
+        self.theme = theme
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        code = str(self.text).rstrip()
+        syntax = Syntax(
+            code, self.lexer_name, theme=self.theme, word_wrap=True, padding=1
+        )
+        yield syntax
+
+
+class BlockQuote(TextElement):
+    """A block quote."""
+
+    style_name = "markdown.block_quote"
+
+    def __init__(self) -> None:
+        self.elements: Renderables = Renderables()
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        self.elements.append(child)
+        return False
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        render_options = options.update(width=options.max_width - 4)
+        lines = console.render_lines(self.elements, render_options, style=self.style)
+        style = self.style
+        new_line = Segment("\n")
+        padding = Segment("▌ ", style)
+        for line in lines:
+            yield padding
+            yield from line
+            yield new_line
+
+
+class HorizontalRule(MarkdownElement):
+    """A horizontal rule to divide sections."""
+
+    new_line = False
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        style = console.get_style("markdown.hr", default="none")
+        yield Rule(style=style, characters="-")
+        yield Text()
+
+
+class TableElement(MarkdownElement):
+    """MarkdownElement corresponding to `table_open`."""
+
+    def __init__(self) -> None:
+        self.header: TableHeaderElement | None = None
+        self.body: TableBodyElement | None = None
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        if isinstance(child, TableHeaderElement):
+            self.header = child
+        elif isinstance(child, TableBodyElement):
+            self.body = child
+        else:
+            raise RuntimeError("Couldn't process markdown table.")
+        return False
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        table = Table(
+            box=box.SIMPLE,
+            pad_edge=False,
+            style="markdown.table.border",
+            show_edge=True,
+            collapse_padding=True,
+        )
+
+        if self.header is not None and self.header.row is not None:
+            for column in self.header.row.cells:
+                heading = column.content.copy()
+                heading.stylize("markdown.table.header")
+                table.add_column(heading)
+
+        if self.body is not None:
+            for row in self.body.rows:
+                row_content = [element.content for element in row.cells]
+                table.add_row(*row_content)
+
+        yield table
+
+
+class TableHeaderElement(MarkdownElement):
+    """MarkdownElement corresponding to `thead_open` and `thead_close`."""
+
+    def __init__(self) -> None:
+        self.row: TableRowElement | None = None
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        assert isinstance(child, TableRowElement)
+        self.row = child
+        return False
+
+
+class TableBodyElement(MarkdownElement):
+    """MarkdownElement corresponding to `tbody_open` and `tbody_close`."""
+
+    def __init__(self) -> None:
+        self.rows: list[TableRowElement] = []
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        assert isinstance(child, TableRowElement)
+        self.rows.append(child)
+        return False
+
+
+class TableRowElement(MarkdownElement):
+    """MarkdownElement corresponding to `tr_open` and `tr_close`."""
+
+    def __init__(self) -> None:
+        self.cells: list[TableDataElement] = []
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        assert isinstance(child, TableDataElement)
+        self.cells.append(child)
+        return False
+
+
+class TableDataElement(MarkdownElement):
+    """MarkdownElement corresponding to `td_open` and `td_close`
+    and `th_open` and `th_close`."""
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> MarkdownElement:
+        style = str(token.attrs.get("style")) or ""
+
+        justify: JustifyMethod
+        if "text-align:right" in style:
+            justify = "right"
+        elif "text-align:center" in style:
+            justify = "center"
+        elif "text-align:left" in style:
+            justify = "left"
+        else:
+            justify = "default"
+
+        assert justify in get_args(JustifyMethod)
+        return cls(justify=justify)
+
+    def __init__(self, justify: JustifyMethod) -> None:
+        self.content: Text = Text("", justify=justify)
+        self.justify = justify
+
+    def on_text(self, context: MarkdownContext, text: TextType) -> None:
+        text = Text(text) if isinstance(text, str) else text
+        text.stylize(context.current_style)
+        self.content.append_text(text)
+
+
+class ListElement(MarkdownElement):
+    """A list element."""
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> ListElement:
+        return cls(token.type, int(token.attrs.get("start", 1)))
+
+    def __init__(self, list_type: str, list_start: int | None) -> None:
+        self.items: list[ListItem] = []
+        self.list_type = list_type
+        self.list_start = list_start
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        assert isinstance(child, ListItem)
+        self.items.append(child)
+        return False
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        if self.list_type == "bullet_list_open":
+            for item in self.items:
+                yield from item.render_bullet(console, options)
+        else:
+            number = 1 if self.list_start is None else self.list_start
+            last_number = number + len(self.items)
+            for index, item in enumerate(self.items):
+                yield from item.render_number(
+                    console, options, number + index, last_number
+                )
+
+
+class ListItem(TextElement):
+    """An item in a list."""
+
+    style_name = "markdown.item"
+
+    def __init__(self) -> None:
+        self.elements: Renderables = Renderables()
+
+    def on_child_close(self, context: MarkdownContext, child: MarkdownElement) -> bool:
+        self.elements.append(child)
+        return False
+
+    def render_bullet(self, console: Console, options: ConsoleOptions) -> RenderResult:
+        render_options = options.update(width=options.max_width - 3)
+        lines = console.render_lines(self.elements, render_options, style=self.style)
+        bullet_style = console.get_style("markdown.item.bullet", default="none")
+
+        bullet = Segment(" • ", bullet_style)
+        padding = Segment(" " * 3, bullet_style)
+        new_line = Segment("\n")
+        for first, line in loop_first(lines):
+            yield bullet if first else padding
+            yield from line
+            yield new_line
+
+    def render_number(
+        self, console: Console, options: ConsoleOptions, number: int, last_number: int
+    ) -> RenderResult:
+        number_width = len(str(last_number)) + 2
+        render_options = options.update(width=options.max_width - number_width)
+        lines = console.render_lines(self.elements, render_options, style=self.style)
+        number_style = console.get_style("markdown.item.number", default="none")
+
+        new_line = Segment("\n")
+        padding = Segment(" " * number_width, number_style)
+        numeral = Segment(f"{number}".rjust(number_width - 1) + " ", number_style)
+        for first, line in loop_first(lines):
+            yield numeral if first else padding
+            yield from line
+            yield new_line
+
+
+class Link(TextElement):
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> MarkdownElement:
+        url = token.attrs.get("href", "#")
+        return cls(token.content, str(url))
+
+    def __init__(self, text: str, href: str):
+        self.text = Text(text)
+        self.href = href
+
+
+class ImageItem(TextElement):
+    """Renders a placeholder for an image."""
+
+    new_line = False
+
+    @classmethod
+    def create(cls, markdown: Markdown, token: Token) -> MarkdownElement:
+        """Factory to create markdown element,
+
+        Args:
+            markdown (Markdown): The parent Markdown object.
+            token (Any): A token from markdown-it.
+
+        Returns:
+            MarkdownElement: A new markdown element
+        """
+        return cls(str(token.attrs.get("src", "")), markdown.hyperlinks)
+
+    def __init__(self, destination: str, hyperlinks: bool) -> None:
+        self.destination = destination
+        self.hyperlinks = hyperlinks
+        self.link: str | None = None
+        super().__init__()
+
+    def on_enter(self, context: MarkdownContext) -> None:
+        self.link = context.current_style.link
+        self.text = Text(justify="left")
+        super().on_enter(context)
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        link_style = Style(link=self.link or self.destination or None)
+        title = self.text or Text(self.destination.strip("/").rsplit("/", 1)[-1])
+        if self.hyperlinks:
+            title.stylize(link_style)
+        text = Text.assemble("🌆 ", title, " ", end="")
+        yield text
+
+
+class MarkdownContext:
+    """Manages the console render state."""
+
+    def __init__(
+        self,
+        console: Console,
+        options: ConsoleOptions,
+        style: Style,
+        inline_code_lexer: str | None = None,
+        inline_code_theme: str = "monokai",
+    ) -> None:
+        self.console = console
+        self.options = options
+        self.style_stack: StyleStack = StyleStack(style)
+        self.stack: Stack[MarkdownElement] = Stack()
+
+        self._syntax: Syntax | None = None
+        if inline_code_lexer is not None:
+            self._syntax = Syntax("", inline_code_lexer, theme=inline_code_theme)
+
+    @property
+    def current_style(self) -> Style:
+        """Current style which is the product of all styles on the stack."""
+        return self.style_stack.current
+
+    def on_text(self, text: str, node_type: str) -> None:
+        """Called when the parser visits text."""
+        if node_type in {"fence", "code_inline"} and self._syntax is not None:
+            highlight_text = self._syntax.highlight(text)
+            highlight_text.rstrip()
+            self.stack.top.on_text(
+                self, Text.assemble(highlight_text, style=self.style_stack.current)
+            )
+        else:
+            self.stack.top.on_text(self, text)
+
+    def enter_style(self, style_name: str | Style) -> Style:
+        """Enter a style context."""
+        style = self.console.get_style(style_name, default="none")
+        self.style_stack.push(style)
+        return self.current_style
+
+    def leave_style(self) -> Style:
+        """Leave a style context."""
+        style = self.style_stack.pop()
+        return style
+
+
+class Markdown(JupyterMixin):
+    """A Markdown renderable.
+
+    Args:
+        markup (str): A string containing markdown.
+        code_theme (str, optional): Pygments theme for code blocks. Defaults to "monokai". See https://pygments.org/styles/ for code themes.
+        justify (JustifyMethod, optional): Justify value for paragraphs. Defaults to None.
+        style (Union[str, Style], optional): Optional style to apply to markdown.
+        hyperlinks (bool, optional): Enable hyperlinks. Defaults to ``True``.
+        inline_code_lexer: (str, optional): Lexer to use if inline code highlighting is
+            enabled. Defaults to None.
+        inline_code_theme: (Optional[str], optional): Pygments theme for inline code
+            highlighting, or None for no highlighting. Defaults to None.
+    """
+
+    elements: ClassVar[dict[str, type[MarkdownElement]]] = {
+        "paragraph_open": Paragraph,
+        "heading_open": Heading,
+        "fence": CodeBlock,
+        "code_block": CodeBlock,
+        "blockquote_open": BlockQuote,
+        "hr": HorizontalRule,
+        "bullet_list_open": ListElement,
+        "ordered_list_open": ListElement,
+        "list_item_open": ListItem,
+        "image": ImageItem,
+        "table_open": TableElement,
+        "tbody_open": TableBodyElement,
+        "thead_open": TableHeaderElement,
+        "tr_open": TableRowElement,
+        "td_open": TableDataElement,
+        "th_open": TableDataElement,
+    }
+
+    inlines = {"em", "strong", "code", "s"}
+
+    def __init__(
+        self,
+        markup: str,
+        code_theme: str = "monokai",
+        justify: JustifyMethod | None = None,
+        style: str | Style = "none",
+        hyperlinks: bool = True,
+        inline_code_lexer: str | None = None,
+        inline_code_theme: str | None = None,
+    ) -> None:
+        parser = MarkdownIt().enable("strikethrough").enable("table")
+        self.markup = markup
+        self.parsed = parser.parse(markup)
+        self.code_theme = code_theme
+        self.justify: JustifyMethod | None = justify
+        self.style = style
+        self.hyperlinks = hyperlinks
+        self.inline_code_lexer = inline_code_lexer
+        self.inline_code_theme = inline_code_theme or code_theme
+
+    def _flatten_tokens(self, tokens: Iterable[Token]) -> Iterable[Token]:
+        """Flattens the token stream."""
+        for token in tokens:
+            is_fence = token.type == "fence"
+            is_image = token.tag == "img"
+            if token.children and not (is_image or is_fence):
+                yield from self._flatten_tokens(token.children)
+            else:
+                yield token
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        """Render markdown to the console."""
+        style = console.get_style(self.style, default="none")
+        options = options.update(height=None)
+        context = MarkdownContext(
+            console,
+            options,
+            style,
+            inline_code_lexer=self.inline_code_lexer,
+            inline_code_theme=self.inline_code_theme,
+        )
+        tokens = self.parsed
+        inline_style_tags = self.inlines
+        new_line = False
+        _new_line_segment = Segment.line()
+
+        for token in self._flatten_tokens(tokens):
+            node_type = token.type
+            tag = token.tag
+
+            entering = token.nesting == 1
+            exiting = token.nesting == -1
+            self_closing = token.nesting == 0
+
+            if node_type == "text":
+                context.on_text(token.content, node_type)
+            elif node_type == "hardbreak":
+                context.on_text("\n", node_type)
+            elif node_type == "softbreak":
+                context.on_text(" ", node_type)
+            elif node_type == "link_open":
+                href = str(token.attrs.get("href", ""))
+                if self.hyperlinks:
+                    link_style = console.get_style("markdown.link_url", default="none")
+                    link_style += Style(link=href)
+                    context.enter_style(link_style)
+                else:
+                    context.stack.push(Link.create(self, token))
+            elif node_type == "link_close":
+                if self.hyperlinks:
+                    context.leave_style()
+                else:
+                    element = context.stack.pop()
+                    assert isinstance(element, Link)
+                    link_style = console.get_style("markdown.link", default="none")
+                    context.enter_style(link_style)
+                    context.on_text(element.text.plain, node_type)
+                    context.leave_style()
+                    context.on_text(" (", node_type)
+                    link_url_style = console.get_style(
+                        "markdown.link_url", default="none"
+                    )
+                    context.enter_style(link_url_style)
+                    context.on_text(element.href, node_type)
+                    context.leave_style()
+                    context.on_text(")", node_type)
+            elif (
+                tag in inline_style_tags
+                and node_type != "fence"
+                and node_type != "code_block"
+            ):
+                if entering:
+                    # If it's an opening inline token e.g. strong, em, etc.
+                    # Then we move into a style context i.e. push to stack.
+                    context.enter_style(f"markdown.{tag}")
+                elif exiting:
+                    # If it's a closing inline style, then we pop the style
+                    # off of the stack, to move out of the context of it...
+                    context.leave_style()
+                else:
+                    # If it's a self-closing inline style e.g. `code_inline`
+                    context.enter_style(f"markdown.{tag}")
+                    if token.content:
+                        context.on_text(token.content, node_type)
+                    context.leave_style()
+            else:
+                # Map the markdown tag -> MarkdownElement renderable
+                element_class = self.elements.get(token.type) or UnknownElement
+                element = element_class.create(self, token)
+
+                if entering or self_closing:
+                    context.stack.push(element)
+                    element.on_enter(context)
+
+                if exiting:  # CLOSING tag
+                    element = context.stack.pop()
+
+                    should_render = not context.stack or (
+                        context.stack
+                        and context.stack.top.on_child_close(context, element)
+                    )
+
+                    if should_render:
+                        if new_line:
+                            yield _new_line_segment
+
+                        yield from console.render(element, context.options)
+                elif self_closing:  # SELF-CLOSING tags (e.g. text, code, image)
+                    context.stack.pop()
+                    text = token.content
+                    if text is not None:
+                        element.on_text(context, text)
+
+                    should_render = (
+                        not context.stack
+                        or context.stack
+                        and context.stack.top.on_child_close(context, element)
+                    )
+                    if should_render:
+                        if new_line and node_type != "inline":
+                            yield _new_line_segment
+                        yield from console.render(element, context.options)
+
+                if exiting or self_closing:
+                    element.on_leave(context)
+                    new_line = element.new_line
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import argparse
+    import sys
+
+    parser = argparse.ArgumentParser(
+        description="Render Markdown to the console with Rich"
+    )
+    parser.add_argument(
+        "path",
+        metavar="PATH",
+        help="path to markdown file, or - for stdin",
+    )
+    parser.add_argument(
+        "-c",
+        "--force-color",
+        dest="force_color",
+        action="store_true",
+        default=None,
+        help="force color for non-terminals",
+    )
+    parser.add_argument(
+        "-t",
+        "--code-theme",
+        dest="code_theme",
+        default="monokai",
+        help="pygments code theme",
+    )
+    parser.add_argument(
+        "-i",
+        "--inline-code-lexer",
+        dest="inline_code_lexer",
+        default=None,
+        help="inline_code_lexer",
+    )
+    parser.add_argument(
+        "-y",
+        "--hyperlinks",
+        dest="hyperlinks",
+        action="store_true",
+        help="enable hyperlinks",
+    )
+    parser.add_argument(
+        "-w",
+        "--width",
+        type=int,
+        dest="width",
+        default=None,
+        help="width of output (default will auto-detect)",
+    )
+    parser.add_argument(
+        "-j",
+        "--justify",
+        dest="justify",
+        action="store_true",
+        help="enable full text justify",
+    )
+    parser.add_argument(
+        "-p",
+        "--page",
+        dest="page",
+        action="store_true",
+        help="use pager to scroll output",
+    )
+    args = parser.parse_args()
+
+    from rich.console import Console
+
+    if args.path == "-":
+        markdown_body = sys.stdin.read()
+    else:
+        with open(args.path, encoding="utf-8") as markdown_file:
+            markdown_body = markdown_file.read()
+
+    markdown = Markdown(
+        markdown_body,
+        justify="full" if args.justify else "left",
+        code_theme=args.code_theme,
+        hyperlinks=args.hyperlinks,
+        inline_code_lexer=args.inline_code_lexer,
+    )
+    if args.page:
+        import io
+        import pydoc
+
+        fileio = io.StringIO()
+        console = Console(
+            file=fileio, force_terminal=args.force_color, width=args.width
+        )
+        console.print(markdown)
+        pydoc.pager(fileio.getvalue())
+
+    else:
+        console = Console(
+            force_terminal=args.force_color, width=args.width, record=True
+        )
+        console.print(markdown)
diff --git a/lib/rich/markup.py b/lib/rich/markup.py
new file mode 100644
index 0000000..bd9c05a
--- /dev/null
+++ b/lib/rich/markup.py
@@ -0,0 +1,251 @@
+import re
+from ast import literal_eval
+from operator import attrgetter
+from typing import Callable, Iterable, List, Match, NamedTuple, Optional, Tuple, Union
+
+from ._emoji_replace import _emoji_replace
+from .emoji import EmojiVariant
+from .errors import MarkupError
+from .style import Style
+from .text import Span, Text
+
+RE_TAGS = re.compile(
+    r"""((\\*)\[([a-z#/@][^[]*?)])""",
+    re.VERBOSE,
+)
+
+RE_HANDLER = re.compile(r"^([\w.]*?)(\(.*?\))?$")
+
+
+class Tag(NamedTuple):
+    """A tag in console markup."""
+
+    name: str
+    """The tag name. e.g. 'bold'."""
+    parameters: Optional[str]
+    """Any additional parameters after the name."""
+
+    def __str__(self) -> str:
+        return (
+            self.name if self.parameters is None else f"{self.name} {self.parameters}"
+        )
+
+    @property
+    def markup(self) -> str:
+        """Get the string representation of this tag."""
+        return (
+            f"[{self.name}]"
+            if self.parameters is None
+            else f"[{self.name}={self.parameters}]"
+        )
+
+
+_ReStringMatch = Match[str]  # regex match object
+_ReSubCallable = Callable[[_ReStringMatch], str]  # Callable invoked by re.sub
+_EscapeSubMethod = Callable[[_ReSubCallable, str], str]  # Sub method of a compiled re
+
+
+def escape(
+    markup: str,
+    _escape: _EscapeSubMethod = re.compile(r"(\\*)(\[[a-z#/@][^[]*?])").sub,
+) -> str:
+    """Escapes text so that it won't be interpreted as markup.
+
+    Args:
+        markup (str): Content to be inserted in to markup.
+
+    Returns:
+        str: Markup with square brackets escaped.
+    """
+
+    def escape_backslashes(match: Match[str]) -> str:
+        """Called by re.sub replace matches."""
+        backslashes, text = match.groups()
+        return f"{backslashes}{backslashes}\\{text}"
+
+    markup = _escape(escape_backslashes, markup)
+    if markup.endswith("\\") and not markup.endswith("\\\\"):
+        return markup + "\\"
+
+    return markup
+
+
+def _parse(markup: str) -> Iterable[Tuple[int, Optional[str], Optional[Tag]]]:
+    """Parse markup in to an iterable of tuples of (position, text, tag).
+
+    Args:
+        markup (str): A string containing console markup
+
+    """
+    position = 0
+    _divmod = divmod
+    _Tag = Tag
+    for match in RE_TAGS.finditer(markup):
+        full_text, escapes, tag_text = match.groups()
+        start, end = match.span()
+        if start > position:
+            yield start, markup[position:start], None
+        if escapes:
+            backslashes, escaped = _divmod(len(escapes), 2)
+            if backslashes:
+                # Literal backslashes
+                yield start, "\\" * backslashes, None
+                start += backslashes * 2
+            if escaped:
+                # Escape of tag
+                yield start, full_text[len(escapes) :], None
+                position = end
+                continue
+        text, equals, parameters = tag_text.partition("=")
+        yield start, None, _Tag(text, parameters if equals else None)
+        position = end
+    if position < len(markup):
+        yield position, markup[position:], None
+
+
+def render(
+    markup: str,
+    style: Union[str, Style] = "",
+    emoji: bool = True,
+    emoji_variant: Optional[EmojiVariant] = None,
+) -> Text:
+    """Render console markup in to a Text instance.
+
+    Args:
+        markup (str): A string containing console markup.
+        style: (Union[str, Style]): The style to use.
+        emoji (bool, optional): Also render emoji code. Defaults to True.
+        emoji_variant (str, optional): Optional emoji variant, either "text" or "emoji". Defaults to None.
+
+
+    Raises:
+        MarkupError: If there is a syntax error in the markup.
+
+    Returns:
+        Text: A test instance.
+    """
+    emoji_replace = _emoji_replace
+    if "[" not in markup:
+        return Text(
+            emoji_replace(markup, default_variant=emoji_variant) if emoji else markup,
+            style=style,
+        )
+    text = Text(style=style)
+    append = text.append
+    normalize = Style.normalize
+
+    style_stack: List[Tuple[int, Tag]] = []
+    pop = style_stack.pop
+
+    spans: List[Span] = []
+    append_span = spans.append
+
+    _Span = Span
+    _Tag = Tag
+
+    def pop_style(style_name: str) -> Tuple[int, Tag]:
+        """Pop tag matching given style name."""
+        for index, (_, tag) in enumerate(reversed(style_stack), 1):
+            if tag.name == style_name:
+                return pop(-index)
+        raise KeyError(style_name)
+
+    for position, plain_text, tag in _parse(markup):
+        if plain_text is not None:
+            # Handle open brace escapes, where the brace is not part of a tag.
+            plain_text = plain_text.replace("\\[", "[")
+            append(emoji_replace(plain_text) if emoji else plain_text)
+        elif tag is not None:
+            if tag.name.startswith("/"):  # Closing tag
+                style_name = tag.name[1:].strip()
+
+                if style_name:  # explicit close
+                    style_name = normalize(style_name)
+                    try:
+                        start, open_tag = pop_style(style_name)
+                    except KeyError:
+                        raise MarkupError(
+                            f"closing tag '{tag.markup}' at position {position} doesn't match any open tag"
+                        ) from None
+                else:  # implicit close
+                    try:
+                        start, open_tag = pop()
+                    except IndexError:
+                        raise MarkupError(
+                            f"closing tag '[/]' at position {position} has nothing to close"
+                        ) from None
+
+                if open_tag.name.startswith("@"):
+                    if open_tag.parameters:
+                        handler_name = ""
+                        parameters = open_tag.parameters.strip()
+                        handler_match = RE_HANDLER.match(parameters)
+                        if handler_match is not None:
+                            handler_name, match_parameters = handler_match.groups()
+                            parameters = (
+                                "()" if match_parameters is None else match_parameters
+                            )
+
+                        try:
+                            meta_params = literal_eval(parameters)
+                        except SyntaxError as error:
+                            raise MarkupError(
+                                f"error parsing {parameters!r} in {open_tag.parameters!r}; {error.msg}"
+                            )
+                        except Exception as error:
+                            raise MarkupError(
+                                f"error parsing {open_tag.parameters!r}; {error}"
+                            ) from None
+
+                        if handler_name:
+                            meta_params = (
+                                handler_name,
+                                meta_params
+                                if isinstance(meta_params, tuple)
+                                else (meta_params,),
+                            )
+
+                    else:
+                        meta_params = ()
+
+                    append_span(
+                        _Span(
+                            start, len(text), Style(meta={open_tag.name: meta_params})
+                        )
+                    )
+                else:
+                    append_span(_Span(start, len(text), str(open_tag)))
+
+            else:  # Opening tag
+                normalized_tag = _Tag(normalize(tag.name), tag.parameters)
+                style_stack.append((len(text), normalized_tag))
+
+    text_length = len(text)
+    while style_stack:
+        start, tag = style_stack.pop()
+        style = str(tag)
+        if style:
+            append_span(_Span(start, text_length, style))
+
+    text.spans = sorted(spans[::-1], key=attrgetter("start"))
+    return text
+
+
+if __name__ == "__main__":  # pragma: no cover
+    MARKUP = [
+        "[red]Hello World[/red]",
+        "[magenta]Hello [b]World[/b]",
+        "[bold]Bold[italic] bold and italic [/bold]italic[/italic]",
+        "Click [link=https://www.willmcgugan.com]here[/link] to visit my Blog",
+        ":warning-emoji: [bold red blink] DANGER![/]",
+    ]
+
+    from rich import print
+    from rich.table import Table
+
+    grid = Table("Markup", "Result", padding=(0, 1))
+
+    for markup in MARKUP:
+        grid.add_row(Text(markup), markup)
+
+    print(grid)
diff --git a/lib/rich/measure.py b/lib/rich/measure.py
new file mode 100644
index 0000000..a508ffa
--- /dev/null
+++ b/lib/rich/measure.py
@@ -0,0 +1,151 @@
+from operator import itemgetter
+from typing import TYPE_CHECKING, Callable, NamedTuple, Optional, Sequence
+
+from . import errors
+from .protocol import is_renderable, rich_cast
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderableType
+
+
+class Measurement(NamedTuple):
+    """Stores the minimum and maximum widths (in characters) required to render an object."""
+
+    minimum: int
+    """Minimum number of cells required to render."""
+    maximum: int
+    """Maximum number of cells required to render."""
+
+    @property
+    def span(self) -> int:
+        """Get difference between maximum and minimum."""
+        return self.maximum - self.minimum
+
+    def normalize(self) -> "Measurement":
+        """Get measurement that ensures that minimum <= maximum and minimum >= 0
+
+        Returns:
+            Measurement: A normalized measurement.
+        """
+        minimum, maximum = self
+        minimum = min(max(0, minimum), maximum)
+        return Measurement(max(0, minimum), max(0, max(minimum, maximum)))
+
+    def with_maximum(self, width: int) -> "Measurement":
+        """Get a RenderableWith where the widths are <= width.
+
+        Args:
+            width (int): Maximum desired width.
+
+        Returns:
+            Measurement: New Measurement object.
+        """
+        minimum, maximum = self
+        return Measurement(min(minimum, width), min(maximum, width))
+
+    def with_minimum(self, width: int) -> "Measurement":
+        """Get a RenderableWith where the widths are >= width.
+
+        Args:
+            width (int): Minimum desired width.
+
+        Returns:
+            Measurement: New Measurement object.
+        """
+        minimum, maximum = self
+        width = max(0, width)
+        return Measurement(max(minimum, width), max(maximum, width))
+
+    def clamp(
+        self, min_width: Optional[int] = None, max_width: Optional[int] = None
+    ) -> "Measurement":
+        """Clamp a measurement within the specified range.
+
+        Args:
+            min_width (int): Minimum desired width, or ``None`` for no minimum. Defaults to None.
+            max_width (int): Maximum desired width, or ``None`` for no maximum. Defaults to None.
+
+        Returns:
+            Measurement: New Measurement object.
+        """
+        measurement = self
+        if min_width is not None:
+            measurement = measurement.with_minimum(min_width)
+        if max_width is not None:
+            measurement = measurement.with_maximum(max_width)
+        return measurement
+
+    @classmethod
+    def get(
+        cls, console: "Console", options: "ConsoleOptions", renderable: "RenderableType"
+    ) -> "Measurement":
+        """Get a measurement for a renderable.
+
+        Args:
+            console (~rich.console.Console): Console instance.
+            options (~rich.console.ConsoleOptions): Console options.
+            renderable (RenderableType): An object that may be rendered with Rich.
+
+        Raises:
+            errors.NotRenderableError: If the object is not renderable.
+
+        Returns:
+            Measurement: Measurement object containing range of character widths required to render the object.
+        """
+        _max_width = options.max_width
+        if _max_width < 1:
+            return Measurement(0, 0)
+        if isinstance(renderable, str):
+            renderable = console.render_str(
+                renderable, markup=options.markup, highlight=False
+            )
+        renderable = rich_cast(renderable)
+        if is_renderable(renderable):
+            get_console_width: Optional[
+                Callable[["Console", "ConsoleOptions"], "Measurement"]
+            ] = getattr(renderable, "__rich_measure__", None)
+            if get_console_width is not None:
+                render_width = (
+                    get_console_width(console, options)
+                    .normalize()
+                    .with_maximum(_max_width)
+                )
+                if render_width.maximum < 1:
+                    return Measurement(0, 0)
+                return render_width.normalize()
+            else:
+                return Measurement(0, _max_width)
+        else:
+            raise errors.NotRenderableError(
+                f"Unable to get render width for {renderable!r}; "
+                "a str, Segment, or object with __rich_console__ method is required"
+            )
+
+
+def measure_renderables(
+    console: "Console",
+    options: "ConsoleOptions",
+    renderables: Sequence["RenderableType"],
+) -> "Measurement":
+    """Get a measurement that would fit a number of renderables.
+
+    Args:
+        console (~rich.console.Console): Console instance.
+        options (~rich.console.ConsoleOptions): Console options.
+        renderables (Iterable[RenderableType]): One or more renderable objects.
+
+    Returns:
+        Measurement: Measurement object containing range of character widths required to
+            contain all given renderables.
+    """
+    if not renderables:
+        return Measurement(0, 0)
+    get_measurement = Measurement.get
+    measurements = [
+        get_measurement(console, options, renderable) for renderable in renderables
+    ]
+    measured_width = Measurement(
+        max(measurements, key=itemgetter(0)).minimum,
+        max(measurements, key=itemgetter(1)).maximum,
+    )
+    return measured_width
diff --git a/lib/rich/padding.py b/lib/rich/padding.py
new file mode 100644
index 0000000..d1aa01b
--- /dev/null
+++ b/lib/rich/padding.py
@@ -0,0 +1,141 @@
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+
+if TYPE_CHECKING:
+    from .console import (
+        Console,
+        ConsoleOptions,
+        RenderableType,
+        RenderResult,
+    )
+
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .segment import Segment
+from .style import Style
+
+PaddingDimensions = Union[int, Tuple[int], Tuple[int, int], Tuple[int, int, int, int]]
+
+
+class Padding(JupyterMixin):
+    """Draw space around content.
+
+    Example:
+        >>> print(Padding("Hello", (2, 4), style="on blue"))
+
+    Args:
+        renderable (RenderableType): String or other renderable.
+        pad (Union[int, Tuple[int]]): Padding for top, right, bottom, and left borders.
+            May be specified with 1, 2, or 4 integers (CSS style).
+        style (Union[str, Style], optional): Style for padding characters. Defaults to "none".
+        expand (bool, optional): Expand padding to fit available width. Defaults to True.
+    """
+
+    def __init__(
+        self,
+        renderable: "RenderableType",
+        pad: "PaddingDimensions" = (0, 0, 0, 0),
+        *,
+        style: Union[str, Style] = "none",
+        expand: bool = True,
+    ):
+        self.renderable = renderable
+        self.top, self.right, self.bottom, self.left = self.unpack(pad)
+        self.style = style
+        self.expand = expand
+
+    @classmethod
+    def indent(cls, renderable: "RenderableType", level: int) -> "Padding":
+        """Make padding instance to render an indent.
+
+        Args:
+            renderable (RenderableType): String or other renderable.
+            level (int): Number of characters to indent.
+
+        Returns:
+            Padding: A Padding instance.
+        """
+
+        return Padding(renderable, pad=(0, 0, 0, level), expand=False)
+
+    @staticmethod
+    def unpack(pad: "PaddingDimensions") -> Tuple[int, int, int, int]:
+        """Unpack padding specified in CSS style."""
+        if isinstance(pad, int):
+            return (pad, pad, pad, pad)
+        if len(pad) == 1:
+            _pad = pad[0]
+            return (_pad, _pad, _pad, _pad)
+        if len(pad) == 2:
+            pad_top, pad_right = pad
+            return (pad_top, pad_right, pad_top, pad_right)
+        if len(pad) == 4:
+            top, right, bottom, left = pad
+            return (top, right, bottom, left)
+        raise ValueError(f"1, 2 or 4 integers required for padding; {len(pad)} given")
+
+    def __repr__(self) -> str:
+        return f"Padding({self.renderable!r}, ({self.top},{self.right},{self.bottom},{self.left}))"
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        style = console.get_style(self.style)
+        if self.expand:
+            width = options.max_width
+        else:
+            width = min(
+                Measurement.get(console, options, self.renderable).maximum
+                + self.left
+                + self.right,
+                options.max_width,
+            )
+        render_options = options.update_width(width - self.left - self.right)
+        if render_options.height is not None:
+            render_options = render_options.update_height(
+                height=render_options.height - self.top - self.bottom
+            )
+        lines = console.render_lines(
+            self.renderable, render_options, style=style, pad=True
+        )
+        _Segment = Segment
+
+        left = _Segment(" " * self.left, style) if self.left else None
+        right = (
+            [_Segment(f'{" " * self.right}', style), _Segment.line()]
+            if self.right
+            else [_Segment.line()]
+        )
+        blank_line: Optional[List[Segment]] = None
+        if self.top:
+            blank_line = [_Segment(f'{" " * width}\n', style)]
+            yield from blank_line * self.top
+        if left:
+            for line in lines:
+                yield left
+                yield from line
+                yield from right
+        else:
+            for line in lines:
+                yield from line
+                yield from right
+        if self.bottom:
+            blank_line = blank_line or [_Segment(f'{" " * width}\n', style)]
+            yield from blank_line * self.bottom
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        max_width = options.max_width
+        extra_width = self.left + self.right
+        if max_width - extra_width < 1:
+            return Measurement(max_width, max_width)
+        measure_min, measure_max = Measurement.get(console, options, self.renderable)
+        measurement = Measurement(measure_min + extra_width, measure_max + extra_width)
+        measurement = measurement.with_maximum(max_width)
+        return measurement
+
+
+if __name__ == "__main__":  #  pragma: no cover
+    from rich import print
+
+    print(Padding("Hello, World", (2, 4), style="on blue"))
diff --git a/lib/rich/pager.py b/lib/rich/pager.py
new file mode 100644
index 0000000..a3f7aa6
--- /dev/null
+++ b/lib/rich/pager.py
@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Pager(ABC):
+    """Base class for a pager."""
+
+    @abstractmethod
+    def show(self, content: str) -> None:
+        """Show content in pager.
+
+        Args:
+            content (str): Content to be displayed.
+        """
+
+
+class SystemPager(Pager):
+    """Uses the pager installed on the system."""
+
+    def _pager(self, content: str) -> Any:  #  pragma: no cover
+        return __import__("pydoc").pager(content)
+
+    def show(self, content: str) -> None:
+        """Use the same pager used by pydoc."""
+        self._pager(content)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from .__main__ import make_test_card
+    from .console import Console
+
+    console = Console()
+    with console.pager(styles=True):
+        console.print(make_test_card())
diff --git a/lib/rich/palette.py b/lib/rich/palette.py
new file mode 100644
index 0000000..f295879
--- /dev/null
+++ b/lib/rich/palette.py
@@ -0,0 +1,100 @@
+from math import sqrt
+from functools import lru_cache
+from typing import Sequence, Tuple, TYPE_CHECKING
+
+from .color_triplet import ColorTriplet
+
+if TYPE_CHECKING:
+    from rich.table import Table
+
+
+class Palette:
+    """A palette of available colors."""
+
+    def __init__(self, colors: Sequence[Tuple[int, int, int]]):
+        self._colors = colors
+
+    def __getitem__(self, number: int) -> ColorTriplet:
+        return ColorTriplet(*self._colors[number])
+
+    def __rich__(self) -> "Table":
+        from rich.color import Color
+        from rich.style import Style
+        from rich.text import Text
+        from rich.table import Table
+
+        table = Table(
+            "index",
+            "RGB",
+            "Color",
+            title="Palette",
+            caption=f"{len(self._colors)} colors",
+            highlight=True,
+            caption_justify="right",
+        )
+        for index, color in enumerate(self._colors):
+            table.add_row(
+                str(index),
+                repr(color),
+                Text(" " * 16, style=Style(bgcolor=Color.from_rgb(*color))),
+            )
+        return table
+
+    # This is somewhat inefficient and needs caching
+    @lru_cache(maxsize=1024)
+    def match(self, color: Tuple[int, int, int]) -> int:
+        """Find a color from a palette that most closely matches a given color.
+
+        Args:
+            color (Tuple[int, int, int]): RGB components in range 0 > 255.
+
+        Returns:
+            int: Index of closes matching color.
+        """
+        red1, green1, blue1 = color
+        _sqrt = sqrt
+        get_color = self._colors.__getitem__
+
+        def get_color_distance(index: int) -> float:
+            """Get the distance to a color."""
+            red2, green2, blue2 = get_color(index)
+            red_mean = (red1 + red2) // 2
+            red = red1 - red2
+            green = green1 - green2
+            blue = blue1 - blue2
+            return _sqrt(
+                (((512 + red_mean) * red * red) >> 8)
+                + 4 * green * green
+                + (((767 - red_mean) * blue * blue) >> 8)
+            )
+
+        min_index = min(range(len(self._colors)), key=get_color_distance)
+        return min_index
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import colorsys
+    from typing import Iterable
+    from rich.color import Color
+    from rich.console import Console, ConsoleOptions
+    from rich.segment import Segment
+    from rich.style import Style
+
+    class ColorBox:
+        def __rich_console__(
+            self, console: Console, options: ConsoleOptions
+        ) -> Iterable[Segment]:
+            height = console.size.height - 3
+            for y in range(0, height):
+                for x in range(options.max_width):
+                    h = x / options.max_width
+                    l = y / (height + 1)
+                    r1, g1, b1 = colorsys.hls_to_rgb(h, l, 1.0)
+                    r2, g2, b2 = colorsys.hls_to_rgb(h, l + (1 / height / 2), 1.0)
+                    bgcolor = Color.from_rgb(r1 * 255, g1 * 255, b1 * 255)
+                    color = Color.from_rgb(r2 * 255, g2 * 255, b2 * 255)
+                    yield Segment("▄", Style(color=color, bgcolor=bgcolor))
+                yield Segment.line()
+
+    console = Console()
+    console.print(ColorBox())
diff --git a/lib/rich/panel.py b/lib/rich/panel.py
new file mode 100644
index 0000000..07587e3
--- /dev/null
+++ b/lib/rich/panel.py
@@ -0,0 +1,317 @@
+from typing import TYPE_CHECKING, Optional
+
+from .align import AlignMethod
+from .box import ROUNDED, Box
+from .cells import cell_len
+from .jupyter import JupyterMixin
+from .measure import Measurement, measure_renderables
+from .padding import Padding, PaddingDimensions
+from .segment import Segment
+from .style import Style, StyleType
+from .text import Text, TextType
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderableType, RenderResult
+
+
+class Panel(JupyterMixin):
+    """A console renderable that draws a border around its contents.
+
+    Example:
+        >>> console.print(Panel("Hello, World!"))
+
+    Args:
+        renderable (RenderableType): A console renderable object.
+        box (Box): A Box instance that defines the look of the border (see :ref:`appendix_box`. Defaults to box.ROUNDED.
+        title (Optional[TextType], optional): Optional title displayed in panel header. Defaults to None.
+        title_align (AlignMethod, optional): Alignment of title. Defaults to "center".
+        subtitle (Optional[TextType], optional): Optional subtitle displayed in panel footer. Defaults to None.
+        subtitle_align (AlignMethod, optional): Alignment of subtitle. Defaults to "center".
+        safe_box (bool, optional): Disable box characters that don't display on windows legacy terminal with *raster* fonts. Defaults to True.
+        expand (bool, optional): If True the panel will stretch to fill the console width, otherwise it will be sized to fit the contents. Defaults to True.
+        style (str, optional): The style of the panel (border and contents). Defaults to "none".
+        border_style (str, optional): The style of the border. Defaults to "none".
+        width (Optional[int], optional): Optional width of panel. Defaults to None to auto-detect.
+        height (Optional[int], optional): Optional height of panel. Defaults to None to auto-detect.
+        padding (Optional[PaddingDimensions]): Optional padding around renderable. Defaults to 0.
+        highlight (bool, optional): Enable automatic highlighting of panel title (if str). Defaults to False.
+    """
+
+    def __init__(
+        self,
+        renderable: "RenderableType",
+        box: Box = ROUNDED,
+        *,
+        title: Optional[TextType] = None,
+        title_align: AlignMethod = "center",
+        subtitle: Optional[TextType] = None,
+        subtitle_align: AlignMethod = "center",
+        safe_box: Optional[bool] = None,
+        expand: bool = True,
+        style: StyleType = "none",
+        border_style: StyleType = "none",
+        width: Optional[int] = None,
+        height: Optional[int] = None,
+        padding: PaddingDimensions = (0, 1),
+        highlight: bool = False,
+    ) -> None:
+        self.renderable = renderable
+        self.box = box
+        self.title = title
+        self.title_align: AlignMethod = title_align
+        self.subtitle = subtitle
+        self.subtitle_align = subtitle_align
+        self.safe_box = safe_box
+        self.expand = expand
+        self.style = style
+        self.border_style = border_style
+        self.width = width
+        self.height = height
+        self.padding = padding
+        self.highlight = highlight
+
+    @classmethod
+    def fit(
+        cls,
+        renderable: "RenderableType",
+        box: Box = ROUNDED,
+        *,
+        title: Optional[TextType] = None,
+        title_align: AlignMethod = "center",
+        subtitle: Optional[TextType] = None,
+        subtitle_align: AlignMethod = "center",
+        safe_box: Optional[bool] = None,
+        style: StyleType = "none",
+        border_style: StyleType = "none",
+        width: Optional[int] = None,
+        height: Optional[int] = None,
+        padding: PaddingDimensions = (0, 1),
+        highlight: bool = False,
+    ) -> "Panel":
+        """An alternative constructor that sets expand=False."""
+        return cls(
+            renderable,
+            box,
+            title=title,
+            title_align=title_align,
+            subtitle=subtitle,
+            subtitle_align=subtitle_align,
+            safe_box=safe_box,
+            style=style,
+            border_style=border_style,
+            width=width,
+            height=height,
+            padding=padding,
+            highlight=highlight,
+            expand=False,
+        )
+
+    @property
+    def _title(self) -> Optional[Text]:
+        if self.title:
+            title_text = (
+                Text.from_markup(self.title)
+                if isinstance(self.title, str)
+                else self.title.copy()
+            )
+            title_text.end = ""
+            title_text.plain = title_text.plain.replace("\n", " ")
+            title_text.no_wrap = True
+            title_text.expand_tabs()
+            title_text.pad(1)
+            return title_text
+        return None
+
+    @property
+    def _subtitle(self) -> Optional[Text]:
+        if self.subtitle:
+            subtitle_text = (
+                Text.from_markup(self.subtitle)
+                if isinstance(self.subtitle, str)
+                else self.subtitle.copy()
+            )
+            subtitle_text.end = ""
+            subtitle_text.plain = subtitle_text.plain.replace("\n", " ")
+            subtitle_text.no_wrap = True
+            subtitle_text.expand_tabs()
+            subtitle_text.pad(1)
+            return subtitle_text
+        return None
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        _padding = Padding.unpack(self.padding)
+        renderable = (
+            Padding(self.renderable, _padding) if any(_padding) else self.renderable
+        )
+        style = console.get_style(self.style)
+        border_style = style + console.get_style(self.border_style)
+        width = (
+            options.max_width
+            if self.width is None
+            else min(options.max_width, self.width)
+        )
+
+        safe_box: bool = console.safe_box if self.safe_box is None else self.safe_box
+        box = self.box.substitute(options, safe=safe_box)
+
+        def align_text(
+            text: Text, width: int, align: str, character: str, style: Style
+        ) -> Text:
+            """Gets new aligned text.
+
+            Args:
+                text (Text): Title or subtitle text.
+                width (int): Desired width.
+                align (str): Alignment.
+                character (str): Character for alignment.
+                style (Style): Border style
+
+            Returns:
+                Text: New text instance
+            """
+            text = text.copy()
+            text.truncate(width)
+            excess_space = width - cell_len(text.plain)
+            if text.style:
+                text.stylize(console.get_style(text.style))
+
+            if excess_space:
+                if align == "left":
+                    return Text.assemble(
+                        text,
+                        (character * excess_space, style),
+                        no_wrap=True,
+                        end="",
+                    )
+                elif align == "center":
+                    left = excess_space // 2
+                    return Text.assemble(
+                        (character * left, style),
+                        text,
+                        (character * (excess_space - left), style),
+                        no_wrap=True,
+                        end="",
+                    )
+                else:
+                    return Text.assemble(
+                        (character * excess_space, style),
+                        text,
+                        no_wrap=True,
+                        end="",
+                    )
+            return text
+
+        title_text = self._title
+        if title_text is not None:
+            title_text.stylize_before(border_style)
+
+        child_width = (
+            width - 2
+            if self.expand
+            else console.measure(
+                renderable, options=options.update_width(width - 2)
+            ).maximum
+        )
+        child_height = self.height or options.height or None
+        if child_height:
+            child_height -= 2
+        if title_text is not None:
+            child_width = min(
+                options.max_width - 2, max(child_width, title_text.cell_len + 2)
+            )
+
+        width = child_width + 2
+        child_options = options.update(
+            width=child_width, height=child_height, highlight=self.highlight
+        )
+        lines = console.render_lines(renderable, child_options, style=style)
+
+        line_start = Segment(box.mid_left, border_style)
+        line_end = Segment(f"{box.mid_right}", border_style)
+        new_line = Segment.line()
+        if title_text is None or width <= 4:
+            yield Segment(box.get_top([width - 2]), border_style)
+        else:
+            title_text = align_text(
+                title_text,
+                width - 4,
+                self.title_align,
+                box.top,
+                border_style,
+            )
+            yield Segment(box.top_left + box.top, border_style)
+            yield from console.render(title_text, child_options.update_width(width - 4))
+            yield Segment(box.top + box.top_right, border_style)
+
+        yield new_line
+        for line in lines:
+            yield line_start
+            yield from line
+            yield line_end
+            yield new_line
+
+        subtitle_text = self._subtitle
+        if subtitle_text is not None:
+            subtitle_text.stylize_before(border_style)
+
+        if subtitle_text is None or width <= 4:
+            yield Segment(box.get_bottom([width - 2]), border_style)
+        else:
+            subtitle_text = align_text(
+                subtitle_text,
+                width - 4,
+                self.subtitle_align,
+                box.bottom,
+                border_style,
+            )
+            yield Segment(box.bottom_left + box.bottom, border_style)
+            yield from console.render(
+                subtitle_text, child_options.update_width(width - 4)
+            )
+            yield Segment(box.bottom + box.bottom_right, border_style)
+
+        yield new_line
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        _title = self._title
+        _, right, _, left = Padding.unpack(self.padding)
+        padding = left + right
+        renderables = [self.renderable, _title] if _title else [self.renderable]
+
+        if self.width is None:
+            width = (
+                measure_renderables(
+                    console,
+                    options.update_width(options.max_width - padding - 2),
+                    renderables,
+                ).maximum
+                + padding
+                + 2
+            )
+        else:
+            width = self.width
+        return Measurement(width, width)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from .console import Console
+
+    c = Console()
+
+    from .box import DOUBLE, ROUNDED
+    from .padding import Padding
+
+    p = Panel(
+        "Hello, World!",
+        title="rich.Panel",
+        style="white on blue",
+        box=DOUBLE,
+        padding=1,
+    )
+
+    c.print()
+    c.print(p)
diff --git a/lib/rich/pretty.py b/lib/rich/pretty.py
new file mode 100644
index 0000000..00abeca
--- /dev/null
+++ b/lib/rich/pretty.py
@@ -0,0 +1,1016 @@
+import builtins
+import collections
+import dataclasses
+import inspect
+import os
+import reprlib
+import sys
+from array import array
+from collections import Counter, UserDict, UserList, defaultdict, deque
+from dataclasses import dataclass, fields, is_dataclass
+from inspect import isclass
+from itertools import islice
+from types import MappingProxyType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    DefaultDict,
+    Deque,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Union,
+)
+
+from rich.repr import RichReprResult
+
+try:
+    import attr as _attr_module
+
+    _has_attrs = hasattr(_attr_module, "ib")
+except ImportError:  # pragma: no cover
+    _has_attrs = False
+
+from . import get_console
+from ._loop import loop_last
+from ._pick import pick_bool
+from .abc import RichRenderable
+from .cells import cell_len
+from .highlighter import ReprHighlighter
+from .jupyter import JupyterMixin, JupyterRenderable
+from .measure import Measurement
+from .text import Text
+
+if TYPE_CHECKING:
+    from .console import (
+        Console,
+        ConsoleOptions,
+        HighlighterType,
+        JustifyMethod,
+        OverflowMethod,
+        RenderResult,
+    )
+
+
+def _is_attr_object(obj: Any) -> bool:
+    """Check if an object was created with attrs module."""
+    return _has_attrs and _attr_module.has(type(obj))
+
+
+def _get_attr_fields(obj: Any) -> Sequence["_attr_module.Attribute[Any]"]:
+    """Get fields for an attrs object."""
+    return _attr_module.fields(type(obj)) if _has_attrs else []
+
+
+def _is_dataclass_repr(obj: object) -> bool:
+    """Check if an instance of a dataclass contains the default repr.
+
+    Args:
+        obj (object): A dataclass instance.
+
+    Returns:
+        bool: True if the default repr is used, False if there is a custom repr.
+    """
+    # Digging in to a lot of internals here
+    # Catching all exceptions in case something is missing on a non CPython implementation
+    try:
+        return obj.__repr__.__code__.co_filename in (
+            dataclasses.__file__,
+            reprlib.__file__,
+        )
+    except Exception:  # pragma: no coverage
+        return False
+
+
+_dummy_namedtuple = collections.namedtuple("_dummy_namedtuple", [])
+
+
+def _has_default_namedtuple_repr(obj: object) -> bool:
+    """Check if an instance of namedtuple contains the default repr
+
+    Args:
+        obj (object): A namedtuple
+
+    Returns:
+        bool: True if the default repr is used, False if there's a custom repr.
+    """
+    obj_file = None
+    try:
+        obj_file = inspect.getfile(obj.__repr__)
+    except (OSError, TypeError):
+        # OSError handles case where object is defined in __main__ scope, e.g. REPL - no filename available.
+        # TypeError trapped defensively, in case of object without filename slips through.
+        pass
+    default_repr_file = inspect.getfile(_dummy_namedtuple.__repr__)
+    return obj_file == default_repr_file
+
+
+def _ipy_display_hook(
+    value: Any,
+    console: Optional["Console"] = None,
+    overflow: "OverflowMethod" = "ignore",
+    crop: bool = False,
+    indent_guides: bool = False,
+    max_length: Optional[int] = None,
+    max_string: Optional[int] = None,
+    max_depth: Optional[int] = None,
+    expand_all: bool = False,
+) -> Union[str, None]:
+    # needed here to prevent circular import:
+    from .console import ConsoleRenderable
+
+    # always skip rich generated jupyter renderables or None values
+    if _safe_isinstance(value, JupyterRenderable) or value is None:
+        return None
+
+    console = console or get_console()
+
+    with console.capture() as capture:
+        # certain renderables should start on a new line
+        if _safe_isinstance(value, ConsoleRenderable):
+            console.line()
+        console.print(
+            (
+                value
+                if _safe_isinstance(value, RichRenderable)
+                else Pretty(
+                    value,
+                    overflow=overflow,
+                    indent_guides=indent_guides,
+                    max_length=max_length,
+                    max_string=max_string,
+                    max_depth=max_depth,
+                    expand_all=expand_all,
+                    margin=12,
+                )
+            ),
+            crop=crop,
+            new_line_start=True,
+            end="",
+        )
+    # strip trailing newline, not usually part of a text repr
+    # I'm not sure if this should be prevented at a lower level
+    return capture.get().rstrip("\n")
+
+
+def _safe_isinstance(
+    obj: object, class_or_tuple: Union[type, Tuple[type, ...]]
+) -> bool:
+    """isinstance can fail in rare cases, for example types with no __class__"""
+    try:
+        return isinstance(obj, class_or_tuple)
+    except Exception:
+        return False
+
+
+def install(
+    console: Optional["Console"] = None,
+    overflow: "OverflowMethod" = "ignore",
+    crop: bool = False,
+    indent_guides: bool = False,
+    max_length: Optional[int] = None,
+    max_string: Optional[int] = None,
+    max_depth: Optional[int] = None,
+    expand_all: bool = False,
+) -> None:
+    """Install automatic pretty printing in the Python REPL.
+
+    Args:
+        console (Console, optional): Console instance or ``None`` to use global console. Defaults to None.
+        overflow (Optional[OverflowMethod], optional): Overflow method. Defaults to "ignore".
+        crop (Optional[bool], optional): Enable cropping of long lines. Defaults to False.
+        indent_guides (bool, optional): Enable indentation guides. Defaults to False.
+        max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to None.
+        max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to None.
+        max_depth (int, optional): Maximum depth of nested data structures, or None for no maximum. Defaults to None.
+        expand_all (bool, optional): Expand all containers. Defaults to False.
+        max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100.
+    """
+    from rich import get_console
+
+    console = console or get_console()
+    assert console is not None
+
+    def display_hook(value: Any) -> None:
+        """Replacement sys.displayhook which prettifies objects with Rich."""
+        if value is not None:
+            assert console is not None
+            builtins._ = None  # type: ignore[attr-defined]
+            console.print(
+                (
+                    value
+                    if _safe_isinstance(value, RichRenderable)
+                    else Pretty(
+                        value,
+                        overflow=overflow,
+                        indent_guides=indent_guides,
+                        max_length=max_length,
+                        max_string=max_string,
+                        max_depth=max_depth,
+                        expand_all=expand_all,
+                    )
+                ),
+                crop=crop,
+            )
+            builtins._ = value  # type: ignore[attr-defined]
+
+    try:
+        ip = get_ipython()  # type: ignore[name-defined]
+    except NameError:
+        sys.displayhook = display_hook
+    else:
+        from IPython.core.formatters import BaseFormatter
+
+        class RichFormatter(BaseFormatter):  # type: ignore[misc]
+            pprint: bool = True
+
+            def __call__(self, value: Any) -> Any:
+                if self.pprint:
+                    return _ipy_display_hook(
+                        value,
+                        console=console,
+                        overflow=overflow,
+                        indent_guides=indent_guides,
+                        max_length=max_length,
+                        max_string=max_string,
+                        max_depth=max_depth,
+                        expand_all=expand_all,
+                    )
+                else:
+                    return repr(value)
+
+        # replace plain text formatter with rich formatter
+        rich_formatter = RichFormatter()
+        ip.display_formatter.formatters["text/plain"] = rich_formatter
+
+
+class Pretty(JupyterMixin):
+    """A rich renderable that pretty prints an object.
+
+    Args:
+        _object (Any): An object to pretty print.
+        highlighter (HighlighterType, optional): Highlighter object to apply to result, or None for ReprHighlighter. Defaults to None.
+        indent_size (int, optional): Number of spaces in indent. Defaults to 4.
+        justify (JustifyMethod, optional): Justify method, or None for default. Defaults to None.
+        overflow (OverflowMethod, optional): Overflow method, or None for default. Defaults to None.
+        no_wrap (Optional[bool], optional): Disable word wrapping. Defaults to False.
+        indent_guides (bool, optional): Enable indentation guides. Defaults to False.
+        max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to None.
+        max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to None.
+        max_depth (int, optional): Maximum depth of nested data structures, or None for no maximum. Defaults to None.
+        expand_all (bool, optional): Expand all containers. Defaults to False.
+        margin (int, optional): Subtrace a margin from width to force containers to expand earlier. Defaults to 0.
+        insert_line (bool, optional): Insert a new line if the output has multiple new lines. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        _object: Any,
+        highlighter: Optional["HighlighterType"] = None,
+        *,
+        indent_size: int = 4,
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+        no_wrap: Optional[bool] = False,
+        indent_guides: bool = False,
+        max_length: Optional[int] = None,
+        max_string: Optional[int] = None,
+        max_depth: Optional[int] = None,
+        expand_all: bool = False,
+        margin: int = 0,
+        insert_line: bool = False,
+    ) -> None:
+        self._object = _object
+        self.highlighter = highlighter or ReprHighlighter()
+        self.indent_size = indent_size
+        self.justify: Optional["JustifyMethod"] = justify
+        self.overflow: Optional["OverflowMethod"] = overflow
+        self.no_wrap = no_wrap
+        self.indent_guides = indent_guides
+        self.max_length = max_length
+        self.max_string = max_string
+        self.max_depth = max_depth
+        self.expand_all = expand_all
+        self.margin = margin
+        self.insert_line = insert_line
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        pretty_str = pretty_repr(
+            self._object,
+            max_width=options.max_width - self.margin,
+            indent_size=self.indent_size,
+            max_length=self.max_length,
+            max_string=self.max_string,
+            max_depth=self.max_depth,
+            expand_all=self.expand_all,
+        )
+        pretty_text = Text.from_ansi(
+            pretty_str,
+            justify=self.justify or options.justify,
+            overflow=self.overflow or options.overflow,
+            no_wrap=pick_bool(self.no_wrap, options.no_wrap),
+            style="pretty",
+        )
+        pretty_text = (
+            self.highlighter(pretty_text)
+            if pretty_text
+            else Text(
+                f"{type(self._object)}.__repr__ returned empty string",
+                style="dim italic",
+            )
+        )
+        if self.indent_guides and not options.ascii_only:
+            pretty_text = pretty_text.with_indent_guides(
+                self.indent_size, style="repr.indent"
+            )
+        if self.insert_line and "\n" in pretty_text:
+            yield ""
+        yield pretty_text
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        pretty_str = pretty_repr(
+            self._object,
+            max_width=options.max_width,
+            indent_size=self.indent_size,
+            max_length=self.max_length,
+            max_string=self.max_string,
+            max_depth=self.max_depth,
+            expand_all=self.expand_all,
+        )
+        text_width = (
+            max(cell_len(line) for line in pretty_str.splitlines()) if pretty_str else 0
+        )
+        return Measurement(text_width, text_width)
+
+
+def _get_braces_for_defaultdict(_object: DefaultDict[Any, Any]) -> Tuple[str, str, str]:
+    return (
+        f"defaultdict({_object.default_factory!r}, {{",
+        "})",
+        f"defaultdict({_object.default_factory!r}, {{}})",
+    )
+
+
+def _get_braces_for_deque(_object: Deque[Any]) -> Tuple[str, str, str]:
+    if _object.maxlen is None:
+        return ("deque([", "])", "deque()")
+    return (
+        "deque([",
+        f"], maxlen={_object.maxlen})",
+        f"deque(maxlen={_object.maxlen})",
+    )
+
+
+def _get_braces_for_array(_object: "array[Any]") -> Tuple[str, str, str]:
+    return (f"array({_object.typecode!r}, [", "])", f"array({_object.typecode!r})")
+
+
+_BRACES: Dict[type, Callable[[Any], Tuple[str, str, str]]] = {
+    os._Environ: lambda _object: ("environ({", "})", "environ({})"),
+    array: _get_braces_for_array,
+    defaultdict: _get_braces_for_defaultdict,
+    Counter: lambda _object: ("Counter({", "})", "Counter()"),
+    deque: _get_braces_for_deque,
+    dict: lambda _object: ("{", "}", "{}"),
+    UserDict: lambda _object: ("{", "}", "{}"),
+    frozenset: lambda _object: ("frozenset({", "})", "frozenset()"),
+    list: lambda _object: ("[", "]", "[]"),
+    UserList: lambda _object: ("[", "]", "[]"),
+    set: lambda _object: ("{", "}", "set()"),
+    tuple: lambda _object: ("(", ")", "()"),
+    MappingProxyType: lambda _object: ("mappingproxy({", "})", "mappingproxy({})"),
+}
+_CONTAINERS = tuple(_BRACES.keys())
+_MAPPING_CONTAINERS = (dict, os._Environ, MappingProxyType, UserDict)
+
+
+def is_expandable(obj: Any) -> bool:
+    """Check if an object may be expanded by pretty print."""
+    return (
+        _safe_isinstance(obj, _CONTAINERS)
+        or (is_dataclass(obj))
+        or (hasattr(obj, "__rich_repr__"))
+        or _is_attr_object(obj)
+    ) and not isclass(obj)
+
+
+@dataclass
+class Node:
+    """A node in a repr tree. May be atomic or a container."""
+
+    key_repr: str = ""
+    value_repr: str = ""
+    open_brace: str = ""
+    close_brace: str = ""
+    empty: str = ""
+    last: bool = False
+    is_tuple: bool = False
+    is_namedtuple: bool = False
+    children: Optional[List["Node"]] = None
+    key_separator: str = ": "
+    separator: str = ", "
+
+    def iter_tokens(self) -> Iterable[str]:
+        """Generate tokens for this node."""
+        if self.key_repr:
+            yield self.key_repr
+            yield self.key_separator
+        if self.value_repr:
+            yield self.value_repr
+        elif self.children is not None:
+            if self.children:
+                yield self.open_brace
+                if self.is_tuple and not self.is_namedtuple and len(self.children) == 1:
+                    yield from self.children[0].iter_tokens()
+                    yield ","
+                else:
+                    for child in self.children:
+                        yield from child.iter_tokens()
+                        if not child.last:
+                            yield self.separator
+                yield self.close_brace
+            else:
+                yield self.empty
+
+    def check_length(self, start_length: int, max_length: int) -> bool:
+        """Check the length fits within a limit.
+
+        Args:
+            start_length (int): Starting length of the line (indent, prefix, suffix).
+            max_length (int): Maximum length.
+
+        Returns:
+            bool: True if the node can be rendered within max length, otherwise False.
+        """
+        total_length = start_length
+        for token in self.iter_tokens():
+            total_length += cell_len(token)
+            if total_length > max_length:
+                return False
+        return True
+
+    def __str__(self) -> str:
+        repr_text = "".join(self.iter_tokens())
+        return repr_text
+
+    def render(
+        self, max_width: int = 80, indent_size: int = 4, expand_all: bool = False
+    ) -> str:
+        """Render the node to a pretty repr.
+
+        Args:
+            max_width (int, optional): Maximum width of the repr. Defaults to 80.
+            indent_size (int, optional): Size of indents. Defaults to 4.
+            expand_all (bool, optional): Expand all levels. Defaults to False.
+
+        Returns:
+            str: A repr string of the original object.
+        """
+        lines = [_Line(node=self, is_root=True)]
+        line_no = 0
+        while line_no < len(lines):
+            line = lines[line_no]
+            if line.expandable and not line.expanded:
+                if expand_all or not line.check_length(max_width):
+                    lines[line_no : line_no + 1] = line.expand(indent_size)
+            line_no += 1
+
+        repr_str = "\n".join(str(line) for line in lines)
+        return repr_str
+
+
+@dataclass
+class _Line:
+    """A line in repr output."""
+
+    parent: Optional["_Line"] = None
+    is_root: bool = False
+    node: Optional[Node] = None
+    text: str = ""
+    suffix: str = ""
+    whitespace: str = ""
+    expanded: bool = False
+    last: bool = False
+
+    @property
+    def expandable(self) -> bool:
+        """Check if the line may be expanded."""
+        return bool(self.node is not None and self.node.children)
+
+    def check_length(self, max_length: int) -> bool:
+        """Check this line fits within a given number of cells."""
+        start_length = (
+            len(self.whitespace) + cell_len(self.text) + cell_len(self.suffix)
+        )
+        assert self.node is not None
+        return self.node.check_length(start_length, max_length)
+
+    def expand(self, indent_size: int) -> Iterable["_Line"]:
+        """Expand this line by adding children on their own line."""
+        node = self.node
+        assert node is not None
+        whitespace = self.whitespace
+        assert node.children
+        if node.key_repr:
+            new_line = yield _Line(
+                text=f"{node.key_repr}{node.key_separator}{node.open_brace}",
+                whitespace=whitespace,
+            )
+        else:
+            new_line = yield _Line(text=node.open_brace, whitespace=whitespace)
+        child_whitespace = self.whitespace + " " * indent_size
+        tuple_of_one = node.is_tuple and len(node.children) == 1
+        for last, child in loop_last(node.children):
+            separator = "," if tuple_of_one else node.separator
+            line = _Line(
+                parent=new_line,
+                node=child,
+                whitespace=child_whitespace,
+                suffix=separator,
+                last=last and not tuple_of_one,
+            )
+            yield line
+
+        yield _Line(
+            text=node.close_brace,
+            whitespace=whitespace,
+            suffix=self.suffix,
+            last=self.last,
+        )
+
+    def __str__(self) -> str:
+        if self.last:
+            return f"{self.whitespace}{self.text}{self.node or ''}"
+        else:
+            return (
+                f"{self.whitespace}{self.text}{self.node or ''}{self.suffix.rstrip()}"
+            )
+
+
+def _is_namedtuple(obj: Any) -> bool:
+    """Checks if an object is most likely a namedtuple. It is possible
+    to craft an object that passes this check and isn't a namedtuple, but
+    there is only a minuscule chance of this happening unintentionally.
+
+    Args:
+        obj (Any): The object to test
+
+    Returns:
+        bool: True if the object is a namedtuple. False otherwise.
+    """
+    try:
+        fields = getattr(obj, "_fields", None)
+    except Exception:
+        # Being very defensive - if we cannot get the attr then its not a namedtuple
+        return False
+    return isinstance(obj, tuple) and isinstance(fields, tuple)
+
+
+def traverse(
+    _object: Any,
+    max_length: Optional[int] = None,
+    max_string: Optional[int] = None,
+    max_depth: Optional[int] = None,
+) -> Node:
+    """Traverse object and generate a tree.
+
+    Args:
+        _object (Any): Object to be traversed.
+        max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to None.
+        max_string (int, optional): Maximum length of string before truncating, or None to disable truncating.
+            Defaults to None.
+        max_depth (int, optional): Maximum depth of data structures, or None for no maximum.
+            Defaults to None.
+
+    Returns:
+        Node: The root of a tree structure which can be used to render a pretty repr.
+    """
+
+    def to_repr(obj: Any) -> str:
+        """Get repr string for an object, but catch errors."""
+        if (
+            max_string is not None
+            and _safe_isinstance(obj, (bytes, str))
+            and len(obj) > max_string
+        ):
+            truncated = len(obj) - max_string
+            obj_repr = f"{obj[:max_string]!r}+{truncated}"
+        else:
+            try:
+                obj_repr = repr(obj)
+            except Exception as error:
+                obj_repr = f""
+        return obj_repr
+
+    visited_ids: Set[int] = set()
+    push_visited = visited_ids.add
+    pop_visited = visited_ids.remove
+
+    def _traverse(obj: Any, root: bool = False, depth: int = 0) -> Node:
+        """Walk the object depth first."""
+
+        obj_id = id(obj)
+        if obj_id in visited_ids:
+            # Recursion detected
+            return Node(value_repr="...")
+
+        obj_type = type(obj)
+        children: List[Node]
+        reached_max_depth = max_depth is not None and depth >= max_depth
+
+        def iter_rich_args(rich_args: Any) -> Iterable[Union[Any, Tuple[str, Any]]]:
+            for arg in rich_args:
+                if _safe_isinstance(arg, tuple):
+                    if len(arg) == 3:
+                        key, child, default = arg
+                        if default == child:
+                            continue
+                        yield key, child
+                    elif len(arg) == 2:
+                        key, child = arg
+                        yield key, child
+                    elif len(arg) == 1:
+                        yield arg[0]
+                else:
+                    yield arg
+
+        try:
+            fake_attributes = hasattr(
+                obj, "awehoi234_wdfjwljet234_234wdfoijsdfmmnxpi492"
+            )
+        except Exception:
+            fake_attributes = False
+
+        rich_repr_result: Optional[RichReprResult] = None
+        if not fake_attributes:
+            try:
+                if hasattr(obj, "__rich_repr__") and not isclass(obj):
+                    rich_repr_result = obj.__rich_repr__()
+            except Exception:
+                pass
+
+        if rich_repr_result is not None:
+            push_visited(obj_id)
+            angular = getattr(obj.__rich_repr__, "angular", False)
+            args = list(iter_rich_args(rich_repr_result))
+            class_name = obj.__class__.__name__
+
+            if args:
+                children = []
+                append = children.append
+
+                if reached_max_depth:
+                    if angular:
+                        node = Node(value_repr=f"<{class_name}...>")
+                    else:
+                        node = Node(value_repr=f"{class_name}(...)")
+                else:
+                    if angular:
+                        node = Node(
+                            open_brace=f"<{class_name} ",
+                            close_brace=">",
+                            children=children,
+                            last=root,
+                            separator=" ",
+                        )
+                    else:
+                        node = Node(
+                            open_brace=f"{class_name}(",
+                            close_brace=")",
+                            children=children,
+                            last=root,
+                        )
+                    for last, arg in loop_last(args):
+                        if _safe_isinstance(arg, tuple):
+                            key, child = arg
+                            child_node = _traverse(child, depth=depth + 1)
+                            child_node.last = last
+                            child_node.key_repr = key
+                            child_node.key_separator = "="
+                            append(child_node)
+                        else:
+                            child_node = _traverse(arg, depth=depth + 1)
+                            child_node.last = last
+                            append(child_node)
+            else:
+                node = Node(
+                    value_repr=f"<{class_name}>" if angular else f"{class_name}()",
+                    children=[],
+                    last=root,
+                )
+            pop_visited(obj_id)
+        elif _is_attr_object(obj) and not fake_attributes:
+            push_visited(obj_id)
+            children = []
+            append = children.append
+
+            attr_fields = _get_attr_fields(obj)
+            if attr_fields:
+                if reached_max_depth:
+                    node = Node(value_repr=f"{obj.__class__.__name__}(...)")
+                else:
+                    node = Node(
+                        open_brace=f"{obj.__class__.__name__}(",
+                        close_brace=")",
+                        children=children,
+                        last=root,
+                    )
+
+                    def iter_attrs() -> (
+                        Iterable[Tuple[str, Any, Optional[Callable[[Any], str]]]]
+                    ):
+                        """Iterate over attr fields and values."""
+                        for attr in attr_fields:
+                            if attr.repr:
+                                try:
+                                    value = getattr(obj, attr.name)
+                                except Exception as error:
+                                    # Can happen, albeit rarely
+                                    yield (attr.name, error, None)
+                                else:
+                                    yield (
+                                        attr.name,
+                                        value,
+                                        attr.repr if callable(attr.repr) else None,
+                                    )
+
+                    for last, (name, value, repr_callable) in loop_last(iter_attrs()):
+                        if repr_callable:
+                            child_node = Node(value_repr=str(repr_callable(value)))
+                        else:
+                            child_node = _traverse(value, depth=depth + 1)
+                        child_node.last = last
+                        child_node.key_repr = name
+                        child_node.key_separator = "="
+                        append(child_node)
+            else:
+                node = Node(
+                    value_repr=f"{obj.__class__.__name__}()", children=[], last=root
+                )
+            pop_visited(obj_id)
+        elif (
+            is_dataclass(obj)
+            and not _safe_isinstance(obj, type)
+            and not fake_attributes
+            and _is_dataclass_repr(obj)
+        ):
+            push_visited(obj_id)
+            children = []
+            append = children.append
+            if reached_max_depth:
+                node = Node(value_repr=f"{obj.__class__.__name__}(...)")
+            else:
+                node = Node(
+                    open_brace=f"{obj.__class__.__name__}(",
+                    close_brace=")",
+                    children=children,
+                    last=root,
+                    empty=f"{obj.__class__.__name__}()",
+                )
+
+                for last, field in loop_last(
+                    field
+                    for field in fields(obj)
+                    if field.repr and hasattr(obj, field.name)
+                ):
+                    child_node = _traverse(getattr(obj, field.name), depth=depth + 1)
+                    child_node.key_repr = field.name
+                    child_node.last = last
+                    child_node.key_separator = "="
+                    append(child_node)
+
+            pop_visited(obj_id)
+        elif _is_namedtuple(obj) and _has_default_namedtuple_repr(obj):
+            push_visited(obj_id)
+            class_name = obj.__class__.__name__
+            if reached_max_depth:
+                # If we've reached the max depth, we still show the class name, but not its contents
+                node = Node(
+                    value_repr=f"{class_name}(...)",
+                )
+            else:
+                children = []
+                append = children.append
+                node = Node(
+                    open_brace=f"{class_name}(",
+                    close_brace=")",
+                    children=children,
+                    empty=f"{class_name}()",
+                )
+                for last, (key, value) in loop_last(obj._asdict().items()):
+                    child_node = _traverse(value, depth=depth + 1)
+                    child_node.key_repr = key
+                    child_node.last = last
+                    child_node.key_separator = "="
+                    append(child_node)
+            pop_visited(obj_id)
+        elif _safe_isinstance(obj, _CONTAINERS):
+            for container_type in _CONTAINERS:
+                if _safe_isinstance(obj, container_type):
+                    obj_type = container_type
+                    break
+
+            push_visited(obj_id)
+
+            open_brace, close_brace, empty = _BRACES[obj_type](obj)
+
+            if reached_max_depth:
+                node = Node(value_repr=f"{open_brace}...{close_brace}")
+            elif obj_type.__repr__ != type(obj).__repr__:
+                node = Node(value_repr=to_repr(obj), last=root)
+            elif obj:
+                children = []
+                node = Node(
+                    open_brace=open_brace,
+                    close_brace=close_brace,
+                    children=children,
+                    last=root,
+                )
+                append = children.append
+                num_items = len(obj)
+                last_item_index = num_items - 1
+
+                if _safe_isinstance(obj, _MAPPING_CONTAINERS):
+                    iter_items = iter(obj.items())
+                    if max_length is not None:
+                        iter_items = islice(iter_items, max_length)
+                    for index, (key, child) in enumerate(iter_items):
+                        child_node = _traverse(child, depth=depth + 1)
+                        child_node.key_repr = to_repr(key)
+                        child_node.last = index == last_item_index
+                        append(child_node)
+                else:
+                    iter_values = iter(obj)
+                    if max_length is not None:
+                        iter_values = islice(iter_values, max_length)
+                    for index, child in enumerate(iter_values):
+                        child_node = _traverse(child, depth=depth + 1)
+                        child_node.last = index == last_item_index
+                        append(child_node)
+                if max_length is not None and num_items > max_length:
+                    append(Node(value_repr=f"... +{num_items - max_length}", last=True))
+            else:
+                node = Node(empty=empty, children=[], last=root)
+
+            pop_visited(obj_id)
+        else:
+            node = Node(value_repr=to_repr(obj), last=root)
+        node.is_tuple = type(obj) == tuple
+        node.is_namedtuple = _is_namedtuple(obj)
+        return node
+
+    node = _traverse(_object, root=True)
+    return node
+
+
+def pretty_repr(
+    _object: Any,
+    *,
+    max_width: int = 80,
+    indent_size: int = 4,
+    max_length: Optional[int] = None,
+    max_string: Optional[int] = None,
+    max_depth: Optional[int] = None,
+    expand_all: bool = False,
+) -> str:
+    """Prettify repr string by expanding on to new lines to fit within a given width.
+
+    Args:
+        _object (Any): Object to repr.
+        max_width (int, optional): Desired maximum width of repr string. Defaults to 80.
+        indent_size (int, optional): Number of spaces to indent. Defaults to 4.
+        max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to None.
+        max_string (int, optional): Maximum length of string before truncating, or None to disable truncating.
+            Defaults to None.
+        max_depth (int, optional): Maximum depth of nested data structure, or None for no depth.
+            Defaults to None.
+        expand_all (bool, optional): Expand all containers regardless of available width. Defaults to False.
+
+    Returns:
+        str: A possibly multi-line representation of the object.
+    """
+
+    if _safe_isinstance(_object, Node):
+        node = _object
+    else:
+        node = traverse(
+            _object, max_length=max_length, max_string=max_string, max_depth=max_depth
+        )
+    repr_str: str = node.render(
+        max_width=max_width, indent_size=indent_size, expand_all=expand_all
+    )
+    return repr_str
+
+
+def pprint(
+    _object: Any,
+    *,
+    console: Optional["Console"] = None,
+    indent_guides: bool = True,
+    max_length: Optional[int] = None,
+    max_string: Optional[int] = None,
+    max_depth: Optional[int] = None,
+    expand_all: bool = False,
+) -> None:
+    """A convenience function for pretty printing.
+
+    Args:
+        _object (Any): Object to pretty print.
+        console (Console, optional): Console instance, or None to use default. Defaults to None.
+        max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to None.
+        max_string (int, optional): Maximum length of strings before truncating, or None to disable. Defaults to None.
+        max_depth (int, optional): Maximum depth for nested data structures, or None for unlimited depth. Defaults to None.
+        indent_guides (bool, optional): Enable indentation guides. Defaults to True.
+        expand_all (bool, optional): Expand all containers. Defaults to False.
+    """
+    _console = get_console() if console is None else console
+    _console.print(
+        Pretty(
+            _object,
+            max_length=max_length,
+            max_string=max_string,
+            max_depth=max_depth,
+            indent_guides=indent_guides,
+            expand_all=expand_all,
+            overflow="ignore",
+        ),
+        soft_wrap=True,
+    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+
+    class BrokenRepr:
+        def __repr__(self) -> str:
+            1 / 0
+            return "this will fail"
+
+    from typing import NamedTuple
+
+    class StockKeepingUnit(NamedTuple):
+        name: str
+        description: str
+        price: float
+        category: str
+        reviews: List[str]
+
+    d = defaultdict(int)
+    d["foo"] = 5
+    data = {
+        "foo": [
+            1,
+            "Hello World!",
+            100.123,
+            323.232,
+            432324.0,
+            {5, 6, 7, (1, 2, 3, 4), 8},
+        ],
+        "bar": frozenset({1, 2, 3}),
+        "defaultdict": defaultdict(
+            list, {"crumble": ["apple", "rhubarb", "butter", "sugar", "flour"]}
+        ),
+        "counter": Counter(
+            [
+                "apple",
+                "orange",
+                "pear",
+                "kumquat",
+                "kumquat",
+                "durian" * 100,
+            ]
+        ),
+        "atomic": (False, True, None),
+        "namedtuple": StockKeepingUnit(
+            "Sparkling British Spring Water",
+            "Carbonated spring water",
+            0.9,
+            "water",
+            ["its amazing!", "its terrible!"],
+        ),
+        "Broken": BrokenRepr(),
+    }
+    data["foo"].append(data)  # type: ignore[attr-defined]
+
+    from rich import print
+
+    print(Pretty(data, indent_guides=True, max_string=20))
+
+    class Thing:
+        def __repr__(self) -> str:
+            return "Hello\x1b[38;5;239m World!"
+
+    print(Pretty(Thing()))
diff --git a/lib/rich/progress.py b/lib/rich/progress.py
new file mode 100644
index 0000000..c2de125
--- /dev/null
+++ b/lib/rich/progress.py
@@ -0,0 +1,1716 @@
+from __future__ import annotations
+
+import io
+import typing
+import warnings
+from abc import ABC, abstractmethod
+from collections import deque
+from dataclasses import dataclass, field
+from datetime import timedelta
+from io import RawIOBase, UnsupportedOperation
+from math import ceil
+from mmap import mmap
+from operator import length_hint
+from os import PathLike, stat
+from threading import Event, RLock, Thread
+from types import TracebackType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    BinaryIO,
+    Callable,
+    ContextManager,
+    Deque,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Literal,
+    NamedTuple,
+    NewType,
+    Optional,
+    TextIO,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
+
+if TYPE_CHECKING:
+    # Can be replaced with `from typing import Self` in Python 3.11+
+    from typing_extensions import Self  # pragma: no cover
+
+from . import filesize, get_console
+from .console import Console, Group, JustifyMethod, RenderableType
+from .highlighter import Highlighter
+from .jupyter import JupyterMixin
+from .live import Live
+from .progress_bar import ProgressBar
+from .spinner import Spinner
+from .style import StyleType
+from .table import Column, Table
+from .text import Text, TextType
+
+TaskID = NewType("TaskID", int)
+
+ProgressType = TypeVar("ProgressType")
+
+GetTimeCallable = Callable[[], float]
+
+
+_I = typing.TypeVar("_I", TextIO, BinaryIO)
+
+
+class _TrackThread(Thread):
+    """A thread to periodically update progress."""
+
+    def __init__(self, progress: "Progress", task_id: "TaskID", update_period: float):
+        self.progress = progress
+        self.task_id = task_id
+        self.update_period = update_period
+        self.done = Event()
+
+        self.completed = 0
+        super().__init__(daemon=True)
+
+    def run(self) -> None:
+        task_id = self.task_id
+        advance = self.progress.advance
+        update_period = self.update_period
+        last_completed = 0
+        wait = self.done.wait
+        while not wait(update_period) and self.progress.live.is_started:
+            completed = self.completed
+            if last_completed != completed:
+                advance(task_id, completed - last_completed)
+                last_completed = completed
+
+        self.progress.update(self.task_id, completed=self.completed, refresh=True)
+
+    def __enter__(self) -> "_TrackThread":
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.done.set()
+        self.join()
+
+
+def track(
+    sequence: Iterable[ProgressType],
+    description: str = "Working...",
+    total: Optional[float] = None,
+    completed: int = 0,
+    auto_refresh: bool = True,
+    console: Optional[Console] = None,
+    transient: bool = False,
+    get_time: Optional[Callable[[], float]] = None,
+    refresh_per_second: float = 10,
+    style: StyleType = "bar.back",
+    complete_style: StyleType = "bar.complete",
+    finished_style: StyleType = "bar.finished",
+    pulse_style: StyleType = "bar.pulse",
+    update_period: float = 0.1,
+    disable: bool = False,
+    show_speed: bool = True,
+) -> Iterable[ProgressType]:
+    """Track progress by iterating over a sequence.
+
+    You can also track progress of an iterable, which might require that you additionally specify ``total``.
+
+    Args:
+        sequence (Iterable[ProgressType]): Values you wish to iterate over and track progress.
+        description (str, optional): Description of task show next to progress bar. Defaults to "Working".
+        total: (float, optional): Total number of steps. Default is len(sequence).
+        completed (int, optional): Number of steps completed so far. Defaults to 0.
+        auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True.
+        transient: (bool, optional): Clear the progress on exit. Defaults to False.
+        console (Console, optional): Console to write to. Default creates internal Console instance.
+        refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10.
+        style (StyleType, optional): Style for the bar background. Defaults to "bar.back".
+        complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete".
+        finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.finished".
+        pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse".
+        update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1.
+        disable (bool, optional): Disable display of progress.
+        show_speed (bool, optional): Show speed if total isn't known. Defaults to True.
+    Returns:
+        Iterable[ProgressType]: An iterable of the values in the sequence.
+
+    """
+
+    columns: List["ProgressColumn"] = (
+        [TextColumn("[progress.description]{task.description}")] if description else []
+    )
+    columns.extend(
+        (
+            BarColumn(
+                style=style,
+                complete_style=complete_style,
+                finished_style=finished_style,
+                pulse_style=pulse_style,
+            ),
+            TaskProgressColumn(show_speed=show_speed),
+            TimeRemainingColumn(elapsed_when_finished=True),
+        )
+    )
+    progress = Progress(
+        *columns,
+        auto_refresh=auto_refresh,
+        console=console,
+        transient=transient,
+        get_time=get_time,
+        refresh_per_second=refresh_per_second or 10,
+        disable=disable,
+    )
+
+    with progress:
+        yield from progress.track(
+            sequence,
+            total=total,
+            completed=completed,
+            description=description,
+            update_period=update_period,
+        )
+
+
+class _Reader(RawIOBase, BinaryIO):
+    """A reader that tracks progress while it's being read from."""
+
+    def __init__(
+        self,
+        handle: BinaryIO,
+        progress: "Progress",
+        task: TaskID,
+        close_handle: bool = True,
+    ) -> None:
+        self.handle = handle
+        self.progress = progress
+        self.task = task
+        self.close_handle = close_handle
+        self._closed = False
+
+    def __enter__(self) -> "_Reader":
+        self.handle.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.close()
+
+    def __iter__(self) -> BinaryIO:
+        return self
+
+    def __next__(self) -> bytes:
+        line = next(self.handle)
+        self.progress.advance(self.task, advance=len(line))
+        return line
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    def fileno(self) -> int:
+        return self.handle.fileno()
+
+    def isatty(self) -> bool:
+        return self.handle.isatty()
+
+    @property
+    def mode(self) -> str:
+        return self.handle.mode
+
+    @property
+    def name(self) -> str:
+        return self.handle.name
+
+    def readable(self) -> bool:
+        return self.handle.readable()
+
+    def seekable(self) -> bool:
+        return self.handle.seekable()
+
+    def writable(self) -> bool:
+        return False
+
+    def read(self, size: int = -1) -> bytes:
+        block = self.handle.read(size)
+        self.progress.advance(self.task, advance=len(block))
+        return block
+
+    def readinto(self, b: Union[bytearray, memoryview, mmap]):  # type: ignore[no-untyped-def, override]
+        n = self.handle.readinto(b)  # type: ignore[attr-defined]
+        self.progress.advance(self.task, advance=n)
+        return n
+
+    def readline(self, size: int = -1) -> bytes:  # type: ignore[override]
+        line = self.handle.readline(size)
+        self.progress.advance(self.task, advance=len(line))
+        return line
+
+    def readlines(self, hint: int = -1) -> List[bytes]:
+        lines = self.handle.readlines(hint)
+        self.progress.advance(self.task, advance=sum(map(len, lines)))
+        return lines
+
+    def close(self) -> None:
+        if self.close_handle:
+            self.handle.close()
+        self._closed = True
+
+    def seek(self, offset: int, whence: int = 0) -> int:
+        pos = self.handle.seek(offset, whence)
+        self.progress.update(self.task, completed=pos)
+        return pos
+
+    def tell(self) -> int:
+        return self.handle.tell()
+
+    def write(self, s: Any) -> int:
+        raise UnsupportedOperation("write")
+
+    def writelines(self, lines: Iterable[Any]) -> None:
+        raise UnsupportedOperation("writelines")
+
+
+class _ReadContext(ContextManager[_I], Generic[_I]):
+    """A utility class to handle a context for both a reader and a progress."""
+
+    def __init__(self, progress: "Progress", reader: _I) -> None:
+        self.progress = progress
+        self.reader: _I = reader
+
+    def __enter__(self) -> _I:
+        self.progress.start()
+        return self.reader.__enter__()
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.progress.stop()
+        self.reader.__exit__(exc_type, exc_val, exc_tb)
+
+
+def wrap_file(
+    file: BinaryIO,
+    total: int,
+    *,
+    description: str = "Reading...",
+    auto_refresh: bool = True,
+    console: Optional[Console] = None,
+    transient: bool = False,
+    get_time: Optional[Callable[[], float]] = None,
+    refresh_per_second: float = 10,
+    style: StyleType = "bar.back",
+    complete_style: StyleType = "bar.complete",
+    finished_style: StyleType = "bar.finished",
+    pulse_style: StyleType = "bar.pulse",
+    disable: bool = False,
+) -> ContextManager[BinaryIO]:
+    """Read bytes from a file while tracking progress.
+
+    Args:
+        file (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode.
+        total (int): Total number of bytes to read.
+        description (str, optional): Description of task show next to progress bar. Defaults to "Reading".
+        auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True.
+        transient: (bool, optional): Clear the progress on exit. Defaults to False.
+        console (Console, optional): Console to write to. Default creates internal Console instance.
+        refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10.
+        style (StyleType, optional): Style for the bar background. Defaults to "bar.back".
+        complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete".
+        finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.finished".
+        pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse".
+        disable (bool, optional): Disable display of progress.
+    Returns:
+        ContextManager[BinaryIO]: A context manager yielding a progress reader.
+
+    """
+
+    columns: List["ProgressColumn"] = (
+        [TextColumn("[progress.description]{task.description}")] if description else []
+    )
+    columns.extend(
+        (
+            BarColumn(
+                style=style,
+                complete_style=complete_style,
+                finished_style=finished_style,
+                pulse_style=pulse_style,
+            ),
+            DownloadColumn(),
+            TimeRemainingColumn(),
+        )
+    )
+    progress = Progress(
+        *columns,
+        auto_refresh=auto_refresh,
+        console=console,
+        transient=transient,
+        get_time=get_time,
+        refresh_per_second=refresh_per_second or 10,
+        disable=disable,
+    )
+
+    reader = progress.wrap_file(file, total=total, description=description)
+    return _ReadContext(progress, reader)
+
+
+@typing.overload
+def open(
+    file: Union[str, "PathLike[str]", bytes],
+    mode: Union[Literal["rt"], Literal["r"]],
+    buffering: int = -1,
+    encoding: Optional[str] = None,
+    errors: Optional[str] = None,
+    newline: Optional[str] = None,
+    *,
+    total: Optional[int] = None,
+    description: str = "Reading...",
+    auto_refresh: bool = True,
+    console: Optional[Console] = None,
+    transient: bool = False,
+    get_time: Optional[Callable[[], float]] = None,
+    refresh_per_second: float = 10,
+    style: StyleType = "bar.back",
+    complete_style: StyleType = "bar.complete",
+    finished_style: StyleType = "bar.finished",
+    pulse_style: StyleType = "bar.pulse",
+    disable: bool = False,
+) -> ContextManager[TextIO]:
+    pass
+
+
+@typing.overload
+def open(
+    file: Union[str, "PathLike[str]", bytes],
+    mode: Literal["rb"],
+    buffering: int = -1,
+    encoding: Optional[str] = None,
+    errors: Optional[str] = None,
+    newline: Optional[str] = None,
+    *,
+    total: Optional[int] = None,
+    description: str = "Reading...",
+    auto_refresh: bool = True,
+    console: Optional[Console] = None,
+    transient: bool = False,
+    get_time: Optional[Callable[[], float]] = None,
+    refresh_per_second: float = 10,
+    style: StyleType = "bar.back",
+    complete_style: StyleType = "bar.complete",
+    finished_style: StyleType = "bar.finished",
+    pulse_style: StyleType = "bar.pulse",
+    disable: bool = False,
+) -> ContextManager[BinaryIO]:
+    pass
+
+
+def open(
+    file: Union[str, "PathLike[str]", bytes],
+    mode: Union[Literal["rb"], Literal["rt"], Literal["r"]] = "r",
+    buffering: int = -1,
+    encoding: Optional[str] = None,
+    errors: Optional[str] = None,
+    newline: Optional[str] = None,
+    *,
+    total: Optional[int] = None,
+    description: str = "Reading...",
+    auto_refresh: bool = True,
+    console: Optional[Console] = None,
+    transient: bool = False,
+    get_time: Optional[Callable[[], float]] = None,
+    refresh_per_second: float = 10,
+    style: StyleType = "bar.back",
+    complete_style: StyleType = "bar.complete",
+    finished_style: StyleType = "bar.finished",
+    pulse_style: StyleType = "bar.pulse",
+    disable: bool = False,
+) -> Union[ContextManager[BinaryIO], ContextManager[TextIO]]:
+    """Read bytes from a file while tracking progress.
+
+    Args:
+        path (Union[str, PathLike[str], BinaryIO]): The path to the file to read, or a file-like object in binary mode.
+        mode (str): The mode to use to open the file. Only supports "r", "rb" or "rt".
+        buffering (int): The buffering strategy to use, see :func:`io.open`.
+        encoding (str, optional): The encoding to use when reading in text mode, see :func:`io.open`.
+        errors (str, optional): The error handling strategy for decoding errors, see :func:`io.open`.
+        newline (str, optional): The strategy for handling newlines in text mode, see :func:`io.open`
+        total: (int, optional): Total number of bytes to read. Must be provided if reading from a file handle. Default for a path is os.stat(file).st_size.
+        description (str, optional): Description of task show next to progress bar. Defaults to "Reading".
+        auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True.
+        transient: (bool, optional): Clear the progress on exit. Defaults to False.
+        console (Console, optional): Console to write to. Default creates internal Console instance.
+        refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10.
+        style (StyleType, optional): Style for the bar background. Defaults to "bar.back".
+        complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete".
+        finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.finished".
+        pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse".
+        disable (bool, optional): Disable display of progress.
+        encoding (str, optional): The encoding to use when reading in text mode.
+
+    Returns:
+        ContextManager[BinaryIO]: A context manager yielding a progress reader.
+
+    """
+
+    columns: List["ProgressColumn"] = (
+        [TextColumn("[progress.description]{task.description}")] if description else []
+    )
+    columns.extend(
+        (
+            BarColumn(
+                style=style,
+                complete_style=complete_style,
+                finished_style=finished_style,
+                pulse_style=pulse_style,
+            ),
+            DownloadColumn(),
+            TimeRemainingColumn(),
+        )
+    )
+    progress = Progress(
+        *columns,
+        auto_refresh=auto_refresh,
+        console=console,
+        transient=transient,
+        get_time=get_time,
+        refresh_per_second=refresh_per_second or 10,
+        disable=disable,
+    )
+
+    reader = progress.open(
+        file,
+        mode=mode,
+        buffering=buffering,
+        encoding=encoding,
+        errors=errors,
+        newline=newline,
+        total=total,
+        description=description,
+    )
+    return _ReadContext(progress, reader)  # type: ignore[return-value, type-var]
+
+
+class ProgressColumn(ABC):
+    """Base class for a widget to use in progress display."""
+
+    max_refresh: Optional[float] = None
+
+    def __init__(self, table_column: Optional[Column] = None) -> None:
+        self._table_column = table_column
+        self._renderable_cache: Dict[TaskID, Tuple[float, RenderableType]] = {}
+        self._update_time: Optional[float] = None
+
+    def get_table_column(self) -> Column:
+        """Get a table column, used to build tasks table."""
+        return self._table_column or Column()
+
+    def __call__(self, task: "Task") -> RenderableType:
+        """Called by the Progress object to return a renderable for the given task.
+
+        Args:
+            task (Task): An object containing information regarding the task.
+
+        Returns:
+            RenderableType: Anything renderable (including str).
+        """
+        current_time = task.get_time()
+        if self.max_refresh is not None and not task.completed:
+            try:
+                timestamp, renderable = self._renderable_cache[task.id]
+            except KeyError:
+                pass
+            else:
+                if timestamp + self.max_refresh > current_time:
+                    return renderable
+
+        renderable = self.render(task)
+        self._renderable_cache[task.id] = (current_time, renderable)
+        return renderable
+
+    @abstractmethod
+    def render(self, task: "Task") -> RenderableType:
+        """Should return a renderable object."""
+
+
+class RenderableColumn(ProgressColumn):
+    """A column to insert an arbitrary column.
+
+    Args:
+        renderable (RenderableType, optional): Any renderable. Defaults to empty string.
+    """
+
+    def __init__(
+        self, renderable: RenderableType = "", *, table_column: Optional[Column] = None
+    ):
+        self.renderable = renderable
+        super().__init__(table_column=table_column)
+
+    def render(self, task: "Task") -> RenderableType:
+        return self.renderable
+
+
+class SpinnerColumn(ProgressColumn):
+    """A column with a 'spinner' animation.
+
+    Args:
+        spinner_name (str, optional): Name of spinner animation. Defaults to "dots".
+        style (StyleType, optional): Style of spinner. Defaults to "progress.spinner".
+        speed (float, optional): Speed factor of spinner. Defaults to 1.0.
+        finished_text (TextType, optional): Text used when task is finished. Defaults to " ".
+    """
+
+    def __init__(
+        self,
+        spinner_name: str = "dots",
+        style: Optional[StyleType] = "progress.spinner",
+        speed: float = 1.0,
+        finished_text: TextType = " ",
+        table_column: Optional[Column] = None,
+    ):
+        self.spinner = Spinner(spinner_name, style=style, speed=speed)
+        self.finished_text = (
+            Text.from_markup(finished_text)
+            if isinstance(finished_text, str)
+            else finished_text
+        )
+        super().__init__(table_column=table_column)
+
+    def set_spinner(
+        self,
+        spinner_name: str,
+        spinner_style: Optional[StyleType] = "progress.spinner",
+        speed: float = 1.0,
+    ) -> None:
+        """Set a new spinner.
+
+        Args:
+            spinner_name (str): Spinner name, see python -m rich.spinner.
+            spinner_style (Optional[StyleType], optional): Spinner style. Defaults to "progress.spinner".
+            speed (float, optional): Speed factor of spinner. Defaults to 1.0.
+        """
+        self.spinner = Spinner(spinner_name, style=spinner_style, speed=speed)
+
+    def render(self, task: "Task") -> RenderableType:
+        text = (
+            self.finished_text
+            if task.finished
+            else self.spinner.render(task.get_time())
+        )
+        return text
+
+
+class TextColumn(ProgressColumn):
+    """A column containing text."""
+
+    def __init__(
+        self,
+        text_format: str,
+        style: StyleType = "none",
+        justify: JustifyMethod = "left",
+        markup: bool = True,
+        highlighter: Optional[Highlighter] = None,
+        table_column: Optional[Column] = None,
+    ) -> None:
+        self.text_format = text_format
+        self.justify: JustifyMethod = justify
+        self.style = style
+        self.markup = markup
+        self.highlighter = highlighter
+        super().__init__(table_column=table_column or Column(no_wrap=True))
+
+    def render(self, task: "Task") -> Text:
+        _text = self.text_format.format(task=task)
+        if self.markup:
+            text = Text.from_markup(_text, style=self.style, justify=self.justify)
+        else:
+            text = Text(_text, style=self.style, justify=self.justify)
+        if self.highlighter:
+            self.highlighter.highlight(text)
+        return text
+
+
+class BarColumn(ProgressColumn):
+    """Renders a visual progress bar.
+
+    Args:
+        bar_width (Optional[int], optional): Width of bar or None for full width. Defaults to 40.
+        style (StyleType, optional): Style for the bar background. Defaults to "bar.back".
+        complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete".
+        finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.finished".
+        pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse".
+    """
+
+    def __init__(
+        self,
+        bar_width: Optional[int] = 40,
+        style: StyleType = "bar.back",
+        complete_style: StyleType = "bar.complete",
+        finished_style: StyleType = "bar.finished",
+        pulse_style: StyleType = "bar.pulse",
+        table_column: Optional[Column] = None,
+    ) -> None:
+        self.bar_width = bar_width
+        self.style = style
+        self.complete_style = complete_style
+        self.finished_style = finished_style
+        self.pulse_style = pulse_style
+        super().__init__(table_column=table_column)
+
+    def render(self, task: "Task") -> ProgressBar:
+        """Gets a progress bar widget for a task."""
+        return ProgressBar(
+            total=max(0, task.total) if task.total is not None else None,
+            completed=max(0, task.completed),
+            width=None if self.bar_width is None else max(1, self.bar_width),
+            pulse=not task.started,
+            animation_time=task.get_time(),
+            style=self.style,
+            complete_style=self.complete_style,
+            finished_style=self.finished_style,
+            pulse_style=self.pulse_style,
+        )
+
+
+class TimeElapsedColumn(ProgressColumn):
+    """Renders time elapsed."""
+
+    def render(self, task: "Task") -> Text:
+        """Show time elapsed."""
+        elapsed = task.finished_time if task.finished else task.elapsed
+        if elapsed is None:
+            return Text("-:--:--", style="progress.elapsed")
+        delta = timedelta(seconds=max(0, int(elapsed)))
+        return Text(str(delta), style="progress.elapsed")
+
+
+class TaskProgressColumn(TextColumn):
+    """Show task progress as a percentage.
+
+    Args:
+        text_format (str, optional): Format for percentage display. Defaults to "[progress.percentage]{task.percentage:>3.0f}%".
+        text_format_no_percentage (str, optional): Format if percentage is unknown. Defaults to "".
+        style (StyleType, optional): Style of output. Defaults to "none".
+        justify (JustifyMethod, optional): Text justification. Defaults to "left".
+        markup (bool, optional): Enable markup. Defaults to True.
+        highlighter (Optional[Highlighter], optional): Highlighter to apply to output. Defaults to None.
+        table_column (Optional[Column], optional): Table Column to use. Defaults to None.
+        show_speed (bool, optional): Show speed if total is unknown. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        text_format: str = "[progress.percentage]{task.percentage:>3.0f}%",
+        text_format_no_percentage: str = "",
+        style: StyleType = "none",
+        justify: JustifyMethod = "left",
+        markup: bool = True,
+        highlighter: Optional[Highlighter] = None,
+        table_column: Optional[Column] = None,
+        show_speed: bool = False,
+    ) -> None:
+        self.text_format_no_percentage = text_format_no_percentage
+        self.show_speed = show_speed
+        super().__init__(
+            text_format=text_format,
+            style=style,
+            justify=justify,
+            markup=markup,
+            highlighter=highlighter,
+            table_column=table_column,
+        )
+
+    @classmethod
+    def render_speed(cls, speed: Optional[float]) -> Text:
+        """Render the speed in iterations per second.
+
+        Args:
+            task (Task): A Task object.
+
+        Returns:
+            Text: Text object containing the task speed.
+        """
+        if speed is None:
+            return Text("", style="progress.percentage")
+        unit, suffix = filesize.pick_unit_and_suffix(
+            int(speed),
+            ["", "×10³", "×10⁶", "×10⁹", "×10¹²"],
+            1000,
+        )
+        data_speed = speed / unit
+        return Text(f"{data_speed:.1f}{suffix} it/s", style="progress.percentage")
+
+    def render(self, task: "Task") -> Text:
+        if task.total is None and self.show_speed:
+            return self.render_speed(task.finished_speed or task.speed)
+        text_format = (
+            self.text_format_no_percentage if task.total is None else self.text_format
+        )
+        _text = text_format.format(task=task)
+        if self.markup:
+            text = Text.from_markup(_text, style=self.style, justify=self.justify)
+        else:
+            text = Text(_text, style=self.style, justify=self.justify)
+        if self.highlighter:
+            self.highlighter.highlight(text)
+        return text
+
+
+class TimeRemainingColumn(ProgressColumn):
+    """Renders estimated time remaining.
+
+    Args:
+        compact (bool, optional): Render MM:SS when time remaining is less than an hour. Defaults to False.
+        elapsed_when_finished (bool, optional): Render time elapsed when the task is finished. Defaults to False.
+    """
+
+    # Only refresh twice a second to prevent jitter
+    max_refresh = 0.5
+
+    def __init__(
+        self,
+        compact: bool = False,
+        elapsed_when_finished: bool = False,
+        table_column: Optional[Column] = None,
+    ):
+        self.compact = compact
+        self.elapsed_when_finished = elapsed_when_finished
+        super().__init__(table_column=table_column)
+
+    def render(self, task: "Task") -> Text:
+        """Show time remaining."""
+        if self.elapsed_when_finished and task.finished:
+            task_time = task.finished_time
+            style = "progress.elapsed"
+        else:
+            task_time = task.time_remaining
+            style = "progress.remaining"
+
+        if task.total is None:
+            return Text("", style=style)
+
+        if task_time is None:
+            return Text("--:--" if self.compact else "-:--:--", style=style)
+
+        # Based on https://github.com/tqdm/tqdm/blob/master/tqdm/std.py
+        minutes, seconds = divmod(int(task_time), 60)
+        hours, minutes = divmod(minutes, 60)
+
+        if self.compact and not hours:
+            formatted = f"{minutes:02d}:{seconds:02d}"
+        else:
+            formatted = f"{hours:d}:{minutes:02d}:{seconds:02d}"
+
+        return Text(formatted, style=style)
+
+
+class FileSizeColumn(ProgressColumn):
+    """Renders completed filesize."""
+
+    def render(self, task: "Task") -> Text:
+        """Show data completed."""
+        data_size = filesize.decimal(int(task.completed))
+        return Text(data_size, style="progress.filesize")
+
+
+class TotalFileSizeColumn(ProgressColumn):
+    """Renders total filesize."""
+
+    def render(self, task: "Task") -> Text:
+        """Show data completed."""
+        data_size = filesize.decimal(int(task.total)) if task.total is not None else ""
+        return Text(data_size, style="progress.filesize.total")
+
+
+class MofNCompleteColumn(ProgressColumn):
+    """Renders completed count/total, e.g. '  10/1000'.
+
+    Best for bounded tasks with int quantities.
+
+    Space pads the completed count so that progress length does not change as task progresses
+    past powers of 10.
+
+    Args:
+        separator (str, optional): Text to separate completed and total values. Defaults to "/".
+    """
+
+    def __init__(self, separator: str = "/", table_column: Optional[Column] = None):
+        self.separator = separator
+        super().__init__(table_column=table_column)
+
+    def render(self, task: "Task") -> Text:
+        """Show completed/total."""
+        completed = int(task.completed)
+        total = int(task.total) if task.total is not None else "?"
+        total_width = len(str(total))
+        return Text(
+            f"{completed:{total_width}d}{self.separator}{total}",
+            style="progress.download",
+        )
+
+
+class DownloadColumn(ProgressColumn):
+    """Renders file size downloaded and total, e.g. '0.5/2.3 GB'.
+
+    Args:
+        binary_units (bool, optional): Use binary units, KiB, MiB etc. Defaults to False.
+    """
+
+    def __init__(
+        self, binary_units: bool = False, table_column: Optional[Column] = None
+    ) -> None:
+        self.binary_units = binary_units
+        super().__init__(table_column=table_column)
+
+    def render(self, task: "Task") -> Text:
+        """Calculate common unit for completed and total."""
+        completed = int(task.completed)
+
+        unit_and_suffix_calculation_base = (
+            int(task.total) if task.total is not None else completed
+        )
+        if self.binary_units:
+            unit, suffix = filesize.pick_unit_and_suffix(
+                unit_and_suffix_calculation_base,
+                ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"],
+                1024,
+            )
+        else:
+            unit, suffix = filesize.pick_unit_and_suffix(
+                unit_and_suffix_calculation_base,
+                ["bytes", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"],
+                1000,
+            )
+        precision = 0 if unit == 1 else 1
+
+        completed_ratio = completed / unit
+        completed_str = f"{completed_ratio:,.{precision}f}"
+
+        if task.total is not None:
+            total = int(task.total)
+            total_ratio = total / unit
+            total_str = f"{total_ratio:,.{precision}f}"
+        else:
+            total_str = "?"
+
+        download_status = f"{completed_str}/{total_str} {suffix}"
+        download_text = Text(download_status, style="progress.download")
+        return download_text
+
+
+class TransferSpeedColumn(ProgressColumn):
+    """Renders human readable transfer speed."""
+
+    def render(self, task: "Task") -> Text:
+        """Show data transfer speed."""
+        speed = task.finished_speed or task.speed
+        if speed is None:
+            return Text("?", style="progress.data.speed")
+        data_speed = filesize.decimal(int(speed))
+        return Text(f"{data_speed}/s", style="progress.data.speed")
+
+
+class ProgressSample(NamedTuple):
+    """Sample of progress for a given time."""
+
+    timestamp: float
+    """Timestamp of sample."""
+    completed: float
+    """Number of steps completed."""
+
+
+@dataclass
+class Task:
+    """Information regarding a progress task.
+
+    This object should be considered read-only outside of the :class:`~Progress` class.
+
+    """
+
+    id: TaskID
+    """Task ID associated with this task (used in Progress methods)."""
+
+    description: str
+    """str: Description of the task."""
+
+    total: Optional[float]
+    """Optional[float]: Total number of steps in this task."""
+
+    completed: float
+    """float: Number of steps completed"""
+
+    _get_time: GetTimeCallable
+    """Callable to get the current time."""
+
+    finished_time: Optional[float] = None
+    """float: Time task was finished."""
+
+    visible: bool = True
+    """bool: Indicates if this task is visible in the progress display."""
+
+    fields: Dict[str, Any] = field(default_factory=dict)
+    """dict: Arbitrary fields passed in via Progress.update."""
+
+    start_time: Optional[float] = field(default=None, init=False, repr=False)
+    """Optional[float]: Time this task was started, or None if not started."""
+
+    stop_time: Optional[float] = field(default=None, init=False, repr=False)
+    """Optional[float]: Time this task was stopped, or None if not stopped."""
+
+    finished_speed: Optional[float] = None
+    """Optional[float]: The last speed for a finished task."""
+
+    _progress: Deque[ProgressSample] = field(
+        default_factory=lambda: deque(maxlen=1000), init=False, repr=False
+    )
+
+    _lock: RLock = field(repr=False, default_factory=RLock)
+    """Thread lock."""
+
+    def get_time(self) -> float:
+        """float: Get the current time, in seconds."""
+        return self._get_time()
+
+    @property
+    def started(self) -> bool:
+        """bool: Check if the task as started."""
+        return self.start_time is not None
+
+    @property
+    def remaining(self) -> Optional[float]:
+        """Optional[float]: Get the number of steps remaining, if a non-None total was set."""
+        if self.total is None:
+            return None
+        return self.total - self.completed
+
+    @property
+    def elapsed(self) -> Optional[float]:
+        """Optional[float]: Time elapsed since task was started, or ``None`` if the task hasn't started."""
+        if self.start_time is None:
+            return None
+        if self.stop_time is not None:
+            return self.stop_time - self.start_time
+        return self.get_time() - self.start_time
+
+    @property
+    def finished(self) -> bool:
+        """Check if the task has finished."""
+        return self.finished_time is not None
+
+    @property
+    def percentage(self) -> float:
+        """float: Get progress of task as a percentage. If a None total was set, returns 0"""
+        if not self.total:
+            return 0.0
+        completed = (self.completed / self.total) * 100.0
+        completed = min(100.0, max(0.0, completed))
+        return completed
+
+    @property
+    def speed(self) -> Optional[float]:
+        """Optional[float]: Get the estimated speed in steps per second."""
+        if self.start_time is None:
+            return None
+        with self._lock:
+            progress = self._progress
+            if not progress:
+                return None
+            total_time = progress[-1].timestamp - progress[0].timestamp
+            if total_time == 0:
+                return None
+            iter_progress = iter(progress)
+            next(iter_progress)
+            total_completed = sum(sample.completed for sample in iter_progress)
+            speed = total_completed / total_time
+            return speed
+
+    @property
+    def time_remaining(self) -> Optional[float]:
+        """Optional[float]: Get estimated time to completion, or ``None`` if no data."""
+        if self.finished:
+            return 0.0
+        speed = self.speed
+        if not speed:
+            return None
+        remaining = self.remaining
+        if remaining is None:
+            return None
+        estimate = ceil(remaining / speed)
+        return estimate
+
+    def _reset(self) -> None:
+        """Reset progress."""
+        self._progress.clear()
+        self.finished_time = None
+        self.finished_speed = None
+
+
+class Progress(JupyterMixin):
+    """Renders an auto-updating progress bar(s).
+
+    Args:
+        console (Console, optional): Optional Console instance. Defaults to an internal Console instance writing to stdout.
+        auto_refresh (bool, optional): Enable auto refresh. If disabled, you will need to call `refresh()`.
+        refresh_per_second (float, optional): Number of times per second to refresh the progress information. Defaults to 10.
+        speed_estimate_period: (float, optional): Period (in seconds) used to calculate the speed estimate. Defaults to 30.
+        transient: (bool, optional): Clear the progress on exit. Defaults to False.
+        redirect_stdout: (bool, optional): Enable redirection of stdout, so ``print`` may be used. Defaults to True.
+        redirect_stderr: (bool, optional): Enable redirection of stderr. Defaults to True.
+        get_time: (Callable, optional): A callable that gets the current time, or None to use Console.get_time. Defaults to None.
+        disable (bool, optional): Disable progress display. Defaults to False
+        expand (bool, optional): Expand tasks table to fit width. Defaults to False.
+    """
+
+    def __init__(
+        self,
+        *columns: Union[str, ProgressColumn],
+        console: Optional[Console] = None,
+        auto_refresh: bool = True,
+        refresh_per_second: float = 10,
+        speed_estimate_period: float = 30.0,
+        transient: bool = False,
+        redirect_stdout: bool = True,
+        redirect_stderr: bool = True,
+        get_time: Optional[GetTimeCallable] = None,
+        disable: bool = False,
+        expand: bool = False,
+    ) -> None:
+        assert refresh_per_second > 0, "refresh_per_second must be > 0"
+        self._lock = RLock()
+        self.columns = columns or self.get_default_columns()
+        self.speed_estimate_period = speed_estimate_period
+
+        self.disable = disable
+        self.expand = expand
+        self._tasks: Dict[TaskID, Task] = {}
+        self._task_index: TaskID = TaskID(0)
+        self.live = Live(
+            console=console or get_console(),
+            auto_refresh=auto_refresh,
+            refresh_per_second=refresh_per_second,
+            transient=transient,
+            redirect_stdout=redirect_stdout,
+            redirect_stderr=redirect_stderr,
+            get_renderable=self.get_renderable,
+        )
+        self.get_time = get_time or self.console.get_time
+        self.print = self.console.print
+        self.log = self.console.log
+
+    @classmethod
+    def get_default_columns(cls) -> Tuple[ProgressColumn, ...]:
+        """Get the default columns used for a new Progress instance:
+           - a text column for the description (TextColumn)
+           - the bar itself (BarColumn)
+           - a text column showing completion percentage (TextColumn)
+           - an estimated-time-remaining column (TimeRemainingColumn)
+        If the Progress instance is created without passing a columns argument,
+        the default columns defined here will be used.
+
+        You can also create a Progress instance using custom columns before
+        and/or after the defaults, as in this example:
+
+            progress = Progress(
+                SpinnerColumn(),
+                *Progress.get_default_columns(),
+                "Elapsed:",
+                TimeElapsedColumn(),
+            )
+
+        This code shows the creation of a Progress display, containing
+        a spinner to the left, the default columns, and a labeled elapsed
+        time column.
+        """
+        return (
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TimeRemainingColumn(),
+        )
+
+    @property
+    def console(self) -> Console:
+        return self.live.console
+
+    @property
+    def tasks(self) -> List[Task]:
+        """Get a list of Task instances."""
+        with self._lock:
+            return list(self._tasks.values())
+
+    @property
+    def task_ids(self) -> List[TaskID]:
+        """A list of task IDs."""
+        with self._lock:
+            return list(self._tasks.keys())
+
+    @property
+    def finished(self) -> bool:
+        """Check if all tasks have been completed."""
+        with self._lock:
+            if not self._tasks:
+                return True
+            return all(task.finished for task in self._tasks.values())
+
+    def start(self) -> None:
+        """Start the progress display."""
+        if not self.disable:
+            self.live.start(refresh=True)
+
+    def stop(self) -> None:
+        """Stop the progress display."""
+        if not self.disable:
+            self.live.stop()
+            if not self.console.is_interactive and not self.console.is_jupyter:
+                self.console.print()
+
+    def __enter__(self) -> Self:
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.stop()
+
+    def track(
+        self,
+        sequence: Iterable[ProgressType],
+        total: Optional[float] = None,
+        completed: int = 0,
+        task_id: Optional[TaskID] = None,
+        description: str = "Working...",
+        update_period: float = 0.1,
+    ) -> Iterable[ProgressType]:
+        """Track progress by iterating over a sequence.
+
+        You can also track progress of an iterable, which might require that you additionally specify ``total``.
+
+        Args:
+            sequence (Iterable[ProgressType]): Values you want to iterate over and track progress.
+            total: (float, optional): Total number of steps. Default is len(sequence).
+            completed (int, optional): Number of steps completed so far. Defaults to 0.
+            task_id: (TaskID): Task to track. Default is new task.
+            description: (str, optional): Description of task, if new task is created.
+            update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1.
+
+        Returns:
+            Iterable[ProgressType]: An iterable of values taken from the provided sequence.
+        """
+        if total is None:
+            total = float(length_hint(sequence)) or None
+
+        if task_id is None:
+            task_id = self.add_task(description, total=total, completed=completed)
+        else:
+            self.update(task_id, total=total, completed=completed)
+
+        if self.live.auto_refresh:
+            with _TrackThread(self, task_id, update_period) as track_thread:
+                for value in sequence:
+                    yield value
+                    track_thread.completed += 1
+        else:
+            advance = self.advance
+            refresh = self.refresh
+            for value in sequence:
+                yield value
+                advance(task_id, 1)
+                refresh()
+
+    def wrap_file(
+        self,
+        file: BinaryIO,
+        total: Optional[int] = None,
+        *,
+        task_id: Optional[TaskID] = None,
+        description: str = "Reading...",
+    ) -> BinaryIO:
+        """Track progress file reading from a binary file.
+
+        Args:
+            file (BinaryIO): A file-like object opened in binary mode.
+            total (int, optional): Total number of bytes to read. This must be provided unless a task with a total is also given.
+            task_id (TaskID): Task to track. Default is new task.
+            description (str, optional): Description of task, if new task is created.
+
+        Returns:
+            BinaryIO: A readable file-like object in binary mode.
+
+        Raises:
+            ValueError: When no total value can be extracted from the arguments or the task.
+        """
+        # attempt to recover the total from the task
+        total_bytes: Optional[float] = None
+        if total is not None:
+            total_bytes = total
+        elif task_id is not None:
+            with self._lock:
+                total_bytes = self._tasks[task_id].total
+        if total_bytes is None:
+            raise ValueError(
+                f"unable to get the total number of bytes, please specify 'total'"
+            )
+
+        # update total of task or create new task
+        if task_id is None:
+            task_id = self.add_task(description, total=total_bytes)
+        else:
+            self.update(task_id, total=total_bytes)
+
+        return _Reader(file, self, task_id, close_handle=False)
+
+    @typing.overload
+    def open(
+        self,
+        file: Union[str, "PathLike[str]", bytes],
+        mode: Literal["rb"],
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        *,
+        total: Optional[int] = None,
+        task_id: Optional[TaskID] = None,
+        description: str = "Reading...",
+    ) -> BinaryIO:
+        pass
+
+    @typing.overload
+    def open(
+        self,
+        file: Union[str, "PathLike[str]", bytes],
+        mode: Union[Literal["r"], Literal["rt"]],
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        *,
+        total: Optional[int] = None,
+        task_id: Optional[TaskID] = None,
+        description: str = "Reading...",
+    ) -> TextIO:
+        pass
+
+    def open(
+        self,
+        file: Union[str, "PathLike[str]", bytes],
+        mode: Union[Literal["rb"], Literal["rt"], Literal["r"]] = "r",
+        buffering: int = -1,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        newline: Optional[str] = None,
+        *,
+        total: Optional[int] = None,
+        task_id: Optional[TaskID] = None,
+        description: str = "Reading...",
+    ) -> Union[BinaryIO, TextIO]:
+        """Track progress while reading from a binary file.
+
+        Args:
+            path (Union[str, PathLike[str]]): The path to the file to read.
+            mode (str): The mode to use to open the file. Only supports "r", "rb" or "rt".
+            buffering (int): The buffering strategy to use, see :func:`io.open`.
+            encoding (str, optional): The encoding to use when reading in text mode, see :func:`io.open`.
+            errors (str, optional): The error handling strategy for decoding errors, see :func:`io.open`.
+            newline (str, optional): The strategy for handling newlines in text mode, see :func:`io.open`.
+            total (int, optional): Total number of bytes to read. If none given, os.stat(path).st_size is used.
+            task_id (TaskID): Task to track. Default is new task.
+            description (str, optional): Description of task, if new task is created.
+
+        Returns:
+            BinaryIO: A readable file-like object in binary mode.
+
+        Raises:
+            ValueError: When an invalid mode is given.
+        """
+        # normalize the mode (always rb, rt)
+        _mode = "".join(sorted(mode, reverse=False))
+        if _mode not in ("br", "rt", "r"):
+            raise ValueError(f"invalid mode {mode!r}")
+
+        # patch buffering to provide the same behaviour as the builtin `open`
+        line_buffering = buffering == 1
+        if _mode == "br" and buffering == 1:
+            warnings.warn(
+                "line buffering (buffering=1) isn't supported in binary mode, the default buffer size will be used",
+                RuntimeWarning,
+            )
+            buffering = -1
+        elif _mode in ("rt", "r"):
+            if buffering == 0:
+                raise ValueError("can't have unbuffered text I/O")
+            elif buffering == 1:
+                buffering = -1
+
+        # attempt to get the total with `os.stat`
+        if total is None:
+            total = stat(file).st_size
+
+        # update total of task or create new task
+        if task_id is None:
+            task_id = self.add_task(description, total=total)
+        else:
+            self.update(task_id, total=total)
+
+        # open the file in binary mode,
+        handle = io.open(file, "rb", buffering=buffering)
+        reader = _Reader(handle, self, task_id, close_handle=True)
+
+        # wrap the reader in a `TextIOWrapper` if text mode
+        if mode in ("r", "rt"):
+            return io.TextIOWrapper(
+                reader,
+                encoding=encoding,
+                errors=errors,
+                newline=newline,
+                line_buffering=line_buffering,
+            )
+
+        return reader
+
+    def start_task(self, task_id: TaskID) -> None:
+        """Start a task.
+
+        Starts a task (used when calculating elapsed time). You may need to call this manually,
+        if you called ``add_task`` with ``start=False``.
+
+        Args:
+            task_id (TaskID): ID of task.
+        """
+        with self._lock:
+            task = self._tasks[task_id]
+            if task.start_time is None:
+                task.start_time = self.get_time()
+
+    def stop_task(self, task_id: TaskID) -> None:
+        """Stop a task.
+
+        This will freeze the elapsed time on the task.
+
+        Args:
+            task_id (TaskID): ID of task.
+        """
+        with self._lock:
+            task = self._tasks[task_id]
+            current_time = self.get_time()
+            if task.start_time is None:
+                task.start_time = current_time
+            task.stop_time = current_time
+
+    def update(
+        self,
+        task_id: TaskID,
+        *,
+        total: Optional[float] = None,
+        completed: Optional[float] = None,
+        advance: Optional[float] = None,
+        description: Optional[str] = None,
+        visible: Optional[bool] = None,
+        refresh: bool = False,
+        **fields: Any,
+    ) -> None:
+        """Update information associated with a task.
+
+        Args:
+            task_id (TaskID): Task id (returned by add_task).
+            total (float, optional): Updates task.total if not None.
+            completed (float, optional): Updates task.completed if not None.
+            advance (float, optional): Add a value to task.completed if not None.
+            description (str, optional): Change task description if not None.
+            visible (bool, optional): Set visible flag if not None.
+            refresh (bool): Force a refresh of progress information. Default is False.
+            **fields (Any): Additional data fields required for rendering.
+        """
+        with self._lock:
+            task = self._tasks[task_id]
+            completed_start = task.completed
+
+            if total is not None and total != task.total:
+                task.total = total
+                task._reset()
+            if advance is not None:
+                task.completed += advance
+            if completed is not None:
+                task.completed = completed
+            if description is not None:
+                task.description = description
+            if visible is not None:
+                task.visible = visible
+            task.fields.update(fields)
+            update_completed = task.completed - completed_start
+
+            current_time = self.get_time()
+            old_sample_time = current_time - self.speed_estimate_period
+            _progress = task._progress
+
+            popleft = _progress.popleft
+            while _progress and _progress[0].timestamp < old_sample_time:
+                popleft()
+            if update_completed > 0:
+                _progress.append(ProgressSample(current_time, update_completed))
+            if (
+                task.total is not None
+                and task.completed >= task.total
+                and task.finished_time is None
+            ):
+                task.finished_time = task.elapsed
+
+        if refresh:
+            self.refresh()
+
+    def reset(
+        self,
+        task_id: TaskID,
+        *,
+        start: bool = True,
+        total: Optional[float] = None,
+        completed: int = 0,
+        visible: Optional[bool] = None,
+        description: Optional[str] = None,
+        **fields: Any,
+    ) -> None:
+        """Reset a task so completed is 0 and the clock is reset.
+
+        Args:
+            task_id (TaskID): ID of task.
+            start (bool, optional): Start the task after reset. Defaults to True.
+            total (float, optional): New total steps in task, or None to use current total. Defaults to None.
+            completed (int, optional): Number of steps completed. Defaults to 0.
+            visible (bool, optional): Set visible flag if not None.
+            description (str, optional): Change task description if not None. Defaults to None.
+            **fields (str): Additional data fields required for rendering.
+        """
+        current_time = self.get_time()
+        with self._lock:
+            task = self._tasks[task_id]
+            task._reset()
+            task.start_time = current_time if start else None
+            if total is not None:
+                task.total = total
+            task.completed = completed
+            if visible is not None:
+                task.visible = visible
+            if fields:
+                task.fields = fields
+            if description is not None:
+                task.description = description
+            task.finished_time = None
+        self.refresh()
+
+    def advance(self, task_id: TaskID, advance: float = 1) -> None:
+        """Advance task by a number of steps.
+
+        Args:
+            task_id (TaskID): ID of task.
+            advance (float): Number of steps to advance. Default is 1.
+        """
+        current_time = self.get_time()
+        with self._lock:
+            task = self._tasks[task_id]
+            completed_start = task.completed
+            task.completed += advance
+            update_completed = task.completed - completed_start
+            old_sample_time = current_time - self.speed_estimate_period
+            _progress = task._progress
+
+            popleft = _progress.popleft
+            while _progress and _progress[0].timestamp < old_sample_time:
+                popleft()
+            while len(_progress) > 1000:
+                popleft()
+            _progress.append(ProgressSample(current_time, update_completed))
+            if (
+                task.total is not None
+                and task.completed >= task.total
+                and task.finished_time is None
+            ):
+                task.finished_time = task.elapsed
+                task.finished_speed = task.speed
+
+    def refresh(self) -> None:
+        """Refresh (render) the progress information."""
+        if not self.disable and self.live.is_started:
+            self.live.refresh()
+
+    def get_renderable(self) -> RenderableType:
+        """Get a renderable for the progress display."""
+        renderable = Group(*self.get_renderables())
+        return renderable
+
+    def get_renderables(self) -> Iterable[RenderableType]:
+        """Get a number of renderables for the progress display."""
+        table = self.make_tasks_table(self.tasks)
+        yield table
+
+    def make_tasks_table(self, tasks: Iterable[Task]) -> Table:
+        """Get a table to render the Progress display.
+
+        Args:
+            tasks (Iterable[Task]): An iterable of Task instances, one per row of the table.
+
+        Returns:
+            Table: A table instance.
+        """
+        table_columns = (
+            (
+                Column(no_wrap=True)
+                if isinstance(_column, str)
+                else _column.get_table_column().copy()
+            )
+            for _column in self.columns
+        )
+        table = Table.grid(*table_columns, padding=(0, 1), expand=self.expand)
+
+        for task in tasks:
+            if task.visible:
+                table.add_row(
+                    *(
+                        (
+                            column.format(task=task)
+                            if isinstance(column, str)
+                            else column(task)
+                        )
+                        for column in self.columns
+                    )
+                )
+        return table
+
+    def __rich__(self) -> RenderableType:
+        """Makes the Progress class itself renderable."""
+        with self._lock:
+            return self.get_renderable()
+
+    def add_task(
+        self,
+        description: str,
+        start: bool = True,
+        total: Optional[float] = 100.0,
+        completed: int = 0,
+        visible: bool = True,
+        **fields: Any,
+    ) -> TaskID:
+        """Add a new 'task' to the Progress display.
+
+        Args:
+            description (str): A description of the task.
+            start (bool, optional): Start the task immediately (to calculate elapsed time). If set to False,
+                you will need to call `start` manually. Defaults to True.
+            total (float, optional): Number of total steps in the progress if known.
+                Set to None to render a pulsing animation. Defaults to 100.
+            completed (int, optional): Number of steps completed so far. Defaults to 0.
+            visible (bool, optional): Enable display of the task. Defaults to True.
+            **fields (str): Additional data fields required for rendering.
+
+        Returns:
+            TaskID: An ID you can use when calling `update`.
+        """
+        with self._lock:
+            task = Task(
+                self._task_index,
+                description,
+                total,
+                completed,
+                visible=visible,
+                fields=fields,
+                _get_time=self.get_time,
+                _lock=self._lock,
+            )
+            self._tasks[self._task_index] = task
+            if start:
+                self.start_task(self._task_index)
+            new_task_index = self._task_index
+            self._task_index = TaskID(int(self._task_index) + 1)
+        self.refresh()
+        return new_task_index
+
+    def remove_task(self, task_id: TaskID) -> None:
+        """Delete a task if it exists.
+
+        Args:
+            task_id (TaskID): A task ID.
+
+        """
+        with self._lock:
+            del self._tasks[task_id]
+
+
+if __name__ == "__main__":  # pragma: no coverage
+    import random
+    import time
+
+    from .panel import Panel
+    from .rule import Rule
+    from .syntax import Syntax
+    from .table import Table
+
+    syntax = Syntax(
+        '''def loop_last(values: Iterable[T]) -> Iterable[Tuple[bool, T]]:
+    """Iterate and generate a tuple with a flag for last value."""
+    iter_values = iter(values)
+    try:
+        previous_value = next(iter_values)
+    except StopIteration:
+        return
+    for value in iter_values:
+        yield False, previous_value
+        previous_value = value
+    yield True, previous_value''',
+        "python",
+        line_numbers=True,
+    )
+
+    table = Table("foo", "bar", "baz")
+    table.add_row("1", "2", "3")
+
+    progress_renderables = [
+        "Text may be printed while the progress bars are rendering.",
+        Panel("In fact, [i]any[/i] renderable will work"),
+        "Such as [magenta]tables[/]...",
+        table,
+        "Pretty printed structures...",
+        {"type": "example", "text": "Pretty printed"},
+        "Syntax...",
+        syntax,
+        Rule("Give it a try!"),
+    ]
+
+    from itertools import cycle
+
+    examples = cycle(progress_renderables)
+
+    console = Console(record=True)
+
+    with Progress(
+        SpinnerColumn(),
+        *Progress.get_default_columns(),
+        TimeElapsedColumn(),
+        console=console,
+        transient=False,
+    ) as progress:
+        task1 = progress.add_task("[red]Downloading", total=1000)
+        task2 = progress.add_task("[green]Processing", total=1000)
+        task3 = progress.add_task("[yellow]Thinking", total=None)
+
+        while not progress.finished:
+            progress.update(task1, advance=0.5)
+            progress.update(task2, advance=0.3)
+            time.sleep(0.01)
+            if random.randint(0, 100) < 1:
+                progress.log(next(examples))
diff --git a/lib/rich/progress_bar.py b/lib/rich/progress_bar.py
new file mode 100644
index 0000000..41794f7
--- /dev/null
+++ b/lib/rich/progress_bar.py
@@ -0,0 +1,223 @@
+import math
+from functools import lru_cache
+from time import monotonic
+from typing import Iterable, List, Optional
+
+from .color import Color, blend_rgb
+from .color_triplet import ColorTriplet
+from .console import Console, ConsoleOptions, RenderResult
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .segment import Segment
+from .style import Style, StyleType
+
+# Number of characters before 'pulse' animation repeats
+PULSE_SIZE = 20
+
+
+class ProgressBar(JupyterMixin):
+    """Renders a (progress) bar. Used by rich.progress.
+
+    Args:
+        total (float, optional): Number of steps in the bar. Defaults to 100. Set to None to render a pulsing animation.
+        completed (float, optional): Number of steps completed. Defaults to 0.
+        width (int, optional): Width of the bar, or ``None`` for maximum width. Defaults to None.
+        pulse (bool, optional): Enable pulse effect. Defaults to False. Will pulse if a None total was passed.
+        style (StyleType, optional): Style for the bar background. Defaults to "bar.back".
+        complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete".
+        finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.finished".
+        pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse".
+        animation_time (Optional[float], optional): Time in seconds to use for animation, or None to use system time.
+    """
+
+    def __init__(
+        self,
+        total: Optional[float] = 100.0,
+        completed: float = 0,
+        width: Optional[int] = None,
+        pulse: bool = False,
+        style: StyleType = "bar.back",
+        complete_style: StyleType = "bar.complete",
+        finished_style: StyleType = "bar.finished",
+        pulse_style: StyleType = "bar.pulse",
+        animation_time: Optional[float] = None,
+    ):
+        self.total = total
+        self.completed = completed
+        self.width = width
+        self.pulse = pulse
+        self.style = style
+        self.complete_style = complete_style
+        self.finished_style = finished_style
+        self.pulse_style = pulse_style
+        self.animation_time = animation_time
+
+        self._pulse_segments: Optional[List[Segment]] = None
+
+    def __repr__(self) -> str:
+        return f""
+
+    @property
+    def percentage_completed(self) -> Optional[float]:
+        """Calculate percentage complete."""
+        if self.total is None:
+            return None
+        completed = (self.completed / self.total) * 100.0
+        completed = min(100, max(0.0, completed))
+        return completed
+
+    @lru_cache(maxsize=16)
+    def _get_pulse_segments(
+        self,
+        fore_style: Style,
+        back_style: Style,
+        color_system: str,
+        no_color: bool,
+        ascii: bool = False,
+    ) -> List[Segment]:
+        """Get a list of segments to render a pulse animation.
+
+        Returns:
+            List[Segment]: A list of segments, one segment per character.
+        """
+        bar = "-" if ascii else "━"
+        segments: List[Segment] = []
+        if color_system not in ("standard", "eight_bit", "truecolor") or no_color:
+            segments += [Segment(bar, fore_style)] * (PULSE_SIZE // 2)
+            segments += [Segment(" " if no_color else bar, back_style)] * (
+                PULSE_SIZE - (PULSE_SIZE // 2)
+            )
+            return segments
+
+        append = segments.append
+        fore_color = (
+            fore_style.color.get_truecolor()
+            if fore_style.color
+            else ColorTriplet(255, 0, 255)
+        )
+        back_color = (
+            back_style.color.get_truecolor()
+            if back_style.color
+            else ColorTriplet(0, 0, 0)
+        )
+        cos = math.cos
+        pi = math.pi
+        _Segment = Segment
+        _Style = Style
+        from_triplet = Color.from_triplet
+
+        for index in range(PULSE_SIZE):
+            position = index / PULSE_SIZE
+            fade = 0.5 + cos(position * pi * 2) / 2.0
+            color = blend_rgb(fore_color, back_color, cross_fade=fade)
+            append(_Segment(bar, _Style(color=from_triplet(color))))
+        return segments
+
+    def update(self, completed: float, total: Optional[float] = None) -> None:
+        """Update progress with new values.
+
+        Args:
+            completed (float): Number of steps completed.
+            total (float, optional): Total number of steps, or ``None`` to not change. Defaults to None.
+        """
+        self.completed = completed
+        self.total = total if total is not None else self.total
+
+    def _render_pulse(
+        self, console: Console, width: int, ascii: bool = False
+    ) -> Iterable[Segment]:
+        """Renders the pulse animation.
+
+        Args:
+            console (Console): Console instance.
+            width (int): Width in characters of pulse animation.
+
+        Returns:
+            RenderResult: [description]
+
+        Yields:
+            Iterator[Segment]: Segments to render pulse
+        """
+        fore_style = console.get_style(self.pulse_style, default="white")
+        back_style = console.get_style(self.style, default="black")
+
+        pulse_segments = self._get_pulse_segments(
+            fore_style, back_style, console.color_system, console.no_color, ascii=ascii
+        )
+        segment_count = len(pulse_segments)
+        current_time = (
+            monotonic() if self.animation_time is None else self.animation_time
+        )
+        segments = pulse_segments * (int(width / segment_count) + 2)
+        offset = int(-current_time * 15) % segment_count
+        segments = segments[offset : offset + width]
+        yield from segments
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        width = min(self.width or options.max_width, options.max_width)
+        ascii = options.legacy_windows or options.ascii_only
+        should_pulse = self.pulse or self.total is None
+        if should_pulse:
+            yield from self._render_pulse(console, width, ascii=ascii)
+            return
+
+        completed: Optional[float] = (
+            min(self.total, max(0, self.completed)) if self.total is not None else None
+        )
+
+        bar = "-" if ascii else "━"
+        half_bar_right = " " if ascii else "╸"
+        half_bar_left = " " if ascii else "╺"
+        complete_halves = (
+            int(width * 2 * completed / self.total)
+            if self.total and completed is not None
+            else width * 2
+        )
+        bar_count = complete_halves // 2
+        half_bar_count = complete_halves % 2
+        style = console.get_style(self.style)
+        is_finished = self.total is None or self.completed >= self.total
+        complete_style = console.get_style(
+            self.finished_style if is_finished else self.complete_style
+        )
+        _Segment = Segment
+        if bar_count:
+            yield _Segment(bar * bar_count, complete_style)
+        if half_bar_count:
+            yield _Segment(half_bar_right * half_bar_count, complete_style)
+
+        if not console.no_color:
+            remaining_bars = width - bar_count - half_bar_count
+            if remaining_bars and console.color_system is not None:
+                if not half_bar_count and bar_count:
+                    yield _Segment(half_bar_left, style)
+                    remaining_bars -= 1
+                if remaining_bars:
+                    yield _Segment(bar * remaining_bars, style)
+
+    def __rich_measure__(
+        self, console: Console, options: ConsoleOptions
+    ) -> Measurement:
+        return (
+            Measurement(self.width, self.width)
+            if self.width is not None
+            else Measurement(4, options.max_width)
+        )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    console = Console()
+    bar = ProgressBar(width=50, total=100)
+
+    import time
+
+    console.show_cursor(False)
+    for n in range(0, 101, 1):
+        bar.update(n)
+        console.print(bar)
+        console.file.write("\r")
+        time.sleep(0.05)
+    console.show_cursor(True)
+    console.print()
diff --git a/lib/rich/prompt.py b/lib/rich/prompt.py
new file mode 100644
index 0000000..ae94d9b
--- /dev/null
+++ b/lib/rich/prompt.py
@@ -0,0 +1,400 @@
+from typing import Any, Generic, List, Optional, TextIO, TypeVar, Union, overload
+
+from . import get_console
+from .console import Console
+from .text import Text, TextType
+
+PromptType = TypeVar("PromptType")
+DefaultType = TypeVar("DefaultType")
+
+
+class PromptError(Exception):
+    """Exception base class for prompt related errors."""
+
+
+class InvalidResponse(PromptError):
+    """Exception to indicate a response was invalid. Raise this within process_response() to indicate an error
+    and provide an error message.
+
+    Args:
+        message (Union[str, Text]): Error message.
+    """
+
+    def __init__(self, message: TextType) -> None:
+        self.message = message
+
+    def __rich__(self) -> TextType:
+        return self.message
+
+
+class PromptBase(Generic[PromptType]):
+    """Ask the user for input until a valid response is received. This is the base class, see one of
+    the concrete classes for examples.
+
+    Args:
+        prompt (TextType, optional): Prompt text. Defaults to "".
+        console (Console, optional): A Console instance or None to use global console. Defaults to None.
+        password (bool, optional): Enable password input. Defaults to False.
+        choices (List[str], optional): A list of valid choices. Defaults to None.
+        case_sensitive (bool, optional): Matching of choices should be case-sensitive. Defaults to True.
+        show_default (bool, optional): Show default in prompt. Defaults to True.
+        show_choices (bool, optional): Show choices in prompt. Defaults to True.
+    """
+
+    response_type: type = str
+
+    validate_error_message = "[prompt.invalid]Please enter a valid value"
+    illegal_choice_message = (
+        "[prompt.invalid.choice]Please select one of the available options"
+    )
+    prompt_suffix = ": "
+
+    choices: Optional[List[str]] = None
+
+    def __init__(
+        self,
+        prompt: TextType = "",
+        *,
+        console: Optional[Console] = None,
+        password: bool = False,
+        choices: Optional[List[str]] = None,
+        case_sensitive: bool = True,
+        show_default: bool = True,
+        show_choices: bool = True,
+    ) -> None:
+        self.console = console or get_console()
+        self.prompt = (
+            Text.from_markup(prompt, style="prompt")
+            if isinstance(prompt, str)
+            else prompt
+        )
+        self.password = password
+        if choices is not None:
+            self.choices = choices
+        self.case_sensitive = case_sensitive
+        self.show_default = show_default
+        self.show_choices = show_choices
+
+    @classmethod
+    @overload
+    def ask(
+        cls,
+        prompt: TextType = "",
+        *,
+        console: Optional[Console] = None,
+        password: bool = False,
+        choices: Optional[List[str]] = None,
+        case_sensitive: bool = True,
+        show_default: bool = True,
+        show_choices: bool = True,
+        default: DefaultType,
+        stream: Optional[TextIO] = None,
+    ) -> Union[DefaultType, PromptType]:
+        ...
+
+    @classmethod
+    @overload
+    def ask(
+        cls,
+        prompt: TextType = "",
+        *,
+        console: Optional[Console] = None,
+        password: bool = False,
+        choices: Optional[List[str]] = None,
+        case_sensitive: bool = True,
+        show_default: bool = True,
+        show_choices: bool = True,
+        stream: Optional[TextIO] = None,
+    ) -> PromptType:
+        ...
+
+    @classmethod
+    def ask(
+        cls,
+        prompt: TextType = "",
+        *,
+        console: Optional[Console] = None,
+        password: bool = False,
+        choices: Optional[List[str]] = None,
+        case_sensitive: bool = True,
+        show_default: bool = True,
+        show_choices: bool = True,
+        default: Any = ...,
+        stream: Optional[TextIO] = None,
+    ) -> Any:
+        """Shortcut to construct and run a prompt loop and return the result.
+
+        Example:
+            >>> filename = Prompt.ask("Enter a filename")
+
+        Args:
+            prompt (TextType, optional): Prompt text. Defaults to "".
+            console (Console, optional): A Console instance or None to use global console. Defaults to None.
+            password (bool, optional): Enable password input. Defaults to False.
+            choices (List[str], optional): A list of valid choices. Defaults to None.
+            case_sensitive (bool, optional): Matching of choices should be case-sensitive. Defaults to True.
+            show_default (bool, optional): Show default in prompt. Defaults to True.
+            show_choices (bool, optional): Show choices in prompt. Defaults to True.
+            stream (TextIO, optional): Optional text file open for reading to get input. Defaults to None.
+        """
+        _prompt = cls(
+            prompt,
+            console=console,
+            password=password,
+            choices=choices,
+            case_sensitive=case_sensitive,
+            show_default=show_default,
+            show_choices=show_choices,
+        )
+        return _prompt(default=default, stream=stream)
+
+    def render_default(self, default: DefaultType) -> Text:
+        """Turn the supplied default in to a Text instance.
+
+        Args:
+            default (DefaultType): Default value.
+
+        Returns:
+            Text: Text containing rendering of default value.
+        """
+        return Text(f"({default})", "prompt.default")
+
+    def make_prompt(self, default: DefaultType) -> Text:
+        """Make prompt text.
+
+        Args:
+            default (DefaultType): Default value.
+
+        Returns:
+            Text: Text to display in prompt.
+        """
+        prompt = self.prompt.copy()
+        prompt.end = ""
+
+        if self.show_choices and self.choices:
+            _choices = "/".join(self.choices)
+            choices = f"[{_choices}]"
+            prompt.append(" ")
+            prompt.append(choices, "prompt.choices")
+
+        if (
+            default != ...
+            and self.show_default
+            and isinstance(default, (str, self.response_type))
+        ):
+            prompt.append(" ")
+            _default = self.render_default(default)
+            prompt.append(_default)
+
+        prompt.append(self.prompt_suffix)
+
+        return prompt
+
+    @classmethod
+    def get_input(
+        cls,
+        console: Console,
+        prompt: TextType,
+        password: bool,
+        stream: Optional[TextIO] = None,
+    ) -> str:
+        """Get input from user.
+
+        Args:
+            console (Console): Console instance.
+            prompt (TextType): Prompt text.
+            password (bool): Enable password entry.
+
+        Returns:
+            str: String from user.
+        """
+        return console.input(prompt, password=password, stream=stream)
+
+    def check_choice(self, value: str) -> bool:
+        """Check value is in the list of valid choices.
+
+        Args:
+            value (str): Value entered by user.
+
+        Returns:
+            bool: True if choice was valid, otherwise False.
+        """
+        assert self.choices is not None
+        if self.case_sensitive:
+            return value.strip() in self.choices
+        return value.strip().lower() in [choice.lower() for choice in self.choices]
+
+    def process_response(self, value: str) -> PromptType:
+        """Process response from user, convert to prompt type.
+
+        Args:
+            value (str): String typed by user.
+
+        Raises:
+            InvalidResponse: If ``value`` is invalid.
+
+        Returns:
+            PromptType: The value to be returned from ask method.
+        """
+        value = value.strip()
+        try:
+            return_value: PromptType = self.response_type(value)
+        except ValueError:
+            raise InvalidResponse(self.validate_error_message)
+
+        if self.choices is not None:
+            if not self.check_choice(value):
+                raise InvalidResponse(self.illegal_choice_message)
+
+            if not self.case_sensitive:
+                # return the original choice, not the lower case version
+                return_value = self.response_type(
+                    self.choices[
+                        [choice.lower() for choice in self.choices].index(value.lower())
+                    ]
+                )
+        return return_value
+
+    def on_validate_error(self, value: str, error: InvalidResponse) -> None:
+        """Called to handle validation error.
+
+        Args:
+            value (str): String entered by user.
+            error (InvalidResponse): Exception instance the initiated the error.
+        """
+        self.console.print(error, markup=True)
+
+    def pre_prompt(self) -> None:
+        """Hook to display something before the prompt."""
+
+    @overload
+    def __call__(self, *, stream: Optional[TextIO] = None) -> PromptType:
+        ...
+
+    @overload
+    def __call__(
+        self, *, default: DefaultType, stream: Optional[TextIO] = None
+    ) -> Union[PromptType, DefaultType]:
+        ...
+
+    def __call__(self, *, default: Any = ..., stream: Optional[TextIO] = None) -> Any:
+        """Run the prompt loop.
+
+        Args:
+            default (Any, optional): Optional default value.
+
+        Returns:
+            PromptType: Processed value.
+        """
+        while True:
+            self.pre_prompt()
+            prompt = self.make_prompt(default)
+            value = self.get_input(self.console, prompt, self.password, stream=stream)
+            if value == "" and default != ...:
+                return default
+            try:
+                return_value = self.process_response(value)
+            except InvalidResponse as error:
+                self.on_validate_error(value, error)
+                continue
+            else:
+                return return_value
+
+
+class Prompt(PromptBase[str]):
+    """A prompt that returns a str.
+
+    Example:
+        >>> name = Prompt.ask("Enter your name")
+
+
+    """
+
+    response_type = str
+
+
+class IntPrompt(PromptBase[int]):
+    """A prompt that returns an integer.
+
+    Example:
+        >>> burrito_count = IntPrompt.ask("How many burritos do you want to order")
+
+    """
+
+    response_type = int
+    validate_error_message = "[prompt.invalid]Please enter a valid integer number"
+
+
+class FloatPrompt(PromptBase[float]):
+    """A prompt that returns a float.
+
+    Example:
+        >>> temperature = FloatPrompt.ask("Enter desired temperature")
+
+    """
+
+    response_type = float
+    validate_error_message = "[prompt.invalid]Please enter a number"
+
+
+class Confirm(PromptBase[bool]):
+    """A yes / no confirmation prompt.
+
+    Example:
+        >>> if Confirm.ask("Continue"):
+                run_job()
+
+    """
+
+    response_type = bool
+    validate_error_message = "[prompt.invalid]Please enter Y or N"
+    choices: List[str] = ["y", "n"]
+
+    def render_default(self, default: DefaultType) -> Text:
+        """Render the default as (y) or (n) rather than True/False."""
+        yes, no = self.choices
+        return Text(f"({yes})" if default else f"({no})", style="prompt.default")
+
+    def process_response(self, value: str) -> bool:
+        """Convert choices to a bool."""
+        value = value.strip().lower()
+        if value not in self.choices:
+            raise InvalidResponse(self.validate_error_message)
+        return value == self.choices[0]
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich import print
+
+    if Confirm.ask("Run [i]prompt[/i] tests?", default=True):
+        while True:
+            result = IntPrompt.ask(
+                ":rocket: Enter a number between [b]1[/b] and [b]10[/b]", default=5
+            )
+            if result >= 1 and result <= 10:
+                break
+            print(":pile_of_poo: [prompt.invalid]Number must be between 1 and 10")
+        print(f"number={result}")
+
+        while True:
+            password = Prompt.ask(
+                "Please enter a password [cyan](must be at least 5 characters)",
+                password=True,
+            )
+            if len(password) >= 5:
+                break
+            print("[prompt.invalid]password too short")
+        print(f"password={password!r}")
+
+        fruit = Prompt.ask("Enter a fruit", choices=["apple", "orange", "pear"])
+        print(f"fruit={fruit!r}")
+
+        doggie = Prompt.ask(
+            "What's the best Dog? (Case INSENSITIVE)",
+            choices=["Border Terrier", "Collie", "Labradoodle"],
+            case_sensitive=False,
+        )
+        print(f"doggie={doggie!r}")
+
+    else:
+        print("[b]OK :loudly_crying_face:")
diff --git a/lib/rich/protocol.py b/lib/rich/protocol.py
new file mode 100644
index 0000000..c6923dd
--- /dev/null
+++ b/lib/rich/protocol.py
@@ -0,0 +1,42 @@
+from typing import Any, cast, Set, TYPE_CHECKING
+from inspect import isclass
+
+if TYPE_CHECKING:
+    from rich.console import RenderableType
+
+_GIBBERISH = """aihwerij235234ljsdnp34ksodfipwoe234234jlskjdf"""
+
+
+def is_renderable(check_object: Any) -> bool:
+    """Check if an object may be rendered by Rich."""
+    return (
+        isinstance(check_object, str)
+        or hasattr(check_object, "__rich__")
+        or hasattr(check_object, "__rich_console__")
+    )
+
+
+def rich_cast(renderable: object) -> "RenderableType":
+    """Cast an object to a renderable by calling __rich__ if present.
+
+    Args:
+        renderable (object): A potentially renderable object
+
+    Returns:
+        object: The result of recursively calling __rich__.
+    """
+    from rich.console import RenderableType
+
+    rich_visited_set: Set[type] = set()  # Prevent potential infinite loop
+    while hasattr(renderable, "__rich__") and not isclass(renderable):
+        # Detect object which claim to have all the attributes
+        if hasattr(renderable, _GIBBERISH):
+            return repr(renderable)
+        cast_method = getattr(renderable, "__rich__")
+        renderable = cast_method()
+        renderable_type = type(renderable)
+        if renderable_type in rich_visited_set:
+            break
+        rich_visited_set.add(renderable_type)
+
+    return cast(RenderableType, renderable)
diff --git a/lib/rich/py.typed b/lib/rich/py.typed
new file mode 100644
index 0000000..e69de29
diff --git a/lib/rich/region.py b/lib/rich/region.py
new file mode 100644
index 0000000..75b3631
--- /dev/null
+++ b/lib/rich/region.py
@@ -0,0 +1,10 @@
+from typing import NamedTuple
+
+
+class Region(NamedTuple):
+    """Defines a rectangular region of the screen."""
+
+    x: int
+    y: int
+    width: int
+    height: int
diff --git a/lib/rich/repr.py b/lib/rich/repr.py
new file mode 100644
index 0000000..9533100
--- /dev/null
+++ b/lib/rich/repr.py
@@ -0,0 +1,149 @@
+import inspect
+from functools import partial
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    overload,
+)
+
+T = TypeVar("T")
+
+
+Result = Iterable[Union[Any, Tuple[Any], Tuple[str, Any], Tuple[str, Any, Any]]]
+RichReprResult = Result
+
+
+class ReprError(Exception):
+    """An error occurred when attempting to build a repr."""
+
+
+@overload
+def auto(cls: Optional[Type[T]]) -> Type[T]:
+    ...
+
+
+@overload
+def auto(*, angular: bool = False) -> Callable[[Type[T]], Type[T]]:
+    ...
+
+
+def auto(
+    cls: Optional[Type[T]] = None, *, angular: Optional[bool] = None
+) -> Union[Type[T], Callable[[Type[T]], Type[T]]]:
+    """Class decorator to create __repr__ from __rich_repr__"""
+
+    def do_replace(cls: Type[T], angular: Optional[bool] = None) -> Type[T]:
+        def auto_repr(self: T) -> str:
+            """Create repr string from __rich_repr__"""
+            repr_str: List[str] = []
+            append = repr_str.append
+
+            angular: bool = getattr(self.__rich_repr__, "angular", False)  # type: ignore[attr-defined]
+            for arg in self.__rich_repr__():  # type: ignore[attr-defined]
+                if isinstance(arg, tuple):
+                    if len(arg) == 1:
+                        append(repr(arg[0]))
+                    else:
+                        key, value, *default = arg
+                        if key is None:
+                            append(repr(value))
+                        else:
+                            if default and default[0] == value:
+                                continue
+                            append(f"{key}={value!r}")
+                else:
+                    append(repr(arg))
+            if angular:
+                return f"<{self.__class__.__name__} {' '.join(repr_str)}>"
+            else:
+                return f"{self.__class__.__name__}({', '.join(repr_str)})"
+
+        def auto_rich_repr(self: Type[T]) -> Result:
+            """Auto generate __rich_rep__ from signature of __init__"""
+            try:
+                signature = inspect.signature(self.__init__)
+                for name, param in signature.parameters.items():
+                    if param.kind == param.POSITIONAL_ONLY:
+                        yield getattr(self, name)
+                    elif param.kind in (
+                        param.POSITIONAL_OR_KEYWORD,
+                        param.KEYWORD_ONLY,
+                    ):
+                        if param.default is param.empty:
+                            yield getattr(self, param.name)
+                        else:
+                            yield param.name, getattr(self, param.name), param.default
+            except Exception as error:
+                raise ReprError(
+                    f"Failed to auto generate __rich_repr__; {error}"
+                ) from None
+
+        if not hasattr(cls, "__rich_repr__"):
+            auto_rich_repr.__doc__ = "Build a rich repr"
+            cls.__rich_repr__ = auto_rich_repr  # type: ignore[attr-defined]
+
+        auto_repr.__doc__ = "Return repr(self)"
+        cls.__repr__ = auto_repr  # type: ignore[assignment]
+        if angular is not None:
+            cls.__rich_repr__.angular = angular  # type: ignore[attr-defined]
+        return cls
+
+    if cls is None:
+        return partial(do_replace, angular=angular)
+    else:
+        return do_replace(cls, angular=angular)
+
+
+@overload
+def rich_repr(cls: Optional[Type[T]]) -> Type[T]:
+    ...
+
+
+@overload
+def rich_repr(*, angular: bool = False) -> Callable[[Type[T]], Type[T]]:
+    ...
+
+
+def rich_repr(
+    cls: Optional[Type[T]] = None, *, angular: bool = False
+) -> Union[Type[T], Callable[[Type[T]], Type[T]]]:
+    if cls is None:
+        return auto(angular=angular)
+    else:
+        return auto(cls)
+
+
+if __name__ == "__main__":
+
+    @auto
+    class Foo:
+        def __rich_repr__(self) -> Result:
+            yield "foo"
+            yield "bar", {"shopping": ["eggs", "ham", "pineapple"]}
+            yield "buy", "hand sanitizer"
+
+    foo = Foo()
+    from rich.console import Console
+
+    console = Console()
+
+    console.rule("Standard repr")
+    console.print(foo)
+
+    console.print(foo, width=60)
+    console.print(foo, width=30)
+
+    console.rule("Angular repr")
+    Foo.__rich_repr__.angular = True  # type: ignore[attr-defined]
+
+    console.print(foo)
+
+    console.print(foo, width=60)
+    console.print(foo, width=30)
diff --git a/lib/rich/rule.py b/lib/rich/rule.py
new file mode 100644
index 0000000..fb3d432
--- /dev/null
+++ b/lib/rich/rule.py
@@ -0,0 +1,130 @@
+from typing import Union
+
+from .align import AlignMethod
+from .cells import cell_len, set_cell_size
+from .console import Console, ConsoleOptions, RenderResult
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .style import Style
+from .text import Text
+
+
+class Rule(JupyterMixin):
+    """A console renderable to draw a horizontal rule (line).
+
+    Args:
+        title (Union[str, Text], optional): Text to render in the rule. Defaults to "".
+        characters (str, optional): Character(s) used to draw the line. Defaults to "─".
+        style (StyleType, optional): Style of Rule. Defaults to "rule.line".
+        end (str, optional): Character at end of Rule. defaults to "\\\\n"
+        align (str, optional): How to align the title, one of "left", "center", or "right". Defaults to "center".
+    """
+
+    def __init__(
+        self,
+        title: Union[str, Text] = "",
+        *,
+        characters: str = "─",
+        style: Union[str, Style] = "rule.line",
+        end: str = "\n",
+        align: AlignMethod = "center",
+    ) -> None:
+        if cell_len(characters) < 1:
+            raise ValueError(
+                "'characters' argument must have a cell width of at least 1"
+            )
+        if align not in ("left", "center", "right"):
+            raise ValueError(
+                f'invalid value for align, expected "left", "center", "right" (not {align!r})'
+            )
+        self.title = title
+        self.characters = characters
+        self.style = style
+        self.end = end
+        self.align = align
+
+    def __repr__(self) -> str:
+        return f"Rule({self.title!r}, {self.characters!r})"
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        width = options.max_width
+
+        characters = (
+            "-"
+            if (options.ascii_only and not self.characters.isascii())
+            else self.characters
+        )
+
+        chars_len = cell_len(characters)
+        if not self.title:
+            yield self._rule_line(chars_len, width)
+            return
+
+        if isinstance(self.title, Text):
+            title_text = self.title
+        else:
+            title_text = console.render_str(self.title, style="rule.text")
+
+        title_text.plain = title_text.plain.replace("\n", " ")
+        title_text.expand_tabs()
+
+        required_space = 4 if self.align == "center" else 2
+        truncate_width = max(0, width - required_space)
+        if not truncate_width:
+            yield self._rule_line(chars_len, width)
+            return
+
+        rule_text = Text(end=self.end)
+        if self.align == "center":
+            title_text.truncate(truncate_width, overflow="ellipsis")
+            side_width = (width - cell_len(title_text.plain)) // 2
+            left = Text(characters * (side_width // chars_len + 1))
+            left.truncate(side_width - 1)
+            right_length = width - cell_len(left.plain) - cell_len(title_text.plain)
+            right = Text(characters * (side_width // chars_len + 1))
+            right.truncate(right_length)
+            rule_text.append(left.plain + " ", self.style)
+            rule_text.append(title_text)
+            rule_text.append(" " + right.plain, self.style)
+        elif self.align == "left":
+            title_text.truncate(truncate_width, overflow="ellipsis")
+            rule_text.append(title_text)
+            rule_text.append(" ")
+            rule_text.append(characters * (width - rule_text.cell_len), self.style)
+        elif self.align == "right":
+            title_text.truncate(truncate_width, overflow="ellipsis")
+            rule_text.append(characters * (width - title_text.cell_len - 1), self.style)
+            rule_text.append(" ")
+            rule_text.append(title_text)
+
+        rule_text.plain = set_cell_size(rule_text.plain, width)
+        yield rule_text
+
+    def _rule_line(self, chars_len: int, width: int) -> Text:
+        rule_text = Text(self.characters * ((width // chars_len) + 1), self.style)
+        rule_text.truncate(width)
+        rule_text.plain = set_cell_size(rule_text.plain, width)
+        return rule_text
+
+    def __rich_measure__(
+        self, console: Console, options: ConsoleOptions
+    ) -> Measurement:
+        return Measurement(1, 1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import sys
+
+    from rich.console import Console
+
+    try:
+        text = sys.argv[1]
+    except IndexError:
+        text = "Hello, World"
+    console = Console()
+    console.print(Rule(title=text))
+
+    console = Console()
+    console.print(Rule("foo"), width=4)
diff --git a/lib/rich/scope.py b/lib/rich/scope.py
new file mode 100644
index 0000000..41d0299
--- /dev/null
+++ b/lib/rich/scope.py
@@ -0,0 +1,92 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Optional, Tuple
+
+from .highlighter import ReprHighlighter
+from .panel import Panel
+from .pretty import Pretty
+from .table import Table
+from .text import Text, TextType
+
+if TYPE_CHECKING:
+    from .console import ConsoleRenderable, OverflowMethod
+
+
+def render_scope(
+    scope: "Mapping[str, Any]",
+    *,
+    title: Optional[TextType] = None,
+    sort_keys: bool = True,
+    indent_guides: bool = False,
+    max_length: Optional[int] = None,
+    max_string: Optional[int] = None,
+    max_depth: Optional[int] = None,
+    overflow: Optional["OverflowMethod"] = None,
+) -> "ConsoleRenderable":
+    """Render python variables in a given scope.
+
+    Args:
+        scope (Mapping): A mapping containing variable names and values.
+        title (str, optional): Optional title. Defaults to None.
+        sort_keys (bool, optional): Enable sorting of items. Defaults to True.
+        indent_guides (bool, optional): Enable indentation guides. Defaults to False.
+        max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to None.
+        max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to None.
+        max_depth (int, optional): Maximum depths of locals before truncating, or None to disable. Defaults to None.
+        overflow (OverflowMethod, optional): How to handle overflowing locals, or None to disable. Defaults to None.
+
+    Returns:
+        ConsoleRenderable: A renderable object.
+    """
+    highlighter = ReprHighlighter()
+    items_table = Table.grid(padding=(0, 1), expand=False)
+    items_table.add_column(justify="right")
+
+    def sort_items(item: Tuple[str, Any]) -> Tuple[bool, str]:
+        """Sort special variables first, then alphabetically."""
+        key, _ = item
+        return (not key.startswith("__"), key.lower())
+
+    items = sorted(scope.items(), key=sort_items) if sort_keys else scope.items()
+    for key, value in items:
+        key_text = Text.assemble(
+            (key, "scope.key.special" if key.startswith("__") else "scope.key"),
+            (" =", "scope.equals"),
+        )
+        items_table.add_row(
+            key_text,
+            Pretty(
+                value,
+                highlighter=highlighter,
+                indent_guides=indent_guides,
+                max_length=max_length,
+                max_string=max_string,
+                max_depth=max_depth,
+                overflow=overflow,
+            ),
+        )
+    return Panel.fit(
+        items_table,
+        title=title,
+        border_style="scope.border",
+        padding=(0, 1),
+    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich import print
+
+    print()
+
+    def test(foo: float, bar: float) -> None:
+        list_of_things = [1, 2, 3, None, 4, True, False, "Hello World"]
+        dict_of_things = {
+            "version": "1.1",
+            "method": "confirmFruitPurchase",
+            "params": [["apple", "orange", "mangoes", "pomelo"], 1.123],
+            "id": "194521489",
+        }
+        print(render_scope(locals(), title="[i]locals", sort_keys=False))
+
+    test(20.3423, 3.1427)
+    print()
diff --git a/lib/rich/screen.py b/lib/rich/screen.py
new file mode 100644
index 0000000..b4f7fd1
--- /dev/null
+++ b/lib/rich/screen.py
@@ -0,0 +1,54 @@
+from typing import Optional, TYPE_CHECKING
+
+from .segment import Segment
+from .style import StyleType
+from ._loop import loop_last
+
+
+if TYPE_CHECKING:
+    from .console import (
+        Console,
+        ConsoleOptions,
+        RenderResult,
+        RenderableType,
+        Group,
+    )
+
+
+class Screen:
+    """A renderable that fills the terminal screen and crops excess.
+
+    Args:
+        renderable (RenderableType): Child renderable.
+        style (StyleType, optional): Optional background style. Defaults to None.
+    """
+
+    renderable: "RenderableType"
+
+    def __init__(
+        self,
+        *renderables: "RenderableType",
+        style: Optional[StyleType] = None,
+        application_mode: bool = False,
+    ) -> None:
+        from rich.console import Group
+
+        self.renderable = Group(*renderables)
+        self.style = style
+        self.application_mode = application_mode
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        width, height = options.size
+        style = console.get_style(self.style) if self.style else None
+        render_options = options.update(width=width, height=height)
+        lines = console.render_lines(
+            self.renderable or "", render_options, style=style, pad=True
+        )
+        lines = Segment.set_shape(lines, width, height, style=style)
+        new_line = Segment("\n\r") if self.application_mode else Segment.line()
+        for last, line in loop_last(lines):
+            yield from line
+            if not last:
+                yield new_line
diff --git a/lib/rich/segment.py b/lib/rich/segment.py
new file mode 100644
index 0000000..0df63fd
--- /dev/null
+++ b/lib/rich/segment.py
@@ -0,0 +1,783 @@
+from enum import IntEnum
+from functools import lru_cache
+from itertools import filterfalse
+from logging import getLogger
+from operator import attrgetter
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
+
+from .cells import (
+    _is_single_cell_widths,
+    cached_cell_len,
+    cell_len,
+    get_character_cell_size,
+    set_cell_size,
+)
+from .repr import Result, rich_repr
+from .style import Style
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderResult
+
+log = getLogger("rich")
+
+
+class ControlType(IntEnum):
+    """Non-printable control codes which typically translate to ANSI codes."""
+
+    BELL = 1
+    CARRIAGE_RETURN = 2
+    HOME = 3
+    CLEAR = 4
+    SHOW_CURSOR = 5
+    HIDE_CURSOR = 6
+    ENABLE_ALT_SCREEN = 7
+    DISABLE_ALT_SCREEN = 8
+    CURSOR_UP = 9
+    CURSOR_DOWN = 10
+    CURSOR_FORWARD = 11
+    CURSOR_BACKWARD = 12
+    CURSOR_MOVE_TO_COLUMN = 13
+    CURSOR_MOVE_TO = 14
+    ERASE_IN_LINE = 15
+    SET_WINDOW_TITLE = 16
+
+
+ControlCode = Union[
+    Tuple[ControlType],
+    Tuple[ControlType, Union[int, str]],
+    Tuple[ControlType, int, int],
+]
+
+
+@rich_repr()
+class Segment(NamedTuple):
+    """A piece of text with associated style. Segments are produced by the Console render process and
+    are ultimately converted in to strings to be written to the terminal.
+
+    Args:
+        text (str): A piece of text.
+        style (:class:`~rich.style.Style`, optional): An optional style to apply to the text.
+        control (Tuple[ControlCode], optional): Optional sequence of control codes.
+
+    Attributes:
+        cell_length (int): The cell length of this Segment.
+    """
+
+    text: str
+    style: Optional[Style] = None
+    control: Optional[Sequence[ControlCode]] = None
+
+    @property
+    def cell_length(self) -> int:
+        """The number of terminal cells required to display self.text.
+
+        Returns:
+            int: A number of cells.
+        """
+        text, _style, control = self
+        return 0 if control else cell_len(text)
+
+    def __rich_repr__(self) -> Result:
+        yield self.text
+        if self.control is None:
+            if self.style is not None:
+                yield self.style
+        else:
+            yield self.style
+            yield self.control
+
+    def __bool__(self) -> bool:
+        """Check if the segment contains text."""
+        return bool(self.text)
+
+    @property
+    def is_control(self) -> bool:
+        """Check if the segment contains control codes."""
+        return self.control is not None
+
+    @classmethod
+    @lru_cache(1024 * 16)
+    def _split_cells(cls, segment: "Segment", cut: int) -> Tuple["Segment", "Segment"]:
+        """Split a segment in to two at a given cell position.
+
+        Note that splitting a double-width character, may result in that character turning
+        into two spaces.
+
+        Args:
+            segment (Segment): A segment to split.
+            cut (int): A cell position to cut on.
+
+        Returns:
+            A tuple of two segments.
+        """
+        text, style, control = segment
+        _Segment = Segment
+        cell_length = segment.cell_length
+        if cut >= cell_length:
+            return segment, _Segment("", style, control)
+
+        cell_size = get_character_cell_size
+
+        pos = int((cut / cell_length) * len(text))
+
+        while True:
+            before = text[:pos]
+            cell_pos = cell_len(before)
+            out_by = cell_pos - cut
+            if not out_by:
+                return (
+                    _Segment(before, style, control),
+                    _Segment(text[pos:], style, control),
+                )
+            if out_by == -1 and cell_size(text[pos]) == 2:
+                return (
+                    _Segment(text[:pos] + " ", style, control),
+                    _Segment(" " + text[pos + 1 :], style, control),
+                )
+            if out_by == +1 and cell_size(text[pos - 1]) == 2:
+                return (
+                    _Segment(text[: pos - 1] + " ", style, control),
+                    _Segment(" " + text[pos:], style, control),
+                )
+            if cell_pos < cut:
+                pos += 1
+            else:
+                pos -= 1
+
+    def split_cells(self, cut: int) -> Tuple["Segment", "Segment"]:
+        """Split segment in to two segments at the specified column.
+
+        If the cut point falls in the middle of a 2-cell wide character then it is replaced
+        by two spaces, to preserve the display width of the parent segment.
+
+        Args:
+            cut (int): Offset within the segment to cut.
+
+        Returns:
+            Tuple[Segment, Segment]: Two segments.
+        """
+        text, style, control = self
+        assert cut >= 0
+
+        if _is_single_cell_widths(text):
+            # Fast path with all 1 cell characters
+            if cut >= len(text):
+                return self, Segment("", style, control)
+            return (
+                Segment(text[:cut], style, control),
+                Segment(text[cut:], style, control),
+            )
+
+        return self._split_cells(self, cut)
+
+    @classmethod
+    def line(cls) -> "Segment":
+        """Make a new line segment."""
+        return cls("\n")
+
+    @classmethod
+    def apply_style(
+        cls,
+        segments: Iterable["Segment"],
+        style: Optional[Style] = None,
+        post_style: Optional[Style] = None,
+    ) -> Iterable["Segment"]:
+        """Apply style(s) to an iterable of segments.
+
+        Returns an iterable of segments where the style is replaced by ``style + segment.style + post_style``.
+
+        Args:
+            segments (Iterable[Segment]): Segments to process.
+            style (Style, optional): Base style. Defaults to None.
+            post_style (Style, optional): Style to apply on top of segment style. Defaults to None.
+
+        Returns:
+            Iterable[Segments]: A new iterable of segments (possibly the same iterable).
+        """
+        result_segments = segments
+        if style:
+            apply = style.__add__
+            result_segments = (
+                cls(text, None if control else apply(_style), control)
+                for text, _style, control in result_segments
+            )
+        if post_style:
+            result_segments = (
+                cls(
+                    text,
+                    (
+                        None
+                        if control
+                        else (_style + post_style if _style else post_style)
+                    ),
+                    control,
+                )
+                for text, _style, control in result_segments
+            )
+        return result_segments
+
+    @classmethod
+    def filter_control(
+        cls, segments: Iterable["Segment"], is_control: bool = False
+    ) -> Iterable["Segment"]:
+        """Filter segments by ``is_control`` attribute.
+
+        Args:
+            segments (Iterable[Segment]): An iterable of Segment instances.
+            is_control (bool, optional): is_control flag to match in search.
+
+        Returns:
+            Iterable[Segment]: And iterable of Segment instances.
+
+        """
+        if is_control:
+            return filter(attrgetter("control"), segments)
+        else:
+            return filterfalse(attrgetter("control"), segments)
+
+    @classmethod
+    def split_lines(cls, segments: Iterable["Segment"]) -> Iterable[List["Segment"]]:
+        """Split a sequence of segments in to a list of lines.
+
+        Args:
+            segments (Iterable[Segment]): Segments potentially containing line feeds.
+
+        Yields:
+            Iterable[List[Segment]]: Iterable of segment lists, one per line.
+        """
+        line: List[Segment] = []
+        append = line.append
+
+        for segment in segments:
+            if "\n" in segment.text and not segment.control:
+                text, style, _ = segment
+                while text:
+                    _text, new_line, text = text.partition("\n")
+                    if _text:
+                        append(cls(_text, style))
+                    if new_line:
+                        yield line
+                        line = []
+                        append = line.append
+            else:
+                append(segment)
+        if line:
+            yield line
+
+    @classmethod
+    def split_lines_terminator(
+        cls, segments: Iterable["Segment"]
+    ) -> Iterable[Tuple[List["Segment"], bool]]:
+        """Split a sequence of segments in to a list of lines and a boolean to indicate if there was a new line.
+
+        Args:
+            segments (Iterable[Segment]): Segments potentially containing line feeds.
+
+        Yields:
+            Iterable[List[Segment]]: Iterable of segment lists, one per line.
+        """
+        line: List[Segment] = []
+        append = line.append
+
+        for segment in segments:
+            if "\n" in segment.text and not segment.control:
+                text, style, _ = segment
+                while text:
+                    _text, new_line, text = text.partition("\n")
+                    if _text:
+                        append(cls(_text, style))
+                    if new_line:
+                        yield (line, True)
+                        line = []
+                        append = line.append
+            else:
+                append(segment)
+        if line:
+            yield (line, False)
+
+    @classmethod
+    def split_and_crop_lines(
+        cls,
+        segments: Iterable["Segment"],
+        length: int,
+        style: Optional[Style] = None,
+        pad: bool = True,
+        include_new_lines: bool = True,
+    ) -> Iterable[List["Segment"]]:
+        """Split segments in to lines, and crop lines greater than a given length.
+
+        Args:
+            segments (Iterable[Segment]): An iterable of segments, probably
+                generated from console.render.
+            length (int): Desired line length.
+            style (Style, optional): Style to use for any padding.
+            pad (bool): Enable padding of lines that are less than `length`.
+
+        Returns:
+            Iterable[List[Segment]]: An iterable of lines of segments.
+        """
+        line: List[Segment] = []
+        append = line.append
+
+        adjust_line_length = cls.adjust_line_length
+        new_line_segment = cls("\n")
+
+        for segment in segments:
+            if "\n" in segment.text and not segment.control:
+                text, segment_style, _ = segment
+                while text:
+                    _text, new_line, text = text.partition("\n")
+                    if _text:
+                        append(cls(_text, segment_style))
+                    if new_line:
+                        cropped_line = adjust_line_length(
+                            line, length, style=style, pad=pad
+                        )
+                        if include_new_lines:
+                            cropped_line.append(new_line_segment)
+                        yield cropped_line
+                        line.clear()
+            else:
+                append(segment)
+        if line:
+            yield adjust_line_length(line, length, style=style, pad=pad)
+
+    @classmethod
+    def adjust_line_length(
+        cls,
+        line: List["Segment"],
+        length: int,
+        style: Optional[Style] = None,
+        pad: bool = True,
+    ) -> List["Segment"]:
+        """Adjust a line to a given width (cropping or padding as required).
+
+        Args:
+            segments (Iterable[Segment]): A list of segments in a single line.
+            length (int): The desired width of the line.
+            style (Style, optional): The style of padding if used (space on the end). Defaults to None.
+            pad (bool, optional): Pad lines with spaces if they are shorter than `length`. Defaults to True.
+
+        Returns:
+            List[Segment]: A line of segments with the desired length.
+        """
+        line_length = sum(segment.cell_length for segment in line)
+        new_line: List[Segment]
+
+        if line_length < length:
+            if pad:
+                new_line = line + [cls(" " * (length - line_length), style)]
+            else:
+                new_line = line[:]
+        elif line_length > length:
+            new_line = []
+            append = new_line.append
+            line_length = 0
+            for segment in line:
+                segment_length = segment.cell_length
+                if line_length + segment_length < length or segment.control:
+                    append(segment)
+                    line_length += segment_length
+                else:
+                    text, segment_style, _ = segment
+                    text = set_cell_size(text, length - line_length)
+                    append(cls(text, segment_style))
+                    break
+        else:
+            new_line = line[:]
+        return new_line
+
+    @classmethod
+    def get_line_length(cls, line: List["Segment"]) -> int:
+        """Get the length of list of segments.
+
+        Args:
+            line (List[Segment]): A line encoded as a list of Segments (assumes no '\\\\n' characters),
+
+        Returns:
+            int: The length of the line.
+        """
+        _cell_len = cell_len
+        return sum(_cell_len(text) for text, style, control in line if not control)
+
+    @classmethod
+    def get_shape(cls, lines: List[List["Segment"]]) -> Tuple[int, int]:
+        """Get the shape (enclosing rectangle) of a list of lines.
+
+        Args:
+            lines (List[List[Segment]]): A list of lines (no '\\\\n' characters).
+
+        Returns:
+            Tuple[int, int]: Width and height in characters.
+        """
+        get_line_length = cls.get_line_length
+        max_width = max(get_line_length(line) for line in lines) if lines else 0
+        return (max_width, len(lines))
+
+    @classmethod
+    def set_shape(
+        cls,
+        lines: List[List["Segment"]],
+        width: int,
+        height: Optional[int] = None,
+        style: Optional[Style] = None,
+        new_lines: bool = False,
+    ) -> List[List["Segment"]]:
+        """Set the shape of a list of lines (enclosing rectangle).
+
+        Args:
+            lines (List[List[Segment]]): A list of lines.
+            width (int): Desired width.
+            height (int, optional): Desired height or None for no change.
+            style (Style, optional): Style of any padding added.
+            new_lines (bool, optional): Padded lines should include "\n". Defaults to False.
+
+        Returns:
+            List[List[Segment]]: New list of lines.
+        """
+        _height = height or len(lines)
+
+        blank = (
+            [cls(" " * width + "\n", style)] if new_lines else [cls(" " * width, style)]
+        )
+
+        adjust_line_length = cls.adjust_line_length
+        shaped_lines = lines[:_height]
+        shaped_lines[:] = [
+            adjust_line_length(line, width, style=style) for line in lines
+        ]
+        if len(shaped_lines) < _height:
+            shaped_lines.extend([blank] * (_height - len(shaped_lines)))
+        return shaped_lines
+
+    @classmethod
+    def align_top(
+        cls: Type["Segment"],
+        lines: List[List["Segment"]],
+        width: int,
+        height: int,
+        style: Style,
+        new_lines: bool = False,
+    ) -> List[List["Segment"]]:
+        """Aligns lines to top (adds extra lines to bottom as required).
+
+        Args:
+            lines (List[List[Segment]]): A list of lines.
+            width (int): Desired width.
+            height (int, optional): Desired height or None for no change.
+            style (Style): Style of any padding added.
+            new_lines (bool, optional): Padded lines should include "\n". Defaults to False.
+
+        Returns:
+            List[List[Segment]]: New list of lines.
+        """
+        extra_lines = height - len(lines)
+        if not extra_lines:
+            return lines[:]
+        lines = lines[:height]
+        blank = cls(" " * width + "\n", style) if new_lines else cls(" " * width, style)
+        lines = lines + [[blank]] * extra_lines
+        return lines
+
+    @classmethod
+    def align_bottom(
+        cls: Type["Segment"],
+        lines: List[List["Segment"]],
+        width: int,
+        height: int,
+        style: Style,
+        new_lines: bool = False,
+    ) -> List[List["Segment"]]:
+        """Aligns render to bottom (adds extra lines above as required).
+
+        Args:
+            lines (List[List[Segment]]): A list of lines.
+            width (int): Desired width.
+            height (int, optional): Desired height or None for no change.
+            style (Style): Style of any padding added. Defaults to None.
+            new_lines (bool, optional): Padded lines should include "\n". Defaults to False.
+
+        Returns:
+            List[List[Segment]]: New list of lines.
+        """
+        extra_lines = height - len(lines)
+        if not extra_lines:
+            return lines[:]
+        lines = lines[:height]
+        blank = cls(" " * width + "\n", style) if new_lines else cls(" " * width, style)
+        lines = [[blank]] * extra_lines + lines
+        return lines
+
+    @classmethod
+    def align_middle(
+        cls: Type["Segment"],
+        lines: List[List["Segment"]],
+        width: int,
+        height: int,
+        style: Style,
+        new_lines: bool = False,
+    ) -> List[List["Segment"]]:
+        """Aligns lines to middle (adds extra lines to above and below as required).
+
+        Args:
+            lines (List[List[Segment]]): A list of lines.
+            width (int): Desired width.
+            height (int, optional): Desired height or None for no change.
+            style (Style): Style of any padding added.
+            new_lines (bool, optional): Padded lines should include "\n". Defaults to False.
+
+        Returns:
+            List[List[Segment]]: New list of lines.
+        """
+        extra_lines = height - len(lines)
+        if not extra_lines:
+            return lines[:]
+        lines = lines[:height]
+        blank = cls(" " * width + "\n", style) if new_lines else cls(" " * width, style)
+        top_lines = extra_lines // 2
+        bottom_lines = extra_lines - top_lines
+        lines = [[blank]] * top_lines + lines + [[blank]] * bottom_lines
+        return lines
+
+    @classmethod
+    def simplify(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]:
+        """Simplify an iterable of segments by combining contiguous segments with the same style.
+
+        Args:
+            segments (Iterable[Segment]): An iterable of segments.
+
+        Returns:
+            Iterable[Segment]: A possibly smaller iterable of segments that will render the same way.
+        """
+        iter_segments = iter(segments)
+        try:
+            last_segment = next(iter_segments)
+        except StopIteration:
+            return
+
+        _Segment = Segment
+        for segment in iter_segments:
+            if last_segment.style == segment.style and not segment.control:
+                last_segment = _Segment(
+                    last_segment.text + segment.text, last_segment.style
+                )
+            else:
+                yield last_segment
+                last_segment = segment
+        yield last_segment
+
+    @classmethod
+    def strip_links(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]:
+        """Remove all links from an iterable of styles.
+
+        Args:
+            segments (Iterable[Segment]): An iterable segments.
+
+        Yields:
+            Segment: Segments with link removed.
+        """
+        for segment in segments:
+            if segment.control or segment.style is None:
+                yield segment
+            else:
+                text, style, _control = segment
+                yield cls(text, style.update_link(None) if style else None)
+
+    @classmethod
+    def strip_styles(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]:
+        """Remove all styles from an iterable of segments.
+
+        Args:
+            segments (Iterable[Segment]): An iterable segments.
+
+        Yields:
+            Segment: Segments with styles replace with None
+        """
+        for text, _style, control in segments:
+            yield cls(text, None, control)
+
+    @classmethod
+    def remove_color(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]:
+        """Remove all color from an iterable of segments.
+
+        Args:
+            segments (Iterable[Segment]): An iterable segments.
+
+        Yields:
+            Segment: Segments with colorless style.
+        """
+
+        cache: Dict[Style, Style] = {}
+        for text, style, control in segments:
+            if style:
+                colorless_style = cache.get(style)
+                if colorless_style is None:
+                    colorless_style = style.without_color
+                    cache[style] = colorless_style
+                yield cls(text, colorless_style, control)
+            else:
+                yield cls(text, None, control)
+
+    @classmethod
+    def divide(
+        cls, segments: Iterable["Segment"], cuts: Iterable[int]
+    ) -> Iterable[List["Segment"]]:
+        """Divides an iterable of segments in to portions.
+
+        Args:
+            cuts (Iterable[int]): Cell positions where to divide.
+
+        Yields:
+            [Iterable[List[Segment]]]: An iterable of Segments in List.
+        """
+        split_segments: List["Segment"] = []
+        add_segment = split_segments.append
+
+        iter_cuts = iter(cuts)
+
+        while True:
+            cut = next(iter_cuts, -1)
+            if cut == -1:
+                return
+            if cut != 0:
+                break
+            yield []
+        pos = 0
+
+        segments_clear = split_segments.clear
+        segments_copy = split_segments.copy
+
+        _cell_len = cached_cell_len
+        for segment in segments:
+            text, _style, control = segment
+            while text:
+                end_pos = pos if control else pos + _cell_len(text)
+                if end_pos < cut:
+                    add_segment(segment)
+                    pos = end_pos
+                    break
+
+                if end_pos == cut:
+                    add_segment(segment)
+                    yield segments_copy()
+                    segments_clear()
+                    pos = end_pos
+
+                    cut = next(iter_cuts, -1)
+                    if cut == -1:
+                        if split_segments:
+                            yield segments_copy()
+                        return
+
+                    break
+
+                else:
+                    before, segment = segment.split_cells(cut - pos)
+                    text, _style, control = segment
+                    add_segment(before)
+                    yield segments_copy()
+                    segments_clear()
+                    pos = cut
+
+                cut = next(iter_cuts, -1)
+                if cut == -1:
+                    if split_segments:
+                        yield segments_copy()
+                    return
+
+        yield segments_copy()
+
+
+class Segments:
+    """A simple renderable to render an iterable of segments. This class may be useful if
+    you want to print segments outside of a __rich_console__ method.
+
+    Args:
+        segments (Iterable[Segment]): An iterable of segments.
+        new_lines (bool, optional): Add new lines between segments. Defaults to False.
+    """
+
+    def __init__(self, segments: Iterable[Segment], new_lines: bool = False) -> None:
+        self.segments = list(segments)
+        self.new_lines = new_lines
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        if self.new_lines:
+            line = Segment.line()
+            for segment in self.segments:
+                yield segment
+                yield line
+        else:
+            yield from self.segments
+
+
+class SegmentLines:
+    def __init__(self, lines: Iterable[List[Segment]], new_lines: bool = False) -> None:
+        """A simple renderable containing a number of lines of segments. May be used as an intermediate
+        in rendering process.
+
+        Args:
+            lines (Iterable[List[Segment]]): Lists of segments forming lines.
+            new_lines (bool, optional): Insert new lines after each line. Defaults to False.
+        """
+        self.lines = list(lines)
+        self.new_lines = new_lines
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        if self.new_lines:
+            new_line = Segment.line()
+            for line in self.lines:
+                yield from line
+                yield new_line
+        else:
+            for line in self.lines:
+                yield from line
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich.console import Console
+    from rich.syntax import Syntax
+    from rich.text import Text
+
+    code = """from rich.console import Console
+console = Console()
+text = Text.from_markup("Hello, [bold magenta]World[/]!")
+console.print(text)"""
+
+    text = Text.from_markup("Hello, [bold magenta]World[/]!")
+
+    console = Console()
+
+    console.rule("rich.Segment")
+    console.print(
+        "A Segment is the last step in the Rich render process before generating text with ANSI codes."
+    )
+    console.print("\nConsider the following code:\n")
+    console.print(Syntax(code, "python", line_numbers=True))
+    console.print()
+    console.print(
+        "When you call [b]print()[/b], Rich [i]renders[/i] the object in to the following:\n"
+    )
+    fragments = list(console.render(text))
+    console.print(fragments)
+    console.print()
+    console.print("The Segments are then processed to produce the following output:\n")
+    console.print(text)
+    console.print(
+        "\nYou will only need to know this if you are implementing your own Rich renderables."
+    )
diff --git a/lib/rich/spinner.py b/lib/rich/spinner.py
new file mode 100644
index 0000000..a3a3caf
--- /dev/null
+++ b/lib/rich/spinner.py
@@ -0,0 +1,132 @@
+from typing import TYPE_CHECKING, List, Optional, Union, cast
+
+from ._spinners import SPINNERS
+from .measure import Measurement
+from .table import Table
+from .text import Text
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderableType, RenderResult
+    from .style import StyleType
+
+
+class Spinner:
+    """A spinner animation.
+
+    Args:
+        name (str): Name of spinner (run python -m rich.spinner).
+        text (RenderableType, optional): A renderable to display at the right of the spinner (str or Text typically). Defaults to "".
+        style (StyleType, optional): Style for spinner animation. Defaults to None.
+        speed (float, optional): Speed factor for animation. Defaults to 1.0.
+
+    Raises:
+        KeyError: If name isn't one of the supported spinner animations.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        text: "RenderableType" = "",
+        *,
+        style: Optional["StyleType"] = None,
+        speed: float = 1.0,
+    ) -> None:
+        try:
+            spinner = SPINNERS[name]
+        except KeyError:
+            raise KeyError(f"no spinner called {name!r}")
+        self.text: "Union[RenderableType, Text]" = (
+            Text.from_markup(text) if isinstance(text, str) else text
+        )
+        self.name = name
+        self.frames = cast(List[str], spinner["frames"])[:]
+        self.interval = cast(float, spinner["interval"])
+        self.start_time: Optional[float] = None
+        self.style = style
+        self.speed = speed
+        self.frame_no_offset: float = 0.0
+        self._update_speed = 0.0
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        yield self.render(console.get_time())
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> Measurement:
+        text = self.render(0)
+        return Measurement.get(console, options, text)
+
+    def render(self, time: float) -> "RenderableType":
+        """Render the spinner for a given time.
+
+        Args:
+            time (float): Time in seconds.
+
+        Returns:
+            RenderableType: A renderable containing animation frame.
+        """
+        if self.start_time is None:
+            self.start_time = time
+
+        frame_no = ((time - self.start_time) * self.speed) / (
+            self.interval / 1000.0
+        ) + self.frame_no_offset
+        frame = Text(
+            self.frames[int(frame_no) % len(self.frames)], style=self.style or ""
+        )
+
+        if self._update_speed:
+            self.frame_no_offset = frame_no
+            self.start_time = time
+            self.speed = self._update_speed
+            self._update_speed = 0.0
+
+        if not self.text:
+            return frame
+        elif isinstance(self.text, (str, Text)):
+            return Text.assemble(frame, " ", self.text)
+        else:
+            table = Table.grid(padding=1)
+            table.add_row(frame, self.text)
+            return table
+
+    def update(
+        self,
+        *,
+        text: "RenderableType" = "",
+        style: Optional["StyleType"] = None,
+        speed: Optional[float] = None,
+    ) -> None:
+        """Updates attributes of a spinner after it has been started.
+
+        Args:
+            text (RenderableType, optional): A renderable to display at the right of the spinner (str or Text typically). Defaults to "".
+            style (StyleType, optional): Style for spinner animation. Defaults to None.
+            speed (float, optional): Speed factor for animation. Defaults to None.
+        """
+        if text:
+            self.text = Text.from_markup(text) if isinstance(text, str) else text
+        if style:
+            self.style = style
+        if speed:
+            self._update_speed = speed
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from time import sleep
+
+    from .console import Group
+    from .live import Live
+
+    all_spinners = Group(
+        *[
+            Spinner(spinner_name, text=Text(repr(spinner_name), style="green"))
+            for spinner_name in sorted(SPINNERS.keys())
+        ]
+    )
+
+    with Live(all_spinners, refresh_per_second=20) as live:
+        while True:
+            sleep(0.1)
diff --git a/lib/rich/status.py b/lib/rich/status.py
new file mode 100644
index 0000000..6574483
--- /dev/null
+++ b/lib/rich/status.py
@@ -0,0 +1,131 @@
+from types import TracebackType
+from typing import Optional, Type
+
+from .console import Console, RenderableType
+from .jupyter import JupyterMixin
+from .live import Live
+from .spinner import Spinner
+from .style import StyleType
+
+
+class Status(JupyterMixin):
+    """Displays a status indicator with a 'spinner' animation.
+
+    Args:
+        status (RenderableType): A status renderable (str or Text typically).
+        console (Console, optional): Console instance to use, or None for global console. Defaults to None.
+        spinner (str, optional): Name of spinner animation (see python -m rich.spinner). Defaults to "dots".
+        spinner_style (StyleType, optional): Style of spinner. Defaults to "status.spinner".
+        speed (float, optional): Speed factor for spinner animation. Defaults to 1.0.
+        refresh_per_second (float, optional): Number of refreshes per second. Defaults to 12.5.
+    """
+
+    def __init__(
+        self,
+        status: RenderableType,
+        *,
+        console: Optional[Console] = None,
+        spinner: str = "dots",
+        spinner_style: StyleType = "status.spinner",
+        speed: float = 1.0,
+        refresh_per_second: float = 12.5,
+    ):
+        self.status = status
+        self.spinner_style = spinner_style
+        self.speed = speed
+        self._spinner = Spinner(spinner, text=status, style=spinner_style, speed=speed)
+        self._live = Live(
+            self.renderable,
+            console=console,
+            refresh_per_second=refresh_per_second,
+            transient=True,
+        )
+
+    @property
+    def renderable(self) -> Spinner:
+        return self._spinner
+
+    @property
+    def console(self) -> "Console":
+        """Get the Console used by the Status objects."""
+        return self._live.console
+
+    def update(
+        self,
+        status: Optional[RenderableType] = None,
+        *,
+        spinner: Optional[str] = None,
+        spinner_style: Optional[StyleType] = None,
+        speed: Optional[float] = None,
+    ) -> None:
+        """Update status.
+
+        Args:
+            status (Optional[RenderableType], optional): New status renderable or None for no change. Defaults to None.
+            spinner (Optional[str], optional): New spinner or None for no change. Defaults to None.
+            spinner_style (Optional[StyleType], optional): New spinner style or None for no change. Defaults to None.
+            speed (Optional[float], optional): Speed factor for spinner animation or None for no change. Defaults to None.
+        """
+        if status is not None:
+            self.status = status
+        if spinner_style is not None:
+            self.spinner_style = spinner_style
+        if speed is not None:
+            self.speed = speed
+        if spinner is not None:
+            self._spinner = Spinner(
+                spinner, text=self.status, style=self.spinner_style, speed=self.speed
+            )
+            self._live.update(self.renderable, refresh=True)
+        else:
+            self._spinner.update(
+                text=self.status, style=self.spinner_style, speed=self.speed
+            )
+
+    def start(self) -> None:
+        """Start the status animation."""
+        self._live.start()
+
+    def stop(self) -> None:
+        """Stop the spinner animation."""
+        self._live.stop()
+
+    def __rich__(self) -> RenderableType:
+        return self.renderable
+
+    def __enter__(self) -> "Status":
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.stop()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from time import sleep
+
+    from .console import Console
+
+    console = Console()
+    with console.status("[magenta]Covid detector booting up") as status:
+        sleep(3)
+        console.log("Importing advanced AI")
+        sleep(3)
+        console.log("Advanced Covid AI Ready")
+        sleep(3)
+        status.update(status="[bold blue] Scanning for Covid", spinner="earth")
+        sleep(3)
+        console.log("Found 10,000,000,000 copies of Covid32.exe")
+        sleep(3)
+        status.update(
+            status="[bold red]Moving Covid32.exe to Trash",
+            spinner="bouncingBall",
+            spinner_style="yellow",
+        )
+        sleep(5)
+    console.print("[bold green]Covid deleted successfully")
diff --git a/lib/rich/style.py b/lib/rich/style.py
new file mode 100644
index 0000000..5294241
--- /dev/null
+++ b/lib/rich/style.py
@@ -0,0 +1,792 @@
+import sys
+from functools import lru_cache
+from operator import attrgetter
+from pickle import dumps, loads
+from random import randint
+from typing import Any, Dict, Iterable, List, Optional, Type, Union, cast
+
+from . import errors
+from .color import Color, ColorParseError, ColorSystem, blend_rgb
+from .repr import Result, rich_repr
+from .terminal_theme import DEFAULT_TERMINAL_THEME, TerminalTheme
+
+_hash_getter = attrgetter(
+    "_color", "_bgcolor", "_attributes", "_set_attributes", "_link", "_meta"
+)
+
+# Style instances and style definitions are often interchangeable
+StyleType = Union[str, "Style"]
+
+
+class _Bit:
+    """A descriptor to get/set a style attribute bit."""
+
+    __slots__ = ["bit"]
+
+    def __init__(self, bit_no: int) -> None:
+        self.bit = 1 << bit_no
+
+    def __get__(self, obj: "Style", objtype: Type["Style"]) -> Optional[bool]:
+        if obj._set_attributes & self.bit:
+            return obj._attributes & self.bit != 0
+        return None
+
+
+@rich_repr
+class Style:
+    """A terminal style.
+
+    A terminal style consists of a color (`color`), a background color (`bgcolor`), and a number of attributes, such
+    as bold, italic etc. The attributes have 3 states: they can either be on
+    (``True``), off (``False``), or not set (``None``).
+
+    Args:
+        color (Union[Color, str], optional): Color of terminal text. Defaults to None.
+        bgcolor (Union[Color, str], optional): Color of terminal background. Defaults to None.
+        bold (bool, optional): Enable bold text. Defaults to None.
+        dim (bool, optional): Enable dim text. Defaults to None.
+        italic (bool, optional): Enable italic text. Defaults to None.
+        underline (bool, optional): Enable underlined text. Defaults to None.
+        blink (bool, optional): Enabled blinking text. Defaults to None.
+        blink2 (bool, optional): Enable fast blinking text. Defaults to None.
+        reverse (bool, optional): Enabled reverse text. Defaults to None.
+        conceal (bool, optional): Enable concealed text. Defaults to None.
+        strike (bool, optional): Enable strikethrough text. Defaults to None.
+        underline2 (bool, optional): Enable doubly underlined text. Defaults to None.
+        frame (bool, optional): Enable framed text. Defaults to None.
+        encircle (bool, optional): Enable encircled text. Defaults to None.
+        overline (bool, optional): Enable overlined text. Defaults to None.
+        link (str, link): Link URL. Defaults to None.
+
+    """
+
+    _color: Optional[Color]
+    _bgcolor: Optional[Color]
+    _attributes: int
+    _set_attributes: int
+    _hash: Optional[int]
+    _null: bool
+    _meta: Optional[bytes]
+
+    __slots__ = [
+        "_color",
+        "_bgcolor",
+        "_attributes",
+        "_set_attributes",
+        "_link",
+        "_link_id",
+        "_ansi",
+        "_style_definition",
+        "_hash",
+        "_null",
+        "_meta",
+    ]
+
+    # maps bits on to SGR parameter
+    _style_map = {
+        0: "1",
+        1: "2",
+        2: "3",
+        3: "4",
+        4: "5",
+        5: "6",
+        6: "7",
+        7: "8",
+        8: "9",
+        9: "21",
+        10: "51",
+        11: "52",
+        12: "53",
+    }
+
+    STYLE_ATTRIBUTES = {
+        "dim": "dim",
+        "d": "dim",
+        "bold": "bold",
+        "b": "bold",
+        "italic": "italic",
+        "i": "italic",
+        "underline": "underline",
+        "u": "underline",
+        "blink": "blink",
+        "blink2": "blink2",
+        "reverse": "reverse",
+        "r": "reverse",
+        "conceal": "conceal",
+        "c": "conceal",
+        "strike": "strike",
+        "s": "strike",
+        "underline2": "underline2",
+        "uu": "underline2",
+        "frame": "frame",
+        "encircle": "encircle",
+        "overline": "overline",
+        "o": "overline",
+    }
+
+    def __init__(
+        self,
+        *,
+        color: Optional[Union[Color, str]] = None,
+        bgcolor: Optional[Union[Color, str]] = None,
+        bold: Optional[bool] = None,
+        dim: Optional[bool] = None,
+        italic: Optional[bool] = None,
+        underline: Optional[bool] = None,
+        blink: Optional[bool] = None,
+        blink2: Optional[bool] = None,
+        reverse: Optional[bool] = None,
+        conceal: Optional[bool] = None,
+        strike: Optional[bool] = None,
+        underline2: Optional[bool] = None,
+        frame: Optional[bool] = None,
+        encircle: Optional[bool] = None,
+        overline: Optional[bool] = None,
+        link: Optional[str] = None,
+        meta: Optional[Dict[str, Any]] = None,
+    ):
+        self._ansi: Optional[str] = None
+        self._style_definition: Optional[str] = None
+
+        def _make_color(color: Union[Color, str]) -> Color:
+            return color if isinstance(color, Color) else Color.parse(color)
+
+        self._color = None if color is None else _make_color(color)
+        self._bgcolor = None if bgcolor is None else _make_color(bgcolor)
+        self._set_attributes = sum(
+            (
+                bold is not None,
+                dim is not None and 2,
+                italic is not None and 4,
+                underline is not None and 8,
+                blink is not None and 16,
+                blink2 is not None and 32,
+                reverse is not None and 64,
+                conceal is not None and 128,
+                strike is not None and 256,
+                underline2 is not None and 512,
+                frame is not None and 1024,
+                encircle is not None and 2048,
+                overline is not None and 4096,
+            )
+        )
+        self._attributes = (
+            sum(
+                (
+                    bold and 1 or 0,
+                    dim and 2 or 0,
+                    italic and 4 or 0,
+                    underline and 8 or 0,
+                    blink and 16 or 0,
+                    blink2 and 32 or 0,
+                    reverse and 64 or 0,
+                    conceal and 128 or 0,
+                    strike and 256 or 0,
+                    underline2 and 512 or 0,
+                    frame and 1024 or 0,
+                    encircle and 2048 or 0,
+                    overline and 4096 or 0,
+                )
+            )
+            if self._set_attributes
+            else 0
+        )
+
+        self._link = link
+        self._meta = None if meta is None else dumps(meta)
+        self._link_id = (
+            f"{randint(0, 999999)}{hash(self._meta)}" if (link or meta) else ""
+        )
+        self._hash: Optional[int] = None
+        self._null = not (self._set_attributes or color or bgcolor or link or meta)
+
+    @classmethod
+    def null(cls) -> "Style":
+        """Create an 'null' style, equivalent to Style(), but more performant."""
+        return NULL_STYLE
+
+    @classmethod
+    def from_color(
+        cls, color: Optional[Color] = None, bgcolor: Optional[Color] = None
+    ) -> "Style":
+        """Create a new style with colors and no attributes.
+
+        Returns:
+            color (Optional[Color]): A (foreground) color, or None for no color. Defaults to None.
+            bgcolor (Optional[Color]): A (background) color, or None for no color. Defaults to None.
+        """
+        style: Style = cls.__new__(Style)
+        style._ansi = None
+        style._style_definition = None
+        style._color = color
+        style._bgcolor = bgcolor
+        style._set_attributes = 0
+        style._attributes = 0
+        style._link = None
+        style._link_id = ""
+        style._meta = None
+        style._null = not (color or bgcolor)
+        style._hash = None
+        return style
+
+    @classmethod
+    def from_meta(cls, meta: Optional[Dict[str, Any]]) -> "Style":
+        """Create a new style with meta data.
+
+        Returns:
+            meta (Optional[Dict[str, Any]]): A dictionary of meta data. Defaults to None.
+        """
+        style: Style = cls.__new__(Style)
+        style._ansi = None
+        style._style_definition = None
+        style._color = None
+        style._bgcolor = None
+        style._set_attributes = 0
+        style._attributes = 0
+        style._link = None
+        style._meta = dumps(meta)
+        style._link_id = f"{randint(0, 999999)}{hash(style._meta)}"
+        style._hash = None
+        style._null = not (meta)
+        return style
+
+    @classmethod
+    def on(cls, meta: Optional[Dict[str, Any]] = None, **handlers: Any) -> "Style":
+        """Create a blank style with meta information.
+
+        Example:
+            style = Style.on(click=self.on_click)
+
+        Args:
+            meta (Optional[Dict[str, Any]], optional): An optional dict of meta information.
+            **handlers (Any): Keyword arguments are translated in to handlers.
+
+        Returns:
+            Style: A Style with meta information attached.
+        """
+        meta = {} if meta is None else meta
+        meta.update({f"@{key}": value for key, value in handlers.items()})
+        return cls.from_meta(meta)
+
+    bold = _Bit(0)
+    dim = _Bit(1)
+    italic = _Bit(2)
+    underline = _Bit(3)
+    blink = _Bit(4)
+    blink2 = _Bit(5)
+    reverse = _Bit(6)
+    conceal = _Bit(7)
+    strike = _Bit(8)
+    underline2 = _Bit(9)
+    frame = _Bit(10)
+    encircle = _Bit(11)
+    overline = _Bit(12)
+
+    @property
+    def link_id(self) -> str:
+        """Get a link id, used in ansi code for links."""
+        return self._link_id
+
+    def __str__(self) -> str:
+        """Re-generate style definition from attributes."""
+        if self._style_definition is None:
+            attributes: List[str] = []
+            append = attributes.append
+            bits = self._set_attributes
+            if bits & 0b0000000001111:
+                if bits & 1:
+                    append("bold" if self.bold else "not bold")
+                if bits & (1 << 1):
+                    append("dim" if self.dim else "not dim")
+                if bits & (1 << 2):
+                    append("italic" if self.italic else "not italic")
+                if bits & (1 << 3):
+                    append("underline" if self.underline else "not underline")
+            if bits & 0b0000111110000:
+                if bits & (1 << 4):
+                    append("blink" if self.blink else "not blink")
+                if bits & (1 << 5):
+                    append("blink2" if self.blink2 else "not blink2")
+                if bits & (1 << 6):
+                    append("reverse" if self.reverse else "not reverse")
+                if bits & (1 << 7):
+                    append("conceal" if self.conceal else "not conceal")
+                if bits & (1 << 8):
+                    append("strike" if self.strike else "not strike")
+            if bits & 0b1111000000000:
+                if bits & (1 << 9):
+                    append("underline2" if self.underline2 else "not underline2")
+                if bits & (1 << 10):
+                    append("frame" if self.frame else "not frame")
+                if bits & (1 << 11):
+                    append("encircle" if self.encircle else "not encircle")
+                if bits & (1 << 12):
+                    append("overline" if self.overline else "not overline")
+            if self._color is not None:
+                append(self._color.name)
+            if self._bgcolor is not None:
+                append("on")
+                append(self._bgcolor.name)
+            if self._link:
+                append("link")
+                append(self._link)
+            self._style_definition = " ".join(attributes) or "none"
+        return self._style_definition
+
+    def __bool__(self) -> bool:
+        """A Style is false if it has no attributes, colors, or links."""
+        return not self._null
+
+    def _make_ansi_codes(self, color_system: ColorSystem) -> str:
+        """Generate ANSI codes for this style.
+
+        Args:
+            color_system (ColorSystem): Color system.
+
+        Returns:
+            str: String containing codes.
+        """
+
+        if self._ansi is None:
+            sgr: List[str] = []
+            append = sgr.append
+            _style_map = self._style_map
+            attributes = self._attributes & self._set_attributes
+            if attributes:
+                if attributes & 1:
+                    append(_style_map[0])
+                if attributes & 2:
+                    append(_style_map[1])
+                if attributes & 4:
+                    append(_style_map[2])
+                if attributes & 8:
+                    append(_style_map[3])
+                if attributes & 0b0000111110000:
+                    for bit in range(4, 9):
+                        if attributes & (1 << bit):
+                            append(_style_map[bit])
+                if attributes & 0b1111000000000:
+                    for bit in range(9, 13):
+                        if attributes & (1 << bit):
+                            append(_style_map[bit])
+            if self._color is not None:
+                sgr.extend(self._color.downgrade(color_system).get_ansi_codes())
+            if self._bgcolor is not None:
+                sgr.extend(
+                    self._bgcolor.downgrade(color_system).get_ansi_codes(
+                        foreground=False
+                    )
+                )
+            self._ansi = ";".join(sgr)
+        return self._ansi
+
+    @classmethod
+    @lru_cache(maxsize=1024)
+    def normalize(cls, style: str) -> str:
+        """Normalize a style definition so that styles with the same effect have the same string
+        representation.
+
+        Args:
+            style (str): A style definition.
+
+        Returns:
+            str: Normal form of style definition.
+        """
+        try:
+            return str(cls.parse(style))
+        except errors.StyleSyntaxError:
+            return style.strip().lower()
+
+    @classmethod
+    def pick_first(cls, *values: Optional[StyleType]) -> StyleType:
+        """Pick first non-None style."""
+        for value in values:
+            if value is not None:
+                return value
+        raise ValueError("expected at least one non-None style")
+
+    def __rich_repr__(self) -> Result:
+        yield "color", self.color, None
+        yield "bgcolor", self.bgcolor, None
+        yield "bold", self.bold, None,
+        yield "dim", self.dim, None,
+        yield "italic", self.italic, None
+        yield "underline", self.underline, None,
+        yield "blink", self.blink, None
+        yield "blink2", self.blink2, None
+        yield "reverse", self.reverse, None
+        yield "conceal", self.conceal, None
+        yield "strike", self.strike, None
+        yield "underline2", self.underline2, None
+        yield "frame", self.frame, None
+        yield "encircle", self.encircle, None
+        yield "link", self.link, None
+        if self._meta:
+            yield "meta", self.meta
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, Style):
+            return NotImplemented
+        return self.__hash__() == other.__hash__()
+
+    def __ne__(self, other: Any) -> bool:
+        if not isinstance(other, Style):
+            return NotImplemented
+        return self.__hash__() != other.__hash__()
+
+    def __hash__(self) -> int:
+        if self._hash is not None:
+            return self._hash
+        self._hash = hash(_hash_getter(self))
+        return self._hash
+
+    @property
+    def color(self) -> Optional[Color]:
+        """The foreground color or None if it is not set."""
+        return self._color
+
+    @property
+    def bgcolor(self) -> Optional[Color]:
+        """The background color or None if it is not set."""
+        return self._bgcolor
+
+    @property
+    def link(self) -> Optional[str]:
+        """Link text, if set."""
+        return self._link
+
+    @property
+    def transparent_background(self) -> bool:
+        """Check if the style specified a transparent background."""
+        return self.bgcolor is None or self.bgcolor.is_default
+
+    @property
+    def background_style(self) -> "Style":
+        """A Style with background only."""
+        return Style(bgcolor=self.bgcolor)
+
+    @property
+    def meta(self) -> Dict[str, Any]:
+        """Get meta information (can not be changed after construction)."""
+        return {} if self._meta is None else cast(Dict[str, Any], loads(self._meta))
+
+    @property
+    def without_color(self) -> "Style":
+        """Get a copy of the style with color removed."""
+        if self._null:
+            return NULL_STYLE
+        style: Style = self.__new__(Style)
+        style._ansi = None
+        style._style_definition = None
+        style._color = None
+        style._bgcolor = None
+        style._attributes = self._attributes
+        style._set_attributes = self._set_attributes
+        style._link = self._link
+        style._link_id = f"{randint(0, 999999)}" if self._link else ""
+        style._null = False
+        style._meta = None
+        style._hash = None
+        return style
+
+    @classmethod
+    @lru_cache(maxsize=4096)
+    def parse(cls, style_definition: str) -> "Style":
+        """Parse a style definition.
+
+        Args:
+            style_definition (str): A string containing a style.
+
+        Raises:
+            errors.StyleSyntaxError: If the style definition syntax is invalid.
+
+        Returns:
+            `Style`: A Style instance.
+        """
+        if style_definition.strip() == "none" or not style_definition:
+            return cls.null()
+
+        STYLE_ATTRIBUTES = cls.STYLE_ATTRIBUTES
+        color: Optional[str] = None
+        bgcolor: Optional[str] = None
+        attributes: Dict[str, Optional[Any]] = {}
+        link: Optional[str] = None
+
+        words = iter(style_definition.split())
+        for original_word in words:
+            word = original_word.lower()
+            if word == "on":
+                word = next(words, "")
+                if not word:
+                    raise errors.StyleSyntaxError("color expected after 'on'")
+                try:
+                    Color.parse(word)
+                except ColorParseError as error:
+                    raise errors.StyleSyntaxError(
+                        f"unable to parse {word!r} as background color; {error}"
+                    ) from None
+                bgcolor = word
+
+            elif word == "not":
+                word = next(words, "")
+                attribute = STYLE_ATTRIBUTES.get(word)
+                if attribute is None:
+                    raise errors.StyleSyntaxError(
+                        f"expected style attribute after 'not', found {word!r}"
+                    )
+                attributes[attribute] = False
+
+            elif word == "link":
+                word = next(words, "")
+                if not word:
+                    raise errors.StyleSyntaxError("URL expected after 'link'")
+                link = word
+
+            elif word in STYLE_ATTRIBUTES:
+                attributes[STYLE_ATTRIBUTES[word]] = True
+
+            else:
+                try:
+                    Color.parse(word)
+                except ColorParseError as error:
+                    raise errors.StyleSyntaxError(
+                        f"unable to parse {word!r} as color; {error}"
+                    ) from None
+                color = word
+        style = Style(color=color, bgcolor=bgcolor, link=link, **attributes)
+        return style
+
+    @lru_cache(maxsize=1024)
+    def get_html_style(self, theme: Optional[TerminalTheme] = None) -> str:
+        """Get a CSS style rule."""
+        theme = theme or DEFAULT_TERMINAL_THEME
+        css: List[str] = []
+        append = css.append
+
+        color = self.color
+        bgcolor = self.bgcolor
+        if self.reverse:
+            color, bgcolor = bgcolor, color
+        if self.dim:
+            foreground_color = (
+                theme.foreground_color if color is None else color.get_truecolor(theme)
+            )
+            color = Color.from_triplet(
+                blend_rgb(foreground_color, theme.background_color, 0.5)
+            )
+        if color is not None:
+            theme_color = color.get_truecolor(theme)
+            append(f"color: {theme_color.hex}")
+            append(f"text-decoration-color: {theme_color.hex}")
+        if bgcolor is not None:
+            theme_color = bgcolor.get_truecolor(theme, foreground=False)
+            append(f"background-color: {theme_color.hex}")
+        if self.bold:
+            append("font-weight: bold")
+        if self.italic:
+            append("font-style: italic")
+        if self.underline:
+            append("text-decoration: underline")
+        if self.strike:
+            append("text-decoration: line-through")
+        if self.overline:
+            append("text-decoration: overline")
+        return "; ".join(css)
+
+    @classmethod
+    def combine(cls, styles: Iterable["Style"]) -> "Style":
+        """Combine styles and get result.
+
+        Args:
+            styles (Iterable[Style]): Styles to combine.
+
+        Returns:
+            Style: A new style instance.
+        """
+        iter_styles = iter(styles)
+        return sum(iter_styles, next(iter_styles))
+
+    @classmethod
+    def chain(cls, *styles: "Style") -> "Style":
+        """Combine styles from positional argument in to a single style.
+
+        Args:
+            *styles (Iterable[Style]): Styles to combine.
+
+        Returns:
+            Style: A new style instance.
+        """
+        iter_styles = iter(styles)
+        return sum(iter_styles, next(iter_styles))
+
+    def copy(self) -> "Style":
+        """Get a copy of this style.
+
+        Returns:
+            Style: A new Style instance with identical attributes.
+        """
+        if self._null:
+            return NULL_STYLE
+        style: Style = self.__new__(Style)
+        style._ansi = self._ansi
+        style._style_definition = self._style_definition
+        style._color = self._color
+        style._bgcolor = self._bgcolor
+        style._attributes = self._attributes
+        style._set_attributes = self._set_attributes
+        style._link = self._link
+        style._link_id = f"{randint(0, 999999)}" if self._link else ""
+        style._hash = self._hash
+        style._null = False
+        style._meta = self._meta
+        return style
+
+    @lru_cache(maxsize=128)
+    def clear_meta_and_links(self) -> "Style":
+        """Get a copy of this style with link and meta information removed.
+
+        Returns:
+            Style: New style object.
+        """
+        if self._null:
+            return NULL_STYLE
+        style: Style = self.__new__(Style)
+        style._ansi = self._ansi
+        style._style_definition = self._style_definition
+        style._color = self._color
+        style._bgcolor = self._bgcolor
+        style._attributes = self._attributes
+        style._set_attributes = self._set_attributes
+        style._link = None
+        style._link_id = ""
+        style._hash = None
+        style._null = False
+        style._meta = None
+        return style
+
+    def update_link(self, link: Optional[str] = None) -> "Style":
+        """Get a copy with a different value for link.
+
+        Args:
+            link (str, optional): New value for link. Defaults to None.
+
+        Returns:
+            Style: A new Style instance.
+        """
+        style: Style = self.__new__(Style)
+        style._ansi = self._ansi
+        style._style_definition = self._style_definition
+        style._color = self._color
+        style._bgcolor = self._bgcolor
+        style._attributes = self._attributes
+        style._set_attributes = self._set_attributes
+        style._link = link
+        style._link_id = f"{randint(0, 999999)}" if link else ""
+        style._hash = None
+        style._null = False
+        style._meta = self._meta
+        return style
+
+    def render(
+        self,
+        text: str = "",
+        *,
+        color_system: Optional[ColorSystem] = ColorSystem.TRUECOLOR,
+        legacy_windows: bool = False,
+    ) -> str:
+        """Render the ANSI codes for the style.
+
+        Args:
+            text (str, optional): A string to style. Defaults to "".
+            color_system (Optional[ColorSystem], optional): Color system to render to. Defaults to ColorSystem.TRUECOLOR.
+
+        Returns:
+            str: A string containing ANSI style codes.
+        """
+        if not text or color_system is None:
+            return text
+        attrs = self._ansi or self._make_ansi_codes(color_system)
+        rendered = f"\x1b[{attrs}m{text}\x1b[0m" if attrs else text
+        if self._link and not legacy_windows:
+            rendered = (
+                f"\x1b]8;id={self._link_id};{self._link}\x1b\\{rendered}\x1b]8;;\x1b\\"
+            )
+        return rendered
+
+    def test(self, text: Optional[str] = None) -> None:
+        """Write text with style directly to terminal.
+
+        This method is for testing purposes only.
+
+        Args:
+            text (Optional[str], optional): Text to style or None for style name.
+
+        """
+        text = text or str(self)
+        sys.stdout.write(f"{self.render(text)}\n")
+
+    @lru_cache(maxsize=1024)
+    def _add(self, style: Optional["Style"]) -> "Style":
+        if style is None or style._null:
+            return self
+        if self._null:
+            return style
+        new_style: Style = self.__new__(Style)
+        new_style._ansi = None
+        new_style._style_definition = None
+        new_style._color = style._color or self._color
+        new_style._bgcolor = style._bgcolor or self._bgcolor
+        new_style._attributes = (self._attributes & ~style._set_attributes) | (
+            style._attributes & style._set_attributes
+        )
+        new_style._set_attributes = self._set_attributes | style._set_attributes
+        new_style._link = style._link or self._link
+        new_style._link_id = style._link_id or self._link_id
+        new_style._null = style._null
+        if self._meta and style._meta:
+            new_style._meta = dumps({**self.meta, **style.meta})
+        else:
+            new_style._meta = self._meta or style._meta
+        new_style._hash = None
+        return new_style
+
+    def __add__(self, style: Optional["Style"]) -> "Style":
+        combined_style = self._add(style)
+        return combined_style.copy() if combined_style.link else combined_style
+
+
+NULL_STYLE = Style()
+
+
+class StyleStack:
+    """A stack of styles."""
+
+    __slots__ = ["_stack"]
+
+    def __init__(self, default_style: "Style") -> None:
+        self._stack: List[Style] = [default_style]
+
+    def __repr__(self) -> str:
+        return f""
+
+    @property
+    def current(self) -> Style:
+        """Get the Style at the top of the stack."""
+        return self._stack[-1]
+
+    def push(self, style: Style) -> None:
+        """Push a new style on to the stack.
+
+        Args:
+            style (Style): New style to combine with current style.
+        """
+        self._stack.append(self._stack[-1] + style)
+
+    def pop(self) -> Style:
+        """Pop last style and discard.
+
+        Returns:
+            Style: New current style (also available as stack.current)
+        """
+        self._stack.pop()
+        return self._stack[-1]
diff --git a/lib/rich/styled.py b/lib/rich/styled.py
new file mode 100644
index 0000000..27243be
--- /dev/null
+++ b/lib/rich/styled.py
@@ -0,0 +1,42 @@
+from typing import TYPE_CHECKING
+
+from .measure import Measurement
+from .segment import Segment
+from .style import StyleType
+
+if TYPE_CHECKING:
+    from .console import Console, ConsoleOptions, RenderResult, RenderableType
+
+
+class Styled:
+    """Apply a style to a renderable.
+
+    Args:
+        renderable (RenderableType): Any renderable.
+        style (StyleType): A style to apply across the entire renderable.
+    """
+
+    def __init__(self, renderable: "RenderableType", style: "StyleType") -> None:
+        self.renderable = renderable
+        self.style = style
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        style = console.get_style(self.style)
+        rendered_segments = console.render(self.renderable, options)
+        segments = Segment.apply_style(rendered_segments, style)
+        return segments
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> Measurement:
+        return Measurement.get(console, options, self.renderable)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich import print
+    from rich.panel import Panel
+
+    panel = Styled(Panel("hello"), "on blue")
+    print(panel)
diff --git a/lib/rich/syntax.py b/lib/rich/syntax.py
new file mode 100644
index 0000000..5e17b48
--- /dev/null
+++ b/lib/rich/syntax.py
@@ -0,0 +1,985 @@
+from __future__ import annotations
+
+import os.path
+import re
+import sys
+import textwrap
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
+
+from pygments.lexer import Lexer
+from pygments.lexers import get_lexer_by_name, guess_lexer_for_filename
+from pygments.style import Style as PygmentsStyle
+from pygments.styles import get_style_by_name
+from pygments.token import (
+    Comment,
+    Error,
+    Generic,
+    Keyword,
+    Name,
+    Number,
+    Operator,
+    String,
+    Token,
+    Whitespace,
+)
+from pygments.util import ClassNotFound
+
+from rich.containers import Lines
+from rich.padding import Padding, PaddingDimensions
+
+from ._loop import loop_first
+from .cells import cell_len
+from .color import Color, blend_rgb
+from .console import Console, ConsoleOptions, JustifyMethod, RenderResult
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .segment import Segment, Segments
+from .style import Style, StyleType
+from .text import Text
+
+TokenType = Tuple[str, ...]
+
+WINDOWS = sys.platform == "win32"
+DEFAULT_THEME = "monokai"
+
+# The following styles are based on https://github.com/pygments/pygments/blob/master/pygments/formatters/terminal.py
+# A few modifications were made
+
+ANSI_LIGHT: Dict[TokenType, Style] = {
+    Token: Style(),
+    Whitespace: Style(color="white"),
+    Comment: Style(dim=True),
+    Comment.Preproc: Style(color="cyan"),
+    Keyword: Style(color="blue"),
+    Keyword.Type: Style(color="cyan"),
+    Operator.Word: Style(color="magenta"),
+    Name.Builtin: Style(color="cyan"),
+    Name.Function: Style(color="green"),
+    Name.Namespace: Style(color="cyan", underline=True),
+    Name.Class: Style(color="green", underline=True),
+    Name.Exception: Style(color="cyan"),
+    Name.Decorator: Style(color="magenta", bold=True),
+    Name.Variable: Style(color="red"),
+    Name.Constant: Style(color="red"),
+    Name.Attribute: Style(color="cyan"),
+    Name.Tag: Style(color="bright_blue"),
+    String: Style(color="yellow"),
+    Number: Style(color="blue"),
+    Generic.Deleted: Style(color="bright_red"),
+    Generic.Inserted: Style(color="green"),
+    Generic.Heading: Style(bold=True),
+    Generic.Subheading: Style(color="magenta", bold=True),
+    Generic.Prompt: Style(bold=True),
+    Generic.Error: Style(color="bright_red"),
+    Error: Style(color="red", underline=True),
+}
+
+ANSI_DARK: Dict[TokenType, Style] = {
+    Token: Style(),
+    Whitespace: Style(color="bright_black"),
+    Comment: Style(dim=True),
+    Comment.Preproc: Style(color="bright_cyan"),
+    Keyword: Style(color="bright_blue"),
+    Keyword.Type: Style(color="bright_cyan"),
+    Operator.Word: Style(color="bright_magenta"),
+    Name.Builtin: Style(color="bright_cyan"),
+    Name.Function: Style(color="bright_green"),
+    Name.Namespace: Style(color="bright_cyan", underline=True),
+    Name.Class: Style(color="bright_green", underline=True),
+    Name.Exception: Style(color="bright_cyan"),
+    Name.Decorator: Style(color="bright_magenta", bold=True),
+    Name.Variable: Style(color="bright_red"),
+    Name.Constant: Style(color="bright_red"),
+    Name.Attribute: Style(color="bright_cyan"),
+    Name.Tag: Style(color="bright_blue"),
+    String: Style(color="yellow"),
+    Number: Style(color="bright_blue"),
+    Generic.Deleted: Style(color="bright_red"),
+    Generic.Inserted: Style(color="bright_green"),
+    Generic.Heading: Style(bold=True),
+    Generic.Subheading: Style(color="bright_magenta", bold=True),
+    Generic.Prompt: Style(bold=True),
+    Generic.Error: Style(color="bright_red"),
+    Error: Style(color="red", underline=True),
+}
+
+RICH_SYNTAX_THEMES = {"ansi_light": ANSI_LIGHT, "ansi_dark": ANSI_DARK}
+NUMBERS_COLUMN_DEFAULT_PADDING = 2
+
+
+class SyntaxTheme(ABC):
+    """Base class for a syntax theme."""
+
+    @abstractmethod
+    def get_style_for_token(self, token_type: TokenType) -> Style:
+        """Get a style for a given Pygments token."""
+        raise NotImplementedError  # pragma: no cover
+
+    @abstractmethod
+    def get_background_style(self) -> Style:
+        """Get the background color."""
+        raise NotImplementedError  # pragma: no cover
+
+
+class PygmentsSyntaxTheme(SyntaxTheme):
+    """Syntax theme that delegates to Pygments theme."""
+
+    def __init__(self, theme: Union[str, Type[PygmentsStyle]]) -> None:
+        self._style_cache: Dict[TokenType, Style] = {}
+        if isinstance(theme, str):
+            try:
+                self._pygments_style_class = get_style_by_name(theme)
+            except ClassNotFound:
+                self._pygments_style_class = get_style_by_name("default")
+        else:
+            self._pygments_style_class = theme
+
+        self._background_color = self._pygments_style_class.background_color
+        self._background_style = Style(bgcolor=self._background_color)
+
+    def get_style_for_token(self, token_type: TokenType) -> Style:
+        """Get a style from a Pygments class."""
+        try:
+            return self._style_cache[token_type]
+        except KeyError:
+            try:
+                pygments_style = self._pygments_style_class.style_for_token(token_type)
+            except KeyError:
+                style = Style.null()
+            else:
+                color = pygments_style["color"]
+                bgcolor = pygments_style["bgcolor"]
+                style = Style(
+                    color="#" + color if color else "#000000",
+                    bgcolor="#" + bgcolor if bgcolor else self._background_color,
+                    bold=pygments_style["bold"],
+                    italic=pygments_style["italic"],
+                    underline=pygments_style["underline"],
+                )
+            self._style_cache[token_type] = style
+        return style
+
+    def get_background_style(self) -> Style:
+        return self._background_style
+
+
+class ANSISyntaxTheme(SyntaxTheme):
+    """Syntax theme to use standard colors."""
+
+    def __init__(self, style_map: Dict[TokenType, Style]) -> None:
+        self.style_map = style_map
+        self._missing_style = Style.null()
+        self._background_style = Style.null()
+        self._style_cache: Dict[TokenType, Style] = {}
+
+    def get_style_for_token(self, token_type: TokenType) -> Style:
+        """Look up style in the style map."""
+        try:
+            return self._style_cache[token_type]
+        except KeyError:
+            # Styles form a hierarchy
+            # We need to go from most to least specific
+            # e.g. ("foo", "bar", "baz") to ("foo", "bar")  to ("foo",)
+            get_style = self.style_map.get
+            token = tuple(token_type)
+            style = self._missing_style
+            while token:
+                _style = get_style(token)
+                if _style is not None:
+                    style = _style
+                    break
+                token = token[:-1]
+            self._style_cache[token_type] = style
+            return style
+
+    def get_background_style(self) -> Style:
+        return self._background_style
+
+
+SyntaxPosition = Tuple[int, int]
+
+
+class _SyntaxHighlightRange(NamedTuple):
+    """
+    A range to highlight in a Syntax object.
+    `start` and `end` are 2-integers tuples, where the first integer is the line number
+    (starting from 1) and the second integer is the column index (starting from 0).
+    """
+
+    style: StyleType
+    start: SyntaxPosition
+    end: SyntaxPosition
+    style_before: bool = False
+
+
+class PaddingProperty:
+    """Descriptor to get and set padding."""
+
+    def __get__(self, obj: Syntax, objtype: Type[Syntax]) -> Tuple[int, int, int, int]:
+        """Space around the Syntax."""
+        return obj._padding
+
+    def __set__(self, obj: Syntax, padding: PaddingDimensions) -> None:
+        obj._padding = Padding.unpack(padding)
+
+
+class Syntax(JupyterMixin):
+    """Construct a Syntax object to render syntax highlighted code.
+
+    Args:
+        code (str): Code to highlight.
+        lexer (Lexer | str): Lexer to use (see https://pygments.org/docs/lexers/)
+        theme (str, optional): Color theme, aka Pygments style (see https://pygments.org/docs/styles/#getting-a-list-of-available-styles). Defaults to "monokai".
+        dedent (bool, optional): Enable stripping of initial whitespace. Defaults to False.
+        line_numbers (bool, optional): Enable rendering of line numbers. Defaults to False.
+        start_line (int, optional): Starting number for line numbers. Defaults to 1.
+        line_range (Tuple[int | None, int | None], optional): If given should be a tuple of the start and end line to render.
+            A value of None in the tuple indicates the range is open in that direction.
+        highlight_lines (Set[int]): A set of line numbers to highlight.
+        code_width: Width of code to render (not including line numbers), or ``None`` to use all available width.
+        tab_size (int, optional): Size of tabs. Defaults to 4.
+        word_wrap (bool, optional): Enable word wrapping.
+        background_color (str, optional): Optional background color, or None to use theme color. Defaults to None.
+        indent_guides (bool, optional): Show indent guides. Defaults to False.
+        padding (PaddingDimensions): Padding to apply around the syntax. Defaults to 0 (no padding).
+    """
+
+    _pygments_style_class: Type[PygmentsStyle]
+    _theme: SyntaxTheme
+
+    @classmethod
+    def get_theme(cls, name: Union[str, SyntaxTheme]) -> SyntaxTheme:
+        """Get a syntax theme instance."""
+        if isinstance(name, SyntaxTheme):
+            return name
+        theme: SyntaxTheme
+        if name in RICH_SYNTAX_THEMES:
+            theme = ANSISyntaxTheme(RICH_SYNTAX_THEMES[name])
+        else:
+            theme = PygmentsSyntaxTheme(name)
+        return theme
+
+    def __init__(
+        self,
+        code: str,
+        lexer: Union[Lexer, str],
+        *,
+        theme: Union[str, SyntaxTheme] = DEFAULT_THEME,
+        dedent: bool = False,
+        line_numbers: bool = False,
+        start_line: int = 1,
+        line_range: Optional[Tuple[Optional[int], Optional[int]]] = None,
+        highlight_lines: Optional[Set[int]] = None,
+        code_width: Optional[int] = None,
+        tab_size: int = 4,
+        word_wrap: bool = False,
+        background_color: Optional[str] = None,
+        indent_guides: bool = False,
+        padding: PaddingDimensions = 0,
+    ) -> None:
+        self.code = code
+        self._lexer = lexer
+        self.dedent = dedent
+        self.line_numbers = line_numbers
+        self.start_line = start_line
+        self.line_range = line_range
+        self.highlight_lines = highlight_lines or set()
+        self.code_width = code_width
+        self.tab_size = tab_size
+        self.word_wrap = word_wrap
+        self.background_color = background_color
+        self.background_style = (
+            Style(bgcolor=background_color) if background_color else Style()
+        )
+        self.indent_guides = indent_guides
+        self._padding = Padding.unpack(padding)
+
+        self._theme = self.get_theme(theme)
+        self._stylized_ranges: List[_SyntaxHighlightRange] = []
+
+    padding = PaddingProperty()
+
+    @classmethod
+    def from_path(
+        cls,
+        path: str,
+        encoding: str = "utf-8",
+        lexer: Optional[Union[Lexer, str]] = None,
+        theme: Union[str, SyntaxTheme] = DEFAULT_THEME,
+        dedent: bool = False,
+        line_numbers: bool = False,
+        line_range: Optional[Tuple[int, int]] = None,
+        start_line: int = 1,
+        highlight_lines: Optional[Set[int]] = None,
+        code_width: Optional[int] = None,
+        tab_size: int = 4,
+        word_wrap: bool = False,
+        background_color: Optional[str] = None,
+        indent_guides: bool = False,
+        padding: PaddingDimensions = 0,
+    ) -> "Syntax":
+        """Construct a Syntax object from a file.
+
+        Args:
+            path (str): Path to file to highlight.
+            encoding (str): Encoding of file.
+            lexer (str | Lexer, optional): Lexer to use. If None, lexer will be auto-detected from path/file content.
+            theme (str, optional): Color theme, aka Pygments style (see https://pygments.org/docs/styles/#getting-a-list-of-available-styles). Defaults to "emacs".
+            dedent (bool, optional): Enable stripping of initial whitespace. Defaults to True.
+            line_numbers (bool, optional): Enable rendering of line numbers. Defaults to False.
+            start_line (int, optional): Starting number for line numbers. Defaults to 1.
+            line_range (Tuple[int, int], optional): If given should be a tuple of the start and end line to render.
+            highlight_lines (Set[int]): A set of line numbers to highlight.
+            code_width: Width of code to render (not including line numbers), or ``None`` to use all available width.
+            tab_size (int, optional): Size of tabs. Defaults to 4.
+            word_wrap (bool, optional): Enable word wrapping of code.
+            background_color (str, optional): Optional background color, or None to use theme color. Defaults to None.
+            indent_guides (bool, optional): Show indent guides. Defaults to False.
+            padding (PaddingDimensions): Padding to apply around the syntax. Defaults to 0 (no padding).
+
+        Returns:
+            [Syntax]: A Syntax object that may be printed to the console
+        """
+        code = Path(path).read_text(encoding=encoding)
+
+        if not lexer:
+            lexer = cls.guess_lexer(path, code=code)
+
+        return cls(
+            code,
+            lexer,
+            theme=theme,
+            dedent=dedent,
+            line_numbers=line_numbers,
+            line_range=line_range,
+            start_line=start_line,
+            highlight_lines=highlight_lines,
+            code_width=code_width,
+            tab_size=tab_size,
+            word_wrap=word_wrap,
+            background_color=background_color,
+            indent_guides=indent_guides,
+            padding=padding,
+        )
+
+    @classmethod
+    def guess_lexer(cls, path: str, code: Optional[str] = None) -> str:
+        """Guess the alias of the Pygments lexer to use based on a path and an optional string of code.
+        If code is supplied, it will use a combination of the code and the filename to determine the
+        best lexer to use. For example, if the file is ``index.html`` and the file contains Django
+        templating syntax, then "html+django" will be returned. If the file is ``index.html``, and no
+        templating language is used, the "html" lexer will be used. If no string of code
+        is supplied, the lexer will be chosen based on the file extension..
+
+        Args:
+            path (AnyStr): The path to the file containing the code you wish to know the lexer for.
+            code (str, optional): Optional string of code that will be used as a fallback if no lexer
+                is found for the supplied path.
+
+        Returns:
+            str: The name of the Pygments lexer that best matches the supplied path/code.
+        """
+        lexer: Optional[Lexer] = None
+        lexer_name = "default"
+        if code:
+            try:
+                lexer = guess_lexer_for_filename(path, code)
+            except ClassNotFound:
+                pass
+
+        if not lexer:
+            try:
+                _, ext = os.path.splitext(path)
+                if ext:
+                    extension = ext.lstrip(".").lower()
+                    lexer = get_lexer_by_name(extension)
+            except ClassNotFound:
+                pass
+
+        if lexer:
+            if lexer.aliases:
+                lexer_name = lexer.aliases[0]
+            else:
+                lexer_name = lexer.name
+
+        return lexer_name
+
+    def _get_base_style(self) -> Style:
+        """Get the base style."""
+        default_style = self._theme.get_background_style() + self.background_style
+        return default_style
+
+    def _get_token_color(self, token_type: TokenType) -> Optional[Color]:
+        """Get a color (if any) for the given token.
+
+        Args:
+            token_type (TokenType): A token type tuple from Pygments.
+
+        Returns:
+            Optional[Color]: Color from theme, or None for no color.
+        """
+        style = self._theme.get_style_for_token(token_type)
+        return style.color
+
+    @property
+    def lexer(self) -> Optional[Lexer]:
+        """The lexer for this syntax, or None if no lexer was found.
+
+        Tries to find the lexer by name if a string was passed to the constructor.
+        """
+
+        if isinstance(self._lexer, Lexer):
+            return self._lexer
+        try:
+            return get_lexer_by_name(
+                self._lexer,
+                stripnl=False,
+                ensurenl=True,
+                tabsize=self.tab_size,
+            )
+        except ClassNotFound:
+            return None
+
+    @property
+    def default_lexer(self) -> Lexer:
+        """A Pygments Lexer to use if one is not specified or invalid."""
+        return get_lexer_by_name(
+            "text",
+            stripnl=False,
+            ensurenl=True,
+            tabsize=self.tab_size,
+        )
+
+    def highlight(
+        self,
+        code: str,
+        line_range: Optional[Tuple[Optional[int], Optional[int]]] = None,
+    ) -> Text:
+        """Highlight code and return a Text instance.
+
+        Args:
+            code (str): Code to highlight.
+            line_range(Tuple[int, int], optional): Optional line range to highlight.
+
+        Returns:
+            Text: A text instance containing highlighted syntax.
+        """
+
+        base_style = self._get_base_style()
+        justify: JustifyMethod = (
+            "default" if base_style.transparent_background else "left"
+        )
+
+        text = Text(
+            justify=justify,
+            style=base_style,
+            tab_size=self.tab_size,
+            no_wrap=not self.word_wrap,
+        )
+        _get_theme_style = self._theme.get_style_for_token
+
+        lexer = self.lexer or self.default_lexer
+
+        if lexer is None:
+            text.append(code)
+        else:
+            if line_range:
+                # More complicated path to only stylize a portion of the code
+                # This speeds up further operations as there are less spans to process
+                line_start, line_end = line_range
+
+                def line_tokenize() -> Iterable[Tuple[Any, str]]:
+                    """Split tokens to one per line."""
+                    assert lexer  # required to make MyPy happy - we know lexer is not None at this point
+
+                    for token_type, token in lexer.get_tokens(code):
+                        while token:
+                            line_token, new_line, token = token.partition("\n")
+                            yield token_type, line_token + new_line
+
+                def tokens_to_spans() -> Iterable[Tuple[str, Optional[Style]]]:
+                    """Convert tokens to spans."""
+                    tokens = iter(line_tokenize())
+                    line_no = 0
+                    _line_start = line_start - 1 if line_start else 0
+
+                    # Skip over tokens until line start
+                    while line_no < _line_start:
+                        try:
+                            _token_type, token = next(tokens)
+                        except StopIteration:
+                            break
+                        yield (token, None)
+                        if token.endswith("\n"):
+                            line_no += 1
+                    # Generate spans until line end
+                    for token_type, token in tokens:
+                        yield (token, _get_theme_style(token_type))
+                        if token.endswith("\n"):
+                            line_no += 1
+                            if line_end and line_no >= line_end:
+                                break
+
+                text.append_tokens(tokens_to_spans())
+
+            else:
+                text.append_tokens(
+                    (token, _get_theme_style(token_type))
+                    for token_type, token in lexer.get_tokens(code)
+                )
+            if self.background_color is not None:
+                text.stylize(f"on {self.background_color}")
+
+        if self._stylized_ranges:
+            self._apply_stylized_ranges(text)
+
+        return text
+
+    def stylize_range(
+        self,
+        style: StyleType,
+        start: SyntaxPosition,
+        end: SyntaxPosition,
+        style_before: bool = False,
+    ) -> None:
+        """
+        Adds a custom style on a part of the code, that will be applied to the syntax display when it's rendered.
+        Line numbers are 1-based, while column indexes are 0-based.
+
+        Args:
+            style (StyleType): The style to apply.
+            start (Tuple[int, int]): The start of the range, in the form `[line number, column index]`.
+            end (Tuple[int, int]): The end of the range, in the form `[line number, column index]`.
+            style_before (bool): Apply the style before any existing styles.
+        """
+        self._stylized_ranges.append(
+            _SyntaxHighlightRange(style, start, end, style_before)
+        )
+
+    def _get_line_numbers_color(self, blend: float = 0.3) -> Color:
+        background_style = self._theme.get_background_style() + self.background_style
+        background_color = background_style.bgcolor
+        if background_color is None or background_color.is_system_defined:
+            return Color.default()
+        foreground_color = self._get_token_color(Token.Text)
+        if foreground_color is None or foreground_color.is_system_defined:
+            return foreground_color or Color.default()
+        new_color = blend_rgb(
+            background_color.get_truecolor(),
+            foreground_color.get_truecolor(),
+            cross_fade=blend,
+        )
+        return Color.from_triplet(new_color)
+
+    @property
+    def _numbers_column_width(self) -> int:
+        """Get the number of characters used to render the numbers column."""
+        column_width = 0
+        if self.line_numbers:
+            column_width = (
+                len(str(self.start_line + self.code.count("\n")))
+                + NUMBERS_COLUMN_DEFAULT_PADDING
+            )
+        return column_width
+
+    def _get_number_styles(self, console: Console) -> Tuple[Style, Style, Style]:
+        """Get background, number, and highlight styles for line numbers."""
+        background_style = self._get_base_style()
+        if background_style.transparent_background:
+            return Style.null(), Style(dim=True), Style.null()
+        if console.color_system in ("256", "truecolor"):
+            number_style = Style.chain(
+                background_style,
+                self._theme.get_style_for_token(Token.Text),
+                Style(color=self._get_line_numbers_color()),
+                self.background_style,
+            )
+            highlight_number_style = Style.chain(
+                background_style,
+                self._theme.get_style_for_token(Token.Text),
+                Style(bold=True, color=self._get_line_numbers_color(0.9)),
+                self.background_style,
+            )
+        else:
+            number_style = background_style + Style(dim=True)
+            highlight_number_style = background_style + Style(dim=False)
+        return background_style, number_style, highlight_number_style
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        _, right, _, left = self.padding
+        padding = left + right
+        if self.code_width is not None:
+            width = self.code_width + self._numbers_column_width + padding + 1
+            return Measurement(self._numbers_column_width, width)
+        lines = self.code.splitlines()
+        width = (
+            self._numbers_column_width
+            + padding
+            + (max(cell_len(line) for line in lines) if lines else 0)
+        )
+        if self.line_numbers:
+            width += 1
+        return Measurement(self._numbers_column_width, width)
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        segments = Segments(self._get_syntax(console, options))
+        if any(self.padding):
+            yield Padding(segments, style=self._get_base_style(), pad=self.padding)
+        else:
+            yield segments
+
+    def _get_syntax(
+        self,
+        console: Console,
+        options: ConsoleOptions,
+    ) -> Iterable[Segment]:
+        """
+        Get the Segments for the Syntax object, excluding any vertical/horizontal padding
+        """
+        transparent_background = self._get_base_style().transparent_background
+        _pad_top, pad_right, _pad_bottom, pad_left = self.padding
+        horizontal_padding = pad_left + pad_right
+        code_width = (
+            (
+                (options.max_width - self._numbers_column_width - 1)
+                if self.line_numbers
+                else options.max_width
+            )
+            - horizontal_padding
+            if self.code_width is None
+            else self.code_width
+        )
+        code_width = max(0, code_width)
+
+        ends_on_nl, processed_code = self._process_code(self.code)
+        text = self.highlight(processed_code, self.line_range)
+
+        if not self.line_numbers and not self.word_wrap and not self.line_range:
+            if not ends_on_nl:
+                text.remove_suffix("\n")
+            # Simple case of just rendering text
+            style = (
+                self._get_base_style()
+                + self._theme.get_style_for_token(Comment)
+                + Style(dim=True)
+                + self.background_style
+            )
+            if self.indent_guides and not options.ascii_only:
+                text = text.with_indent_guides(self.tab_size, style=style)
+                text.overflow = "crop"
+            if style.transparent_background:
+                yield from console.render(
+                    text, options=options.update(width=code_width)
+                )
+            else:
+                syntax_lines = console.render_lines(
+                    text,
+                    options.update(width=code_width, height=None, justify="left"),
+                    style=self.background_style,
+                    pad=True,
+                    new_lines=True,
+                )
+                for syntax_line in syntax_lines:
+                    yield from syntax_line
+            return
+
+        start_line, end_line = self.line_range or (None, None)
+        line_offset = 0
+        if start_line:
+            line_offset = max(0, start_line - 1)
+        lines: Union[List[Text], Lines] = text.split("\n", allow_blank=ends_on_nl)
+        if self.line_range:
+            if line_offset > len(lines):
+                return
+            lines = lines[line_offset:end_line]
+
+        if self.indent_guides and not options.ascii_only:
+            style = (
+                self._get_base_style()
+                + self._theme.get_style_for_token(Comment)
+                + Style(dim=True)
+                + self.background_style
+            )
+            lines = (
+                Text("\n")
+                .join(lines)
+                .with_indent_guides(self.tab_size, style=style + Style(italic=False))
+                .split("\n", allow_blank=True)
+            )
+
+        numbers_column_width = self._numbers_column_width
+        render_options = options.update(width=code_width)
+
+        highlight_line = self.highlight_lines.__contains__
+        _Segment = Segment
+        new_line = _Segment("\n")
+
+        line_pointer = "> " if options.legacy_windows else "❱ "
+
+        (
+            background_style,
+            number_style,
+            highlight_number_style,
+        ) = self._get_number_styles(console)
+
+        for line_no, line in enumerate(lines, self.start_line + line_offset):
+            if self.word_wrap:
+                wrapped_lines = console.render_lines(
+                    line,
+                    render_options.update(height=None, justify="left"),
+                    style=background_style,
+                    pad=not transparent_background,
+                )
+            else:
+                segments = list(line.render(console, end=""))
+                if options.no_wrap:
+                    wrapped_lines = [segments]
+                else:
+                    wrapped_lines = [
+                        _Segment.adjust_line_length(
+                            segments,
+                            render_options.max_width,
+                            style=background_style,
+                            pad=not transparent_background,
+                        )
+                    ]
+
+            if self.line_numbers:
+                wrapped_line_left_pad = _Segment(
+                    " " * numbers_column_width + " ", background_style
+                )
+                for first, wrapped_line in loop_first(wrapped_lines):
+                    if first:
+                        line_column = str(line_no).rjust(numbers_column_width - 2) + " "
+                        if highlight_line(line_no):
+                            yield _Segment(line_pointer, Style(color="red"))
+                            yield _Segment(line_column, highlight_number_style)
+                        else:
+                            yield _Segment("  ", highlight_number_style)
+                            yield _Segment(line_column, number_style)
+                    else:
+                        yield wrapped_line_left_pad
+                    yield from wrapped_line
+                    yield new_line
+            else:
+                for wrapped_line in wrapped_lines:
+                    yield from wrapped_line
+                    yield new_line
+
+    def _apply_stylized_ranges(self, text: Text) -> None:
+        """
+        Apply stylized ranges to a text instance,
+        using the given code to determine the right portion to apply the style to.
+
+        Args:
+            text (Text): Text instance to apply the style to.
+        """
+        code = text.plain
+        newlines_offsets = [
+            # Let's add outer boundaries at each side of the list:
+            0,
+            # N.B. using "\n" here is much faster than using metacharacters such as "^" or "\Z":
+            *[
+                match.start() + 1
+                for match in re.finditer("\n", code, flags=re.MULTILINE)
+            ],
+            len(code) + 1,
+        ]
+
+        for stylized_range in self._stylized_ranges:
+            start = _get_code_index_for_syntax_position(
+                newlines_offsets, stylized_range.start
+            )
+            end = _get_code_index_for_syntax_position(
+                newlines_offsets, stylized_range.end
+            )
+            if start is not None and end is not None:
+                if stylized_range.style_before:
+                    text.stylize_before(stylized_range.style, start, end)
+                else:
+                    text.stylize(stylized_range.style, start, end)
+
+    def _process_code(self, code: str) -> Tuple[bool, str]:
+        """
+        Applies various processing to a raw code string
+        (normalises it so it always ends with a line return, dedents it if necessary, etc.)
+
+        Args:
+            code (str): The raw code string to process
+
+        Returns:
+            Tuple[bool, str]: the boolean indicates whether the raw code ends with a line return,
+                while the string is the processed code.
+        """
+        ends_on_nl = code.endswith("\n")
+        processed_code = code if ends_on_nl else code + "\n"
+        processed_code = (
+            textwrap.dedent(processed_code) if self.dedent else processed_code
+        )
+        processed_code = processed_code.expandtabs(self.tab_size)
+        return ends_on_nl, processed_code
+
+
+def _get_code_index_for_syntax_position(
+    newlines_offsets: Sequence[int], position: SyntaxPosition
+) -> Optional[int]:
+    """
+    Returns the index of the code string for the given positions.
+
+    Args:
+        newlines_offsets (Sequence[int]): The offset of each newline character found in the code snippet.
+        position (SyntaxPosition): The position to search for.
+
+    Returns:
+        Optional[int]: The index of the code string for this position, or `None`
+            if the given position's line number is out of range (if it's the column that is out of range
+            we silently clamp its value so that it reaches the end of the line)
+    """
+    lines_count = len(newlines_offsets)
+
+    line_number, column_index = position
+    if line_number > lines_count or len(newlines_offsets) < (line_number + 1):
+        return None  # `line_number` is out of range
+    line_index = line_number - 1
+    line_length = newlines_offsets[line_index + 1] - newlines_offsets[line_index] - 1
+    # If `column_index` is out of range: let's silently clamp it:
+    column_index = min(line_length, column_index)
+    return newlines_offsets[line_index] + column_index
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import argparse
+    import sys
+
+    parser = argparse.ArgumentParser(
+        description="Render syntax to the console with Rich"
+    )
+    parser.add_argument(
+        "path",
+        metavar="PATH",
+        help="path to file, or - for stdin",
+    )
+    parser.add_argument(
+        "-c",
+        "--force-color",
+        dest="force_color",
+        action="store_true",
+        default=None,
+        help="force color for non-terminals",
+    )
+    parser.add_argument(
+        "-i",
+        "--indent-guides",
+        dest="indent_guides",
+        action="store_true",
+        default=False,
+        help="display indent guides",
+    )
+    parser.add_argument(
+        "-l",
+        "--line-numbers",
+        dest="line_numbers",
+        action="store_true",
+        help="render line numbers",
+    )
+    parser.add_argument(
+        "-w",
+        "--width",
+        type=int,
+        dest="width",
+        default=None,
+        help="width of output (default will auto-detect)",
+    )
+    parser.add_argument(
+        "-r",
+        "--wrap",
+        dest="word_wrap",
+        action="store_true",
+        default=False,
+        help="word wrap long lines",
+    )
+    parser.add_argument(
+        "-s",
+        "--soft-wrap",
+        action="store_true",
+        dest="soft_wrap",
+        default=False,
+        help="enable soft wrapping mode",
+    )
+    parser.add_argument(
+        "-t", "--theme", dest="theme", default="monokai", help="pygments theme"
+    )
+    parser.add_argument(
+        "-b",
+        "--background-color",
+        dest="background_color",
+        default=None,
+        help="Override background color",
+    )
+    parser.add_argument(
+        "-x",
+        "--lexer",
+        default=None,
+        dest="lexer_name",
+        help="Lexer name",
+    )
+    parser.add_argument(
+        "-p", "--padding", type=int, default=0, dest="padding", help="Padding"
+    )
+    parser.add_argument(
+        "--highlight-line",
+        type=int,
+        default=None,
+        dest="highlight_line",
+        help="The line number (not index!) to highlight",
+    )
+    args = parser.parse_args()
+
+    from rich.console import Console
+
+    console = Console(force_terminal=args.force_color, width=args.width)
+
+    if args.path == "-":
+        code = sys.stdin.read()
+        syntax = Syntax(
+            code=code,
+            lexer=args.lexer_name,
+            line_numbers=args.line_numbers,
+            word_wrap=args.word_wrap,
+            theme=args.theme,
+            background_color=args.background_color,
+            indent_guides=args.indent_guides,
+            padding=args.padding,
+            highlight_lines={args.highlight_line},
+        )
+    else:
+        syntax = Syntax.from_path(
+            args.path,
+            lexer=args.lexer_name,
+            line_numbers=args.line_numbers,
+            word_wrap=args.word_wrap,
+            theme=args.theme,
+            background_color=args.background_color,
+            indent_guides=args.indent_guides,
+            padding=args.padding,
+            highlight_lines={args.highlight_line},
+        )
+    console.print(syntax, soft_wrap=args.soft_wrap)
diff --git a/lib/rich/table.py b/lib/rich/table.py
new file mode 100644
index 0000000..1e2eb2b
--- /dev/null
+++ b/lib/rich/table.py
@@ -0,0 +1,1015 @@
+from dataclasses import dataclass, field, replace
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
+
+from . import box, errors
+from ._loop import loop_first_last, loop_last
+from ._pick import pick_bool
+from ._ratio import ratio_distribute, ratio_reduce
+from .align import VerticalAlignMethod
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .padding import Padding, PaddingDimensions
+from .protocol import is_renderable
+from .segment import Segment
+from .style import Style, StyleType
+from .text import Text, TextType
+
+if TYPE_CHECKING:
+    from .console import (
+        Console,
+        ConsoleOptions,
+        JustifyMethod,
+        OverflowMethod,
+        RenderableType,
+        RenderResult,
+    )
+
+
+@dataclass
+class Column:
+    """Defines a column within a ~Table.
+
+    Args:
+        title (Union[str, Text], optional): The title of the table rendered at the top. Defaults to None.
+        caption (Union[str, Text], optional): The table caption rendered below. Defaults to None.
+        width (int, optional): The width in characters of the table, or ``None`` to automatically fit. Defaults to None.
+        min_width (Optional[int], optional): The minimum width of the table, or ``None`` for no minimum. Defaults to None.
+        box (box.Box, optional): One of the constants in box.py used to draw the edges (see :ref:`appendix_box`), or ``None`` for no box lines. Defaults to box.HEAVY_HEAD.
+        safe_box (Optional[bool], optional): Disable box characters that don't display on windows legacy terminal with *raster* fonts. Defaults to True.
+        padding (PaddingDimensions, optional): Padding for cells (top, right, bottom, left). Defaults to (0, 1).
+        collapse_padding (bool, optional): Enable collapsing of padding around cells. Defaults to False.
+        pad_edge (bool, optional): Enable padding of edge cells. Defaults to True.
+        show_header (bool, optional): Show a header row. Defaults to True.
+        show_footer (bool, optional): Show a footer row. Defaults to False.
+        show_edge (bool, optional): Draw a box around the outside of the table. Defaults to True.
+        show_lines (bool, optional): Draw lines between every row. Defaults to False.
+        leading (int, optional): Number of blank lines between rows (precludes ``show_lines``). Defaults to 0.
+        style (Union[str, Style], optional): Default style for the table. Defaults to "none".
+        row_styles (List[Union, str], optional): Optional list of row styles, if more than one style is given then the styles will alternate. Defaults to None.
+        header_style (Union[str, Style], optional): Style of the header. Defaults to "table.header".
+        footer_style (Union[str, Style], optional): Style of the footer. Defaults to "table.footer".
+        border_style (Union[str, Style], optional): Style of the border. Defaults to None.
+        title_style (Union[str, Style], optional): Style of the title. Defaults to None.
+        caption_style (Union[str, Style], optional): Style of the caption. Defaults to None.
+        title_justify (str, optional): Justify method for title. Defaults to "center".
+        caption_justify (str, optional): Justify method for caption. Defaults to "center".
+        highlight (bool, optional): Highlight cell contents (if str). Defaults to False.
+    """
+
+    header: "RenderableType" = ""
+    """RenderableType: Renderable for the header (typically a string)"""
+
+    footer: "RenderableType" = ""
+    """RenderableType: Renderable for the footer (typically a string)"""
+
+    header_style: StyleType = ""
+    """StyleType: The style of the header."""
+
+    footer_style: StyleType = ""
+    """StyleType: The style of the footer."""
+
+    style: StyleType = ""
+    """StyleType: The style of the column."""
+
+    justify: "JustifyMethod" = "left"
+    """str: How to justify text within the column ("left", "center", "right", or "full")"""
+
+    vertical: "VerticalAlignMethod" = "top"
+    """str: How to vertically align content ("top", "middle", or "bottom")"""
+
+    overflow: "OverflowMethod" = "ellipsis"
+    """str: Overflow method."""
+
+    width: Optional[int] = None
+    """Optional[int]: Width of the column, or ``None`` (default) to auto calculate width."""
+
+    min_width: Optional[int] = None
+    """Optional[int]: Minimum width of column, or ``None`` for no minimum. Defaults to None."""
+
+    max_width: Optional[int] = None
+    """Optional[int]: Maximum width of column, or ``None`` for no maximum. Defaults to None."""
+
+    ratio: Optional[int] = None
+    """Optional[int]: Ratio to use when calculating column width, or ``None`` (default) to adapt to column contents."""
+
+    no_wrap: bool = False
+    """bool: Prevent wrapping of text within the column. Defaults to ``False``."""
+
+    highlight: bool = False
+    """bool: Apply highlighter to column. Defaults to ``False``."""
+
+    _index: int = 0
+    """Index of column."""
+
+    _cells: List["RenderableType"] = field(default_factory=list)
+
+    def copy(self) -> "Column":
+        """Return a copy of this Column."""
+        return replace(self, _cells=[])
+
+    @property
+    def cells(self) -> Iterable["RenderableType"]:
+        """Get all cells in the column, not including header."""
+        yield from self._cells
+
+    @property
+    def flexible(self) -> bool:
+        """Check if this column is flexible."""
+        return self.ratio is not None
+
+
+@dataclass
+class Row:
+    """Information regarding a row."""
+
+    style: Optional[StyleType] = None
+    """Style to apply to row."""
+
+    end_section: bool = False
+    """Indicated end of section, which will force a line beneath the row."""
+
+
+class _Cell(NamedTuple):
+    """A single cell in a table."""
+
+    style: StyleType
+    """Style to apply to cell."""
+    renderable: "RenderableType"
+    """Cell renderable."""
+    vertical: VerticalAlignMethod
+    """Cell vertical alignment."""
+
+
+class Table(JupyterMixin):
+    """A console renderable to draw a table.
+
+    Args:
+        *headers (Union[Column, str]): Column headers, either as a string, or :class:`~rich.table.Column` instance.
+        title (Union[str, Text], optional): The title of the table rendered at the top. Defaults to None.
+        caption (Union[str, Text], optional): The table caption rendered below. Defaults to None.
+        width (int, optional): The width in characters of the table, or ``None`` to automatically fit. Defaults to None.
+        min_width (Optional[int], optional): The minimum width of the table, or ``None`` for no minimum. Defaults to None.
+        box (box.Box, optional): One of the constants in box.py used to draw the edges (see :ref:`appendix_box`), or ``None`` for no box lines. Defaults to box.HEAVY_HEAD.
+        safe_box (Optional[bool], optional): Disable box characters that don't display on windows legacy terminal with *raster* fonts. Defaults to True.
+        padding (PaddingDimensions, optional): Padding for cells (top, right, bottom, left). Defaults to (0, 1).
+        collapse_padding (bool, optional): Enable collapsing of padding around cells. Defaults to False.
+        pad_edge (bool, optional): Enable padding of edge cells. Defaults to True.
+        expand (bool, optional): Expand the table to fit the available space if ``True``, otherwise the table width will be auto-calculated. Defaults to False.
+        show_header (bool, optional): Show a header row. Defaults to True.
+        show_footer (bool, optional): Show a footer row. Defaults to False.
+        show_edge (bool, optional): Draw a box around the outside of the table. Defaults to True.
+        show_lines (bool, optional): Draw lines between every row. Defaults to False.
+        leading (int, optional): Number of blank lines between rows (precludes ``show_lines``). Defaults to 0.
+        style (Union[str, Style], optional): Default style for the table. Defaults to "none".
+        row_styles (List[Union, str], optional): Optional list of row styles, if more than one style is given then the styles will alternate. Defaults to None.
+        header_style (Union[str, Style], optional): Style of the header. Defaults to "table.header".
+        footer_style (Union[str, Style], optional): Style of the footer. Defaults to "table.footer".
+        border_style (Union[str, Style], optional): Style of the border. Defaults to None.
+        title_style (Union[str, Style], optional): Style of the title. Defaults to None.
+        caption_style (Union[str, Style], optional): Style of the caption. Defaults to None.
+        title_justify (str, optional): Justify method for title. Defaults to "center".
+        caption_justify (str, optional): Justify method for caption. Defaults to "center".
+        highlight (bool, optional): Highlight cell contents (if str). Defaults to False.
+    """
+
+    columns: List[Column]
+    rows: List[Row]
+
+    def __init__(
+        self,
+        *headers: Union[Column, str],
+        title: Optional[TextType] = None,
+        caption: Optional[TextType] = None,
+        width: Optional[int] = None,
+        min_width: Optional[int] = None,
+        box: Optional[box.Box] = box.HEAVY_HEAD,
+        safe_box: Optional[bool] = None,
+        padding: PaddingDimensions = (0, 1),
+        collapse_padding: bool = False,
+        pad_edge: bool = True,
+        expand: bool = False,
+        show_header: bool = True,
+        show_footer: bool = False,
+        show_edge: bool = True,
+        show_lines: bool = False,
+        leading: int = 0,
+        style: StyleType = "none",
+        row_styles: Optional[Iterable[StyleType]] = None,
+        header_style: Optional[StyleType] = "table.header",
+        footer_style: Optional[StyleType] = "table.footer",
+        border_style: Optional[StyleType] = None,
+        title_style: Optional[StyleType] = None,
+        caption_style: Optional[StyleType] = None,
+        title_justify: "JustifyMethod" = "center",
+        caption_justify: "JustifyMethod" = "center",
+        highlight: bool = False,
+    ) -> None:
+        self.columns: List[Column] = []
+        self.rows: List[Row] = []
+        self.title = title
+        self.caption = caption
+        self.width = width
+        self.min_width = min_width
+        self.box = box
+        self.safe_box = safe_box
+        self._padding = Padding.unpack(padding)
+        self.pad_edge = pad_edge
+        self._expand = expand
+        self.show_header = show_header
+        self.show_footer = show_footer
+        self.show_edge = show_edge
+        self.show_lines = show_lines
+        self.leading = leading
+        self.collapse_padding = collapse_padding
+        self.style = style
+        self.header_style = header_style or ""
+        self.footer_style = footer_style or ""
+        self.border_style = border_style
+        self.title_style = title_style
+        self.caption_style = caption_style
+        self.title_justify: "JustifyMethod" = title_justify
+        self.caption_justify: "JustifyMethod" = caption_justify
+        self.highlight = highlight
+        self.row_styles: Sequence[StyleType] = list(row_styles or [])
+        append_column = self.columns.append
+        for header in headers:
+            if isinstance(header, str):
+                self.add_column(header=header)
+            else:
+                header._index = len(self.columns)
+                append_column(header)
+
+    @classmethod
+    def grid(
+        cls,
+        *headers: Union[Column, str],
+        padding: PaddingDimensions = 0,
+        collapse_padding: bool = True,
+        pad_edge: bool = False,
+        expand: bool = False,
+    ) -> "Table":
+        """Get a table with no lines, headers, or footer.
+
+        Args:
+            *headers (Union[Column, str]): Column headers, either as a string, or :class:`~rich.table.Column` instance.
+            padding (PaddingDimensions, optional): Get padding around cells. Defaults to 0.
+            collapse_padding (bool, optional): Enable collapsing of padding around cells. Defaults to True.
+            pad_edge (bool, optional): Enable padding around edges of table. Defaults to False.
+            expand (bool, optional): Expand the table to fit the available space if ``True``, otherwise the table width will be auto-calculated. Defaults to False.
+
+        Returns:
+            Table: A table instance.
+        """
+        return cls(
+            *headers,
+            box=None,
+            padding=padding,
+            collapse_padding=collapse_padding,
+            show_header=False,
+            show_footer=False,
+            show_edge=False,
+            pad_edge=pad_edge,
+            expand=expand,
+        )
+
+    @property
+    def expand(self) -> bool:
+        """Setting a non-None self.width implies expand."""
+        return self._expand or self.width is not None
+
+    @expand.setter
+    def expand(self, expand: bool) -> None:
+        """Set expand."""
+        self._expand = expand
+
+    @property
+    def _extra_width(self) -> int:
+        """Get extra width to add to cell content."""
+        width = 0
+        if self.box and self.show_edge:
+            width += 2
+        if self.box:
+            width += len(self.columns) - 1
+        return width
+
+    @property
+    def row_count(self) -> int:
+        """Get the current number of rows."""
+        return len(self.rows)
+
+    def get_row_style(self, console: "Console", index: int) -> StyleType:
+        """Get the current row style."""
+        style = Style.null()
+        if self.row_styles:
+            style += console.get_style(self.row_styles[index % len(self.row_styles)])
+        row_style = self.rows[index].style
+        if row_style is not None:
+            style += console.get_style(row_style)
+        return style
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> Measurement:
+        max_width = options.max_width
+        if self.width is not None:
+            max_width = self.width
+        if max_width < 0:
+            return Measurement(0, 0)
+
+        extra_width = self._extra_width
+        max_width = sum(
+            self._calculate_column_widths(
+                console, options.update_width(max_width - extra_width)
+            )
+        )
+        _measure_column = self._measure_column
+
+        measurements = [
+            _measure_column(console, options.update_width(max_width), column)
+            for column in self.columns
+        ]
+        minimum_width = (
+            sum(measurement.minimum for measurement in measurements) + extra_width
+        )
+        maximum_width = (
+            sum(measurement.maximum for measurement in measurements) + extra_width
+            if (self.width is None)
+            else self.width
+        )
+        measurement = Measurement(minimum_width, maximum_width)
+        measurement = measurement.clamp(self.min_width)
+        return measurement
+
+    @property
+    def padding(self) -> Tuple[int, int, int, int]:
+        """Get cell padding."""
+        return self._padding
+
+    @padding.setter
+    def padding(self, padding: PaddingDimensions) -> "Table":
+        """Set cell padding."""
+        self._padding = Padding.unpack(padding)
+        return self
+
+    def add_column(
+        self,
+        header: "RenderableType" = "",
+        footer: "RenderableType" = "",
+        *,
+        header_style: Optional[StyleType] = None,
+        highlight: Optional[bool] = None,
+        footer_style: Optional[StyleType] = None,
+        style: Optional[StyleType] = None,
+        justify: "JustifyMethod" = "left",
+        vertical: "VerticalAlignMethod" = "top",
+        overflow: "OverflowMethod" = "ellipsis",
+        width: Optional[int] = None,
+        min_width: Optional[int] = None,
+        max_width: Optional[int] = None,
+        ratio: Optional[int] = None,
+        no_wrap: bool = False,
+    ) -> None:
+        """Add a column to the table.
+
+        Args:
+            header (RenderableType, optional): Text or renderable for the header.
+                Defaults to "".
+            footer (RenderableType, optional): Text or renderable for the footer.
+                Defaults to "".
+            header_style (Union[str, Style], optional): Style for the header, or None for default. Defaults to None.
+            highlight (bool, optional): Whether to highlight the text. The default of None uses the value of the table (self) object.
+            footer_style (Union[str, Style], optional): Style for the footer, or None for default. Defaults to None.
+            style (Union[str, Style], optional): Style for the column cells, or None for default. Defaults to None.
+            justify (JustifyMethod, optional): Alignment for cells. Defaults to "left".
+            vertical (VerticalAlignMethod, optional): Vertical alignment, one of "top", "middle", or "bottom". Defaults to "top".
+            overflow (OverflowMethod): Overflow method: "crop", "fold", "ellipsis". Defaults to "ellipsis".
+            width (int, optional): Desired width of column in characters, or None to fit to contents. Defaults to None.
+            min_width (Optional[int], optional): Minimum width of column, or ``None`` for no minimum. Defaults to None.
+            max_width (Optional[int], optional): Maximum width of column, or ``None`` for no maximum. Defaults to None.
+            ratio (int, optional): Flexible ratio for the column (requires ``Table.expand`` or ``Table.width``). Defaults to None.
+            no_wrap (bool, optional): Set to ``True`` to disable wrapping of this column.
+        """
+
+        column = Column(
+            _index=len(self.columns),
+            header=header,
+            footer=footer,
+            header_style=header_style or "",
+            highlight=highlight if highlight is not None else self.highlight,
+            footer_style=footer_style or "",
+            style=style or "",
+            justify=justify,
+            vertical=vertical,
+            overflow=overflow,
+            width=width,
+            min_width=min_width,
+            max_width=max_width,
+            ratio=ratio,
+            no_wrap=no_wrap,
+        )
+        self.columns.append(column)
+
+    def add_row(
+        self,
+        *renderables: Optional["RenderableType"],
+        style: Optional[StyleType] = None,
+        end_section: bool = False,
+    ) -> None:
+        """Add a row of renderables.
+
+        Args:
+            *renderables (None or renderable): Each cell in a row must be a renderable object (including str),
+                or ``None`` for a blank cell.
+            style (StyleType, optional): An optional style to apply to the entire row. Defaults to None.
+            end_section (bool, optional): End a section and draw a line. Defaults to False.
+
+        Raises:
+            errors.NotRenderableError: If you add something that can't be rendered.
+        """
+
+        def add_cell(column: Column, renderable: "RenderableType") -> None:
+            column._cells.append(renderable)
+
+        cell_renderables: List[Optional["RenderableType"]] = list(renderables)
+
+        columns = self.columns
+        if len(cell_renderables) < len(columns):
+            cell_renderables = [
+                *cell_renderables,
+                *[None] * (len(columns) - len(cell_renderables)),
+            ]
+        for index, renderable in enumerate(cell_renderables):
+            if index == len(columns):
+                column = Column(_index=index, highlight=self.highlight)
+                for _ in self.rows:
+                    add_cell(column, Text(""))
+                self.columns.append(column)
+            else:
+                column = columns[index]
+            if renderable is None:
+                add_cell(column, "")
+            elif is_renderable(renderable):
+                add_cell(column, renderable)
+            else:
+                raise errors.NotRenderableError(
+                    f"unable to render {type(renderable).__name__}; a string or other renderable object is required"
+                )
+        self.rows.append(Row(style=style, end_section=end_section))
+
+    def add_section(self) -> None:
+        """Add a new section (draw a line after current row)."""
+
+        if self.rows:
+            self.rows[-1].end_section = True
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        if not self.columns:
+            yield Segment("\n")
+            return
+
+        max_width = options.max_width
+        if self.width is not None:
+            max_width = self.width
+
+        extra_width = self._extra_width
+
+        widths = self._calculate_column_widths(
+            console, options.update_width(max_width - extra_width)
+        )
+        table_width = sum(widths) + extra_width
+
+        render_options = options.update(
+            width=table_width, highlight=self.highlight, height=None
+        )
+
+        def render_annotation(
+            text: TextType, style: StyleType, justify: "JustifyMethod" = "center"
+        ) -> "RenderResult":
+            render_text = (
+                console.render_str(text, style=style, highlight=False)
+                if isinstance(text, str)
+                else text
+            )
+            return console.render(
+                render_text, options=render_options.update(justify=justify)
+            )
+
+        if self.title:
+            yield from render_annotation(
+                self.title,
+                style=Style.pick_first(self.title_style, "table.title"),
+                justify=self.title_justify,
+            )
+        yield from self._render(console, render_options, widths)
+        if self.caption:
+            yield from render_annotation(
+                self.caption,
+                style=Style.pick_first(self.caption_style, "table.caption"),
+                justify=self.caption_justify,
+            )
+
+    def _calculate_column_widths(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> List[int]:
+        """Calculate the widths of each column, including padding, not including borders."""
+        max_width = options.max_width
+        columns = self.columns
+        width_ranges = [
+            self._measure_column(console, options, column) for column in columns
+        ]
+        widths = [_range.maximum or 1 for _range in width_ranges]
+
+        get_padding_width = self._get_padding_width
+        extra_width = self._extra_width
+        if self.expand:
+            ratios = [col.ratio or 0 for col in columns if col.flexible]
+            if any(ratios):
+                fixed_widths = [
+                    0 if column.flexible else _range.maximum
+                    for _range, column in zip(width_ranges, columns)
+                ]
+                flex_minimum = [
+                    (column.width or 1) + get_padding_width(column._index)
+                    for column in columns
+                    if column.flexible
+                ]
+                flexible_width = max_width - sum(fixed_widths)
+                flex_widths = ratio_distribute(flexible_width, ratios, flex_minimum)
+                iter_flex_widths = iter(flex_widths)
+                for index, column in enumerate(columns):
+                    if column.flexible:
+                        widths[index] = fixed_widths[index] + next(iter_flex_widths)
+        table_width = sum(widths)
+
+        if table_width > max_width:
+            widths = self._collapse_widths(
+                widths,
+                [(column.width is None and not column.no_wrap) for column in columns],
+                max_width,
+            )
+            table_width = sum(widths)
+            # last resort, reduce columns evenly
+            if table_width > max_width:
+                excess_width = table_width - max_width
+                widths = ratio_reduce(excess_width, [1] * len(widths), widths, widths)
+                table_width = sum(widths)
+
+            width_ranges = [
+                self._measure_column(console, options.update_width(width), column)
+                for width, column in zip(widths, columns)
+            ]
+            widths = [_range.maximum or 0 for _range in width_ranges]
+
+        if (table_width < max_width and self.expand) or (
+            self.min_width is not None and table_width < (self.min_width - extra_width)
+        ):
+            _max_width = (
+                max_width
+                if self.min_width is None
+                else min(self.min_width - extra_width, max_width)
+            )
+            pad_widths = ratio_distribute(_max_width - table_width, widths)
+            widths = [_width + pad for _width, pad in zip(widths, pad_widths)]
+
+        return widths
+
+    @classmethod
+    def _collapse_widths(
+        cls, widths: List[int], wrapable: List[bool], max_width: int
+    ) -> List[int]:
+        """Reduce widths so that the total is under max_width.
+
+        Args:
+            widths (List[int]): List of widths.
+            wrapable (List[bool]): List of booleans that indicate if a column may shrink.
+            max_width (int): Maximum width to reduce to.
+
+        Returns:
+            List[int]: A new list of widths.
+        """
+        total_width = sum(widths)
+        excess_width = total_width - max_width
+        if any(wrapable):
+            while total_width and excess_width > 0:
+                max_column = max(
+                    width for width, allow_wrap in zip(widths, wrapable) if allow_wrap
+                )
+                second_max_column = max(
+                    width if allow_wrap and width != max_column else 0
+                    for width, allow_wrap in zip(widths, wrapable)
+                )
+                column_difference = max_column - second_max_column
+                ratios = [
+                    (1 if (width == max_column and allow_wrap) else 0)
+                    for width, allow_wrap in zip(widths, wrapable)
+                ]
+                if not any(ratios) or not column_difference:
+                    break
+                max_reduce = [min(excess_width, column_difference)] * len(widths)
+                widths = ratio_reduce(excess_width, ratios, max_reduce, widths)
+
+                total_width = sum(widths)
+                excess_width = total_width - max_width
+        return widths
+
+    def _get_cells(
+        self, console: "Console", column_index: int, column: Column
+    ) -> Iterable[_Cell]:
+        """Get all the cells with padding and optional header."""
+
+        collapse_padding = self.collapse_padding
+        pad_edge = self.pad_edge
+        padding = self.padding
+        any_padding = any(padding)
+
+        first_column = column_index == 0
+        last_column = column_index == len(self.columns) - 1
+
+        _padding_cache: Dict[Tuple[bool, bool], Tuple[int, int, int, int]] = {}
+
+        def get_padding(first_row: bool, last_row: bool) -> Tuple[int, int, int, int]:
+            cached = _padding_cache.get((first_row, last_row))
+            if cached:
+                return cached
+            top, right, bottom, left = padding
+
+            if collapse_padding:
+                if not first_column:
+                    left = max(0, left - right)
+                if not last_row:
+                    bottom = max(0, top - bottom)
+
+            if not pad_edge:
+                if first_column:
+                    left = 0
+                if last_column:
+                    right = 0
+                if first_row:
+                    top = 0
+                if last_row:
+                    bottom = 0
+            _padding = (top, right, bottom, left)
+            _padding_cache[(first_row, last_row)] = _padding
+            return _padding
+
+        raw_cells: List[Tuple[StyleType, "RenderableType"]] = []
+        _append = raw_cells.append
+        get_style = console.get_style
+        if self.show_header:
+            header_style = get_style(self.header_style or "") + get_style(
+                column.header_style
+            )
+            _append((header_style, column.header))
+        cell_style = get_style(column.style or "")
+        for cell in column.cells:
+            _append((cell_style, cell))
+        if self.show_footer:
+            footer_style = get_style(self.footer_style or "") + get_style(
+                column.footer_style
+            )
+            _append((footer_style, column.footer))
+
+        if any_padding:
+            _Padding = Padding
+            for first, last, (style, renderable) in loop_first_last(raw_cells):
+                yield _Cell(
+                    style,
+                    _Padding(renderable, get_padding(first, last)),
+                    getattr(renderable, "vertical", None) or column.vertical,
+                )
+        else:
+            for style, renderable in raw_cells:
+                yield _Cell(
+                    style,
+                    renderable,
+                    getattr(renderable, "vertical", None) or column.vertical,
+                )
+
+    def _get_padding_width(self, column_index: int) -> int:
+        """Get extra width from padding."""
+        _, pad_right, _, pad_left = self.padding
+
+        if self.collapse_padding:
+            pad_left = 0
+            pad_right = abs(pad_left - pad_right)
+
+        if not self.pad_edge:
+            if column_index == 0:
+                pad_left = 0
+            if column_index == len(self.columns) - 1:
+                pad_right = 0
+
+        return pad_left + pad_right
+
+    def _measure_column(
+        self,
+        console: "Console",
+        options: "ConsoleOptions",
+        column: Column,
+    ) -> Measurement:
+        """Get the minimum and maximum width of the column."""
+
+        max_width = options.max_width
+        if max_width < 1:
+            return Measurement(0, 0)
+
+        padding_width = self._get_padding_width(column._index)
+        if column.width is not None:
+            # Fixed width column
+            return Measurement(
+                column.width + padding_width, column.width + padding_width
+            ).with_maximum(max_width)
+        # Flexible column, we need to measure contents
+        min_widths: List[int] = []
+        max_widths: List[int] = []
+        append_min = min_widths.append
+        append_max = max_widths.append
+        get_render_width = Measurement.get
+        for cell in self._get_cells(console, column._index, column):
+            _min, _max = get_render_width(console, options, cell.renderable)
+            append_min(_min)
+            append_max(_max)
+
+        measurement = Measurement(
+            max(min_widths) if min_widths else 1,
+            max(max_widths) if max_widths else max_width,
+        ).with_maximum(max_width)
+        measurement = measurement.clamp(
+            None if column.min_width is None else column.min_width + padding_width,
+            None if column.max_width is None else column.max_width + padding_width,
+        )
+        return measurement
+
+    def _render(
+        self, console: "Console", options: "ConsoleOptions", widths: List[int]
+    ) -> "RenderResult":
+        table_style = console.get_style(self.style or "")
+
+        border_style = table_style + console.get_style(self.border_style or "")
+        _column_cells = (
+            self._get_cells(console, column_index, column)
+            for column_index, column in enumerate(self.columns)
+        )
+
+        row_cells: List[Tuple[_Cell, ...]] = list(zip(*_column_cells))
+        _box = (
+            self.box.substitute(
+                options, safe=pick_bool(self.safe_box, console.safe_box)
+            )
+            if self.box
+            else None
+        )
+        _box = _box.get_plain_headed_box() if _box and not self.show_header else _box
+
+        new_line = Segment.line()
+
+        columns = self.columns
+        show_header = self.show_header
+        show_footer = self.show_footer
+        show_edge = self.show_edge
+        show_lines = self.show_lines
+        leading = self.leading
+
+        _Segment = Segment
+        if _box:
+            box_segments = [
+                (
+                    _Segment(_box.head_left, border_style),
+                    _Segment(_box.head_right, border_style),
+                    _Segment(_box.head_vertical, border_style),
+                ),
+                (
+                    _Segment(_box.mid_left, border_style),
+                    _Segment(_box.mid_right, border_style),
+                    _Segment(_box.mid_vertical, border_style),
+                ),
+                (
+                    _Segment(_box.foot_left, border_style),
+                    _Segment(_box.foot_right, border_style),
+                    _Segment(_box.foot_vertical, border_style),
+                ),
+            ]
+            if show_edge:
+                yield _Segment(_box.get_top(widths), border_style)
+                yield new_line
+        else:
+            box_segments = []
+
+        get_row_style = self.get_row_style
+        get_style = console.get_style
+
+        for index, (first, last, row_cell) in enumerate(loop_first_last(row_cells)):
+            header_row = first and show_header
+            footer_row = last and show_footer
+            row = (
+                self.rows[index - show_header]
+                if (not header_row and not footer_row)
+                else None
+            )
+            max_height = 1
+            cells: List[List[List[Segment]]] = []
+            if header_row or footer_row:
+                row_style = Style.null()
+            else:
+                row_style = get_style(
+                    get_row_style(console, index - 1 if show_header else index)
+                )
+            for width, cell, column in zip(widths, row_cell, columns):
+                render_options = options.update(
+                    width=width,
+                    justify=column.justify,
+                    no_wrap=column.no_wrap,
+                    overflow=column.overflow,
+                    height=None,
+                    highlight=column.highlight,
+                )
+                lines = console.render_lines(
+                    cell.renderable,
+                    render_options,
+                    style=get_style(cell.style) + row_style,
+                )
+                max_height = max(max_height, len(lines))
+                cells.append(lines)
+
+            row_height = max(len(cell) for cell in cells)
+
+            def align_cell(
+                cell: List[List[Segment]],
+                vertical: "VerticalAlignMethod",
+                width: int,
+                style: Style,
+            ) -> List[List[Segment]]:
+                if header_row:
+                    vertical = "bottom"
+                elif footer_row:
+                    vertical = "top"
+
+                if vertical == "top":
+                    return _Segment.align_top(cell, width, row_height, style)
+                elif vertical == "middle":
+                    return _Segment.align_middle(cell, width, row_height, style)
+                return _Segment.align_bottom(cell, width, row_height, style)
+
+            cells[:] = [
+                _Segment.set_shape(
+                    align_cell(
+                        cell,
+                        _cell.vertical,
+                        width,
+                        get_style(_cell.style) + row_style,
+                    ),
+                    width,
+                    max_height,
+                )
+                for width, _cell, cell, column in zip(widths, row_cell, cells, columns)
+            ]
+
+            if _box:
+                if last and show_footer:
+                    yield _Segment(
+                        _box.get_row(widths, "foot", edge=show_edge), border_style
+                    )
+                    yield new_line
+                left, right, _divider = box_segments[0 if first else (2 if last else 1)]
+
+                # If the column divider is whitespace also style it with the row background
+                divider = (
+                    _divider
+                    if _divider.text.strip()
+                    else _Segment(
+                        _divider.text, row_style.background_style + _divider.style
+                    )
+                )
+                for line_no in range(max_height):
+                    if show_edge:
+                        yield left
+                    for last_cell, rendered_cell in loop_last(cells):
+                        yield from rendered_cell[line_no]
+                        if not last_cell:
+                            yield divider
+                    if show_edge:
+                        yield right
+                    yield new_line
+            else:
+                for line_no in range(max_height):
+                    for rendered_cell in cells:
+                        yield from rendered_cell[line_no]
+                    yield new_line
+            if _box and first and show_header:
+                yield _Segment(
+                    _box.get_row(widths, "head", edge=show_edge), border_style
+                )
+                yield new_line
+            end_section = row and row.end_section
+            if _box and (show_lines or leading or end_section):
+                if (
+                    not last
+                    and not (show_footer and index >= len(row_cells) - 2)
+                    and not (show_header and header_row)
+                ):
+                    if leading:
+                        yield _Segment(
+                            _box.get_row(widths, "mid", edge=show_edge) * leading,
+                            border_style,
+                        )
+                    else:
+                        yield _Segment(
+                            _box.get_row(widths, "row", edge=show_edge), border_style
+                        )
+                    yield new_line
+
+        if _box and show_edge:
+            yield _Segment(_box.get_bottom(widths), border_style)
+            yield new_line
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich.console import Console
+    from rich.highlighter import ReprHighlighter
+
+    from ._timer import timer
+
+    with timer("Table render"):
+        table = Table(
+            title="Star Wars Movies",
+            caption="Rich example table",
+            caption_justify="right",
+        )
+
+        table.add_column(
+            "Released", header_style="bright_cyan", style="cyan", no_wrap=True
+        )
+        table.add_column("Title", style="magenta")
+        table.add_column("Box Office", justify="right", style="green")
+
+        table.add_row(
+            "Dec 20, 2019",
+            "Star Wars: The Rise of Skywalker",
+            "$952,110,690",
+        )
+        table.add_row("May 25, 2018", "Solo: A Star Wars Story", "$393,151,347")
+        table.add_row(
+            "Dec 15, 2017",
+            "Star Wars Ep. V111: The Last Jedi",
+            "$1,332,539,889",
+            style="on black",
+            end_section=True,
+        )
+        table.add_row(
+            "Dec 16, 2016",
+            "Rogue One: A Star Wars Story",
+            "$1,332,439,889",
+        )
+
+        def header(text: str) -> None:
+            console.print()
+            console.rule(highlight(text))
+            console.print()
+
+        console = Console()
+        highlight = ReprHighlighter()
+        header("Example Table")
+        console.print(table, justify="center")
+
+        table.expand = True
+        header("expand=True")
+        console.print(table)
+
+        table.width = 50
+        header("width=50")
+
+        console.print(table, justify="center")
+
+        table.width = None
+        table.expand = False
+        table.row_styles = ["dim", "none"]
+        header("row_styles=['dim', 'none']")
+
+        console.print(table, justify="center")
+
+        table.width = None
+        table.expand = False
+        table.row_styles = ["dim", "none"]
+        table.leading = 1
+        header("leading=1, row_styles=['dim', 'none']")
+        console.print(table, justify="center")
+
+        table.width = None
+        table.expand = False
+        table.row_styles = ["dim", "none"]
+        table.show_lines = True
+        table.leading = 0
+        header("show_lines=True, row_styles=['dim', 'none']")
+        console.print(table, justify="center")
diff --git a/lib/rich/terminal_theme.py b/lib/rich/terminal_theme.py
new file mode 100644
index 0000000..565e9d9
--- /dev/null
+++ b/lib/rich/terminal_theme.py
@@ -0,0 +1,153 @@
+from typing import List, Optional, Tuple
+
+from .color_triplet import ColorTriplet
+from .palette import Palette
+
+_ColorTuple = Tuple[int, int, int]
+
+
+class TerminalTheme:
+    """A color theme used when exporting console content.
+
+    Args:
+        background (Tuple[int, int, int]): The background color.
+        foreground (Tuple[int, int, int]): The foreground (text) color.
+        normal (List[Tuple[int, int, int]]): A list of 8 normal intensity colors.
+        bright (List[Tuple[int, int, int]], optional): A list of 8 bright colors, or None
+            to repeat normal intensity. Defaults to None.
+    """
+
+    def __init__(
+        self,
+        background: _ColorTuple,
+        foreground: _ColorTuple,
+        normal: List[_ColorTuple],
+        bright: Optional[List[_ColorTuple]] = None,
+    ) -> None:
+        self.background_color = ColorTriplet(*background)
+        self.foreground_color = ColorTriplet(*foreground)
+        self.ansi_colors = Palette(normal + (bright or normal))
+
+
+DEFAULT_TERMINAL_THEME = TerminalTheme(
+    (255, 255, 255),
+    (0, 0, 0),
+    [
+        (0, 0, 0),
+        (128, 0, 0),
+        (0, 128, 0),
+        (128, 128, 0),
+        (0, 0, 128),
+        (128, 0, 128),
+        (0, 128, 128),
+        (192, 192, 192),
+    ],
+    [
+        (128, 128, 128),
+        (255, 0, 0),
+        (0, 255, 0),
+        (255, 255, 0),
+        (0, 0, 255),
+        (255, 0, 255),
+        (0, 255, 255),
+        (255, 255, 255),
+    ],
+)
+
+MONOKAI = TerminalTheme(
+    (12, 12, 12),
+    (217, 217, 217),
+    [
+        (26, 26, 26),
+        (244, 0, 95),
+        (152, 224, 36),
+        (253, 151, 31),
+        (157, 101, 255),
+        (244, 0, 95),
+        (88, 209, 235),
+        (196, 197, 181),
+        (98, 94, 76),
+    ],
+    [
+        (244, 0, 95),
+        (152, 224, 36),
+        (224, 213, 97),
+        (157, 101, 255),
+        (244, 0, 95),
+        (88, 209, 235),
+        (246, 246, 239),
+    ],
+)
+DIMMED_MONOKAI = TerminalTheme(
+    (25, 25, 25),
+    (185, 188, 186),
+    [
+        (58, 61, 67),
+        (190, 63, 72),
+        (135, 154, 59),
+        (197, 166, 53),
+        (79, 118, 161),
+        (133, 92, 141),
+        (87, 143, 164),
+        (185, 188, 186),
+        (136, 137, 135),
+    ],
+    [
+        (251, 0, 31),
+        (15, 114, 47),
+        (196, 112, 51),
+        (24, 109, 227),
+        (251, 0, 103),
+        (46, 112, 109),
+        (253, 255, 185),
+    ],
+)
+NIGHT_OWLISH = TerminalTheme(
+    (255, 255, 255),
+    (64, 63, 83),
+    [
+        (1, 22, 39),
+        (211, 66, 62),
+        (42, 162, 152),
+        (218, 170, 1),
+        (72, 118, 214),
+        (64, 63, 83),
+        (8, 145, 106),
+        (122, 129, 129),
+        (122, 129, 129),
+    ],
+    [
+        (247, 110, 110),
+        (73, 208, 197),
+        (218, 194, 107),
+        (92, 167, 228),
+        (105, 112, 152),
+        (0, 201, 144),
+        (152, 159, 177),
+    ],
+)
+
+SVG_EXPORT_THEME = TerminalTheme(
+    (41, 41, 41),
+    (197, 200, 198),
+    [
+        (75, 78, 85),
+        (204, 85, 90),
+        (152, 168, 75),
+        (208, 179, 68),
+        (96, 138, 177),
+        (152, 114, 159),
+        (104, 160, 179),
+        (197, 200, 198),
+        (154, 155, 153),
+    ],
+    [
+        (255, 38, 39),
+        (0, 130, 61),
+        (208, 132, 66),
+        (25, 132, 233),
+        (255, 44, 122),
+        (57, 130, 128),
+        (253, 253, 197),
+    ],
+)
diff --git a/lib/rich/text.py b/lib/rich/text.py
new file mode 100644
index 0000000..7e087a4
--- /dev/null
+++ b/lib/rich/text.py
@@ -0,0 +1,1363 @@
+import re
+from functools import partial, reduce
+from math import gcd
+from operator import itemgetter
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    NamedTuple,
+    Optional,
+    Pattern,
+    Tuple,
+    Union,
+)
+
+from ._loop import loop_last
+from ._pick import pick_bool
+from ._wrap import divide_line
+from .align import AlignMethod
+from .cells import cell_len, set_cell_size
+from .containers import Lines
+from .control import strip_control_codes
+from .emoji import EmojiVariant
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .segment import Segment
+from .style import Style, StyleType
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .console import Console, ConsoleOptions, JustifyMethod, OverflowMethod
+
+DEFAULT_JUSTIFY: "JustifyMethod" = "default"
+DEFAULT_OVERFLOW: "OverflowMethod" = "fold"
+
+
+_re_whitespace = re.compile(r"\s+$")
+
+TextType = Union[str, "Text"]
+"""A plain string or a :class:`Text` instance."""
+
+GetStyleCallable = Callable[[str], Optional[StyleType]]
+
+
+class Span(NamedTuple):
+    """A marked up region in some text."""
+
+    start: int
+    """Span start index."""
+    end: int
+    """Span end index."""
+    style: Union[str, Style]
+    """Style associated with the span."""
+
+    def __repr__(self) -> str:
+        return f"Span({self.start}, {self.end}, {self.style!r})"
+
+    def __bool__(self) -> bool:
+        return self.end > self.start
+
+    def split(self, offset: int) -> Tuple["Span", Optional["Span"]]:
+        """Split a span in to 2 from a given offset."""
+
+        if offset < self.start:
+            return self, None
+        if offset >= self.end:
+            return self, None
+
+        start, end, style = self
+        span1 = Span(start, min(end, offset), style)
+        span2 = Span(span1.end, end, style)
+        return span1, span2
+
+    def move(self, offset: int) -> "Span":
+        """Move start and end by a given offset.
+
+        Args:
+            offset (int): Number of characters to add to start and end.
+
+        Returns:
+            TextSpan: A new TextSpan with adjusted position.
+        """
+        start, end, style = self
+        return Span(start + offset, end + offset, style)
+
+    def right_crop(self, offset: int) -> "Span":
+        """Crop the span at the given offset.
+
+        Args:
+            offset (int): A value between start and end.
+
+        Returns:
+            Span: A new (possibly smaller) span.
+        """
+        start, end, style = self
+        if offset >= end:
+            return self
+        return Span(start, min(offset, end), style)
+
+    def extend(self, cells: int) -> "Span":
+        """Extend the span by the given number of cells.
+
+        Args:
+            cells (int): Additional space to add to end of span.
+
+        Returns:
+            Span: A span.
+        """
+        if cells:
+            start, end, style = self
+            return Span(start, end + cells, style)
+        else:
+            return self
+
+
+class Text(JupyterMixin):
+    """Text with color / style.
+
+    Args:
+        text (str, optional): Default unstyled text. Defaults to "".
+        style (Union[str, Style], optional): Base style for text. Defaults to "".
+        justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
+        overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
+        no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None.
+        end (str, optional): Character to end text with. Defaults to "\\\\n".
+        tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to None.
+        spans (List[Span], optional). A list of predefined style spans. Defaults to None.
+    """
+
+    __slots__ = [
+        "_text",
+        "style",
+        "justify",
+        "overflow",
+        "no_wrap",
+        "end",
+        "tab_size",
+        "_spans",
+        "_length",
+    ]
+
+    def __init__(
+        self,
+        text: str = "",
+        style: Union[str, Style] = "",
+        *,
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+        no_wrap: Optional[bool] = None,
+        end: str = "\n",
+        tab_size: Optional[int] = None,
+        spans: Optional[List[Span]] = None,
+    ) -> None:
+        sanitized_text = strip_control_codes(text)
+        self._text = [sanitized_text]
+        self.style = style
+        self.justify: Optional["JustifyMethod"] = justify
+        self.overflow: Optional["OverflowMethod"] = overflow
+        self.no_wrap = no_wrap
+        self.end = end
+        self.tab_size = tab_size
+        self._spans: List[Span] = spans or []
+        self._length: int = len(sanitized_text)
+
+    def __len__(self) -> int:
+        return self._length
+
+    def __bool__(self) -> bool:
+        return bool(self._length)
+
+    def __str__(self) -> str:
+        return self.plain
+
+    def __repr__(self) -> str:
+        return f""
+
+    def __add__(self, other: Any) -> "Text":
+        if isinstance(other, (str, Text)):
+            result = self.copy()
+            result.append(other)
+            return result
+        return NotImplemented
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Text):
+            return NotImplemented
+        return self.plain == other.plain and self._spans == other._spans
+
+    def __contains__(self, other: object) -> bool:
+        if isinstance(other, str):
+            return other in self.plain
+        elif isinstance(other, Text):
+            return other.plain in self.plain
+        return False
+
+    def __getitem__(self, slice: Union[int, slice]) -> "Text":
+        def get_text_at(offset: int) -> "Text":
+            _Span = Span
+            text = Text(
+                self.plain[offset],
+                spans=[
+                    _Span(0, 1, style)
+                    for start, end, style in self._spans
+                    if end > offset >= start
+                ],
+                end="",
+            )
+            return text
+
+        if isinstance(slice, int):
+            return get_text_at(slice)
+        else:
+            start, stop, step = slice.indices(len(self.plain))
+            if step == 1:
+                lines = self.divide([start, stop])
+                return lines[1]
+            else:
+                # This would be a bit of work to implement efficiently
+                # For now, its not required
+                raise TypeError("slices with step!=1 are not supported")
+
+    @property
+    def cell_len(self) -> int:
+        """Get the number of cells required to render this text."""
+        return cell_len(self.plain)
+
+    @property
+    def markup(self) -> str:
+        """Get console markup to render this Text.
+
+        Returns:
+            str: A string potentially creating markup tags.
+        """
+        from .markup import escape
+
+        output: List[str] = []
+
+        plain = self.plain
+        markup_spans = [
+            (0, False, self.style),
+            *((span.start, False, span.style) for span in self._spans),
+            *((span.end, True, span.style) for span in self._spans),
+            (len(plain), True, self.style),
+        ]
+        markup_spans.sort(key=itemgetter(0, 1))
+        position = 0
+        append = output.append
+        for offset, closing, style in markup_spans:
+            if offset > position:
+                append(escape(plain[position:offset]))
+                position = offset
+            if style:
+                append(f"[/{style}]" if closing else f"[{style}]")
+        markup = "".join(output)
+        return markup
+
+    @classmethod
+    def from_markup(
+        cls,
+        text: str,
+        *,
+        style: Union[str, Style] = "",
+        emoji: bool = True,
+        emoji_variant: Optional[EmojiVariant] = None,
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+        end: str = "\n",
+    ) -> "Text":
+        """Create Text instance from markup.
+
+        Args:
+            text (str): A string containing console markup.
+            style (Union[str, Style], optional): Base style for text. Defaults to "".
+            emoji (bool, optional): Also render emoji code. Defaults to True.
+            emoji_variant (str, optional): Optional emoji variant, either "text" or "emoji". Defaults to None.
+            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
+            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
+            end (str, optional): Character to end text with. Defaults to "\\\\n".
+
+        Returns:
+            Text: A Text instance with markup rendered.
+        """
+        from .markup import render
+
+        rendered_text = render(text, style, emoji=emoji, emoji_variant=emoji_variant)
+        rendered_text.justify = justify
+        rendered_text.overflow = overflow
+        rendered_text.end = end
+        return rendered_text
+
+    @classmethod
+    def from_ansi(
+        cls,
+        text: str,
+        *,
+        style: Union[str, Style] = "",
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+        no_wrap: Optional[bool] = None,
+        end: str = "\n",
+        tab_size: Optional[int] = 8,
+    ) -> "Text":
+        """Create a Text object from a string containing ANSI escape codes.
+
+        Args:
+            text (str): A string containing escape codes.
+            style (Union[str, Style], optional): Base style for text. Defaults to "".
+            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
+            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
+            no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None.
+            end (str, optional): Character to end text with. Defaults to "\\\\n".
+            tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to None.
+        """
+        from .ansi import AnsiDecoder
+
+        joiner = Text(
+            "\n",
+            justify=justify,
+            overflow=overflow,
+            no_wrap=no_wrap,
+            end=end,
+            tab_size=tab_size,
+            style=style,
+        )
+        decoder = AnsiDecoder()
+        result = joiner.join(line for line in decoder.decode(text))
+        return result
+
+    @classmethod
+    def styled(
+        cls,
+        text: str,
+        style: StyleType = "",
+        *,
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+    ) -> "Text":
+        """Construct a Text instance with a pre-applied styled. A style applied in this way won't be used
+        to pad the text when it is justified.
+
+        Args:
+            text (str): A string containing console markup.
+            style (Union[str, Style]): Style to apply to the text. Defaults to "".
+            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
+            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
+
+        Returns:
+            Text: A text instance with a style applied to the entire string.
+        """
+        styled_text = cls(text, justify=justify, overflow=overflow)
+        styled_text.stylize(style)
+        return styled_text
+
+    @classmethod
+    def assemble(
+        cls,
+        *parts: Union[str, "Text", Tuple[str, StyleType]],
+        style: Union[str, Style] = "",
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+        no_wrap: Optional[bool] = None,
+        end: str = "\n",
+        tab_size: int = 8,
+        meta: Optional[Dict[str, Any]] = None,
+    ) -> "Text":
+        """Construct a text instance by combining a sequence of strings with optional styles.
+        The positional arguments should be either strings, or a tuple of string + style.
+
+        Args:
+            style (Union[str, Style], optional): Base style for text. Defaults to "".
+            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
+            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
+            no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None.
+            end (str, optional): Character to end text with. Defaults to "\\\\n".
+            tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to None.
+            meta (Dict[str, Any], optional). Meta data to apply to text, or None for no meta data. Default to None
+
+        Returns:
+            Text: A new text instance.
+        """
+        text = cls(
+            style=style,
+            justify=justify,
+            overflow=overflow,
+            no_wrap=no_wrap,
+            end=end,
+            tab_size=tab_size,
+        )
+        append = text.append
+        _Text = Text
+        for part in parts:
+            if isinstance(part, (_Text, str)):
+                append(part)
+            else:
+                append(*part)
+        if meta:
+            text.apply_meta(meta)
+        return text
+
+    @property
+    def plain(self) -> str:
+        """Get the text as a single string."""
+        if len(self._text) != 1:
+            self._text[:] = ["".join(self._text)]
+        return self._text[0]
+
+    @plain.setter
+    def plain(self, new_text: str) -> None:
+        """Set the text to a new value."""
+        if new_text != self.plain:
+            sanitized_text = strip_control_codes(new_text)
+            self._text[:] = [sanitized_text]
+            old_length = self._length
+            self._length = len(sanitized_text)
+            if old_length > self._length:
+                self._trim_spans()
+
+    @property
+    def spans(self) -> List[Span]:
+        """Get a reference to the internal list of spans."""
+        return self._spans
+
+    @spans.setter
+    def spans(self, spans: List[Span]) -> None:
+        """Set spans."""
+        self._spans = spans[:]
+
+    def blank_copy(self, plain: str = "") -> "Text":
+        """Return a new Text instance with copied metadata (but not the string or spans)."""
+        copy_self = Text(
+            plain,
+            style=self.style,
+            justify=self.justify,
+            overflow=self.overflow,
+            no_wrap=self.no_wrap,
+            end=self.end,
+            tab_size=self.tab_size,
+        )
+        return copy_self
+
+    def copy(self) -> "Text":
+        """Return a copy of this instance."""
+        copy_self = Text(
+            self.plain,
+            style=self.style,
+            justify=self.justify,
+            overflow=self.overflow,
+            no_wrap=self.no_wrap,
+            end=self.end,
+            tab_size=self.tab_size,
+        )
+        copy_self._spans[:] = self._spans
+        return copy_self
+
+    def stylize(
+        self,
+        style: Union[str, Style],
+        start: int = 0,
+        end: Optional[int] = None,
+    ) -> None:
+        """Apply a style to the text, or a portion of the text.
+
+        Args:
+            style (Union[str, Style]): Style instance or style definition to apply.
+            start (int): Start offset (negative indexing is supported). Defaults to 0.
+            end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None.
+        """
+        if style:
+            length = len(self)
+            if start < 0:
+                start = length + start
+            if end is None:
+                end = length
+            if end < 0:
+                end = length + end
+            if start >= length or end <= start:
+                # Span not in text or not valid
+                return
+            self._spans.append(Span(start, min(length, end), style))
+
+    def stylize_before(
+        self,
+        style: Union[str, Style],
+        start: int = 0,
+        end: Optional[int] = None,
+    ) -> None:
+        """Apply a style to the text, or a portion of the text. Styles will be applied before other styles already present.
+
+        Args:
+            style (Union[str, Style]): Style instance or style definition to apply.
+            start (int): Start offset (negative indexing is supported). Defaults to 0.
+            end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None.
+        """
+        if style:
+            length = len(self)
+            if start < 0:
+                start = length + start
+            if end is None:
+                end = length
+            if end < 0:
+                end = length + end
+            if start >= length or end <= start:
+                # Span not in text or not valid
+                return
+            self._spans.insert(0, Span(start, min(length, end), style))
+
+    def apply_meta(
+        self, meta: Dict[str, Any], start: int = 0, end: Optional[int] = None
+    ) -> None:
+        """Apply metadata to the text, or a portion of the text.
+
+        Args:
+            meta (Dict[str, Any]): A dict of meta information.
+            start (int): Start offset (negative indexing is supported). Defaults to 0.
+            end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None.
+
+        """
+        style = Style.from_meta(meta)
+        self.stylize(style, start=start, end=end)
+
+    def on(self, meta: Optional[Dict[str, Any]] = None, **handlers: Any) -> "Text":
+        """Apply event handlers (used by Textual project).
+
+        Example:
+            >>> from rich.text import Text
+            >>> text = Text("hello world")
+            >>> text.on(click="view.toggle('world')")
+
+        Args:
+            meta (Dict[str, Any]): Mapping of meta information.
+            **handlers: Keyword args are prefixed with "@" to defined handlers.
+
+        Returns:
+            Text: Self is returned to method may be chained.
+        """
+        meta = {} if meta is None else meta
+        meta.update({f"@{key}": value for key, value in handlers.items()})
+        self.stylize(Style.from_meta(meta))
+        return self
+
+    def remove_suffix(self, suffix: str) -> None:
+        """Remove a suffix if it exists.
+
+        Args:
+            suffix (str): Suffix to remove.
+        """
+        if self.plain.endswith(suffix):
+            self.right_crop(len(suffix))
+
+    def get_style_at_offset(self, console: "Console", offset: int) -> Style:
+        """Get the style of a character at give offset.
+
+        Args:
+            console (~Console): Console where text will be rendered.
+            offset (int): Offset in to text (negative indexing supported)
+
+        Returns:
+            Style: A Style instance.
+        """
+        # TODO: This is a little inefficient, it is only used by full justify
+        if offset < 0:
+            offset = len(self) + offset
+        get_style = console.get_style
+        style = get_style(self.style).copy()
+        for start, end, span_style in self._spans:
+            if end > offset >= start:
+                style += get_style(span_style, default="")
+        return style
+
+    def extend_style(self, spaces: int) -> None:
+        """Extend the Text given number of spaces where the spaces have the same style as the last character.
+
+        Args:
+            spaces (int): Number of spaces to add to the Text.
+        """
+        if spaces <= 0:
+            return
+        spans = self.spans
+        new_spaces = " " * spaces
+        if spans:
+            end_offset = len(self)
+            self._spans[:] = [
+                span.extend(spaces) if span.end >= end_offset else span
+                for span in spans
+            ]
+            self._text.append(new_spaces)
+            self._length += spaces
+        else:
+            self.plain += new_spaces
+
+    def highlight_regex(
+        self,
+        re_highlight: Union[Pattern[str], str],
+        style: Optional[Union[GetStyleCallable, StyleType]] = None,
+        *,
+        style_prefix: str = "",
+    ) -> int:
+        """Highlight text with a regular expression, where group names are
+        translated to styles.
+
+        Args:
+            re_highlight (Union[re.Pattern, str]): A regular expression object or string.
+            style (Union[GetStyleCallable, StyleType]): Optional style to apply to whole match, or a callable
+                which accepts the matched text and returns a style. Defaults to None.
+            style_prefix (str, optional): Optional prefix to add to style group names.
+
+        Returns:
+            int: Number of regex matches
+        """
+        count = 0
+        append_span = self._spans.append
+        _Span = Span
+        plain = self.plain
+        if isinstance(re_highlight, str):
+            re_highlight = re.compile(re_highlight)
+        for match in re_highlight.finditer(plain):
+            get_span = match.span
+            if style:
+                start, end = get_span()
+                match_style = style(plain[start:end]) if callable(style) else style
+                if match_style is not None and end > start:
+                    append_span(_Span(start, end, match_style))
+
+            count += 1
+            for name in match.groupdict().keys():
+                start, end = get_span(name)
+                if start != -1 and end > start:
+                    append_span(_Span(start, end, f"{style_prefix}{name}"))
+        return count
+
+    def highlight_words(
+        self,
+        words: Iterable[str],
+        style: Union[str, Style],
+        *,
+        case_sensitive: bool = True,
+    ) -> int:
+        """Highlight words with a style.
+
+        Args:
+            words (Iterable[str]): Words to highlight.
+            style (Union[str, Style]): Style to apply.
+            case_sensitive (bool, optional): Enable case sensitive matching. Defaults to True.
+
+        Returns:
+            int: Number of words highlighted.
+        """
+        re_words = "|".join(re.escape(word) for word in words)
+        add_span = self._spans.append
+        count = 0
+        _Span = Span
+        for match in re.finditer(
+            re_words, self.plain, flags=0 if case_sensitive else re.IGNORECASE
+        ):
+            start, end = match.span(0)
+            add_span(_Span(start, end, style))
+            count += 1
+        return count
+
+    def rstrip(self) -> None:
+        """Strip whitespace from end of text."""
+        self.plain = self.plain.rstrip()
+
+    def rstrip_end(self, size: int) -> None:
+        """Remove whitespace beyond a certain width at the end of the text.
+
+        Args:
+            size (int): The desired size of the text.
+        """
+        text_length = len(self)
+        if text_length > size:
+            excess = text_length - size
+            whitespace_match = _re_whitespace.search(self.plain)
+            if whitespace_match is not None:
+                whitespace_count = len(whitespace_match.group(0))
+                self.right_crop(min(whitespace_count, excess))
+
+    def set_length(self, new_length: int) -> None:
+        """Set new length of the text, clipping or padding is required."""
+        length = len(self)
+        if length != new_length:
+            if length < new_length:
+                self.pad_right(new_length - length)
+            else:
+                self.right_crop(length - new_length)
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> Iterable[Segment]:
+        tab_size: int = console.tab_size if self.tab_size is None else self.tab_size
+        justify = self.justify or options.justify or DEFAULT_JUSTIFY
+        overflow = self.overflow or options.overflow or DEFAULT_OVERFLOW
+
+        lines = self.wrap(
+            console,
+            options.max_width,
+            justify=justify,
+            overflow=overflow,
+            tab_size=tab_size or 8,
+            no_wrap=pick_bool(self.no_wrap, options.no_wrap, False),
+        )
+        all_lines = Text("\n").join(lines)
+        yield from all_lines.render(console, end=self.end)
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> Measurement:
+        text = self.plain
+        lines = text.splitlines()
+        max_text_width = max(cell_len(line) for line in lines) if lines else 0
+        words = text.split()
+        min_text_width = (
+            max(cell_len(word) for word in words) if words else max_text_width
+        )
+        return Measurement(min_text_width, max_text_width)
+
+    def render(self, console: "Console", end: str = "") -> Iterable["Segment"]:
+        """Render the text as Segments.
+
+        Args:
+            console (Console): Console instance.
+            end (Optional[str], optional): Optional end character.
+
+        Returns:
+            Iterable[Segment]: Result of render that may be written to the console.
+        """
+        _Segment = Segment
+        text = self.plain
+        if not self._spans:
+            yield Segment(text)
+            if end:
+                yield _Segment(end)
+            return
+        get_style = partial(console.get_style, default=Style.null())
+
+        enumerated_spans = list(enumerate(self._spans, 1))
+        style_map = {index: get_style(span.style) for index, span in enumerated_spans}
+        style_map[0] = get_style(self.style)
+
+        spans = [
+            (0, False, 0),
+            *((span.start, False, index) for index, span in enumerated_spans),
+            *((span.end, True, index) for index, span in enumerated_spans),
+            (len(text), True, 0),
+        ]
+        spans.sort(key=itemgetter(0, 1))
+
+        stack: List[int] = []
+        stack_append = stack.append
+        stack_pop = stack.remove
+
+        style_cache: Dict[Tuple[Style, ...], Style] = {}
+        style_cache_get = style_cache.get
+        combine = Style.combine
+
+        def get_current_style() -> Style:
+            """Construct current style from stack."""
+            styles = tuple(style_map[_style_id] for _style_id in sorted(stack))
+            cached_style = style_cache_get(styles)
+            if cached_style is not None:
+                return cached_style
+            current_style = combine(styles)
+            style_cache[styles] = current_style
+            return current_style
+
+        for (offset, leaving, style_id), (next_offset, _, _) in zip(spans, spans[1:]):
+            if leaving:
+                stack_pop(style_id)
+            else:
+                stack_append(style_id)
+            if next_offset > offset:
+                yield _Segment(text[offset:next_offset], get_current_style())
+        if end:
+            yield _Segment(end)
+
+    def join(self, lines: Iterable["Text"]) -> "Text":
+        """Join text together with this instance as the separator.
+
+        Args:
+            lines (Iterable[Text]): An iterable of Text instances to join.
+
+        Returns:
+            Text: A new text instance containing join text.
+        """
+
+        new_text = self.blank_copy()
+
+        def iter_text() -> Iterable["Text"]:
+            if self.plain:
+                for last, line in loop_last(lines):
+                    yield line
+                    if not last:
+                        yield self
+            else:
+                yield from lines
+
+        extend_text = new_text._text.extend
+        append_span = new_text._spans.append
+        extend_spans = new_text._spans.extend
+        offset = 0
+        _Span = Span
+
+        for text in iter_text():
+            extend_text(text._text)
+            if text.style:
+                append_span(_Span(offset, offset + len(text), text.style))
+            extend_spans(
+                _Span(offset + start, offset + end, style)
+                for start, end, style in text._spans
+            )
+            offset += len(text)
+        new_text._length = offset
+        return new_text
+
+    def expand_tabs(self, tab_size: Optional[int] = None) -> None:
+        """Converts tabs to spaces.
+
+        Args:
+            tab_size (int, optional): Size of tabs. Defaults to 8.
+
+        """
+        if "\t" not in self.plain:
+            return
+        if tab_size is None:
+            tab_size = self.tab_size
+        if tab_size is None:
+            tab_size = 8
+
+        new_text: List[Text] = []
+        append = new_text.append
+
+        for line in self.split("\n", include_separator=True):
+            if "\t" not in line.plain:
+                append(line)
+            else:
+                cell_position = 0
+                parts = line.split("\t", include_separator=True)
+                for part in parts:
+                    if part.plain.endswith("\t"):
+                        part._text[-1] = part._text[-1][:-1] + " "
+                        cell_position += part.cell_len
+                        tab_remainder = cell_position % tab_size
+                        if tab_remainder:
+                            spaces = tab_size - tab_remainder
+                            part.extend_style(spaces)
+                            cell_position += spaces
+                    else:
+                        cell_position += part.cell_len
+                    append(part)
+
+        result = Text("").join(new_text)
+
+        self._text = [result.plain]
+        self._length = len(self.plain)
+        self._spans[:] = result._spans
+
+    def truncate(
+        self,
+        max_width: int,
+        *,
+        overflow: Optional["OverflowMethod"] = None,
+        pad: bool = False,
+    ) -> None:
+        """Truncate text if it is longer that a given width.
+
+        Args:
+            max_width (int): Maximum number of characters in text.
+            overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None, to use self.overflow.
+            pad (bool, optional): Pad with spaces if the length is less than max_width. Defaults to False.
+        """
+        _overflow = overflow or self.overflow or DEFAULT_OVERFLOW
+        if _overflow != "ignore":
+            length = cell_len(self.plain)
+            if length > max_width:
+                if _overflow == "ellipsis":
+                    self.plain = set_cell_size(self.plain, max_width - 1) + "…"
+                else:
+                    self.plain = set_cell_size(self.plain, max_width)
+            if pad and length < max_width:
+                spaces = max_width - length
+                self._text = [f"{self.plain}{' ' * spaces}"]
+                self._length = len(self.plain)
+
+    def _trim_spans(self) -> None:
+        """Remove or modify any spans that are over the end of the text."""
+        max_offset = len(self.plain)
+        _Span = Span
+        self._spans[:] = [
+            (
+                span
+                if span.end < max_offset
+                else _Span(span.start, min(max_offset, span.end), span.style)
+            )
+            for span in self._spans
+            if span.start < max_offset
+        ]
+
+    def pad(self, count: int, character: str = " ") -> None:
+        """Pad left and right with a given number of characters.
+
+        Args:
+            count (int): Width of padding.
+            character (str): The character to pad with. Must be a string of length 1.
+        """
+        assert len(character) == 1, "Character must be a string of length 1"
+        if count:
+            pad_characters = character * count
+            self.plain = f"{pad_characters}{self.plain}{pad_characters}"
+            _Span = Span
+            self._spans[:] = [
+                _Span(start + count, end + count, style)
+                for start, end, style in self._spans
+            ]
+
+    def pad_left(self, count: int, character: str = " ") -> None:
+        """Pad the left with a given character.
+
+        Args:
+            count (int): Number of characters to pad.
+            character (str, optional): Character to pad with. Defaults to " ".
+        """
+        assert len(character) == 1, "Character must be a string of length 1"
+        if count:
+            self.plain = f"{character * count}{self.plain}"
+            _Span = Span
+            self._spans[:] = [
+                _Span(start + count, end + count, style)
+                for start, end, style in self._spans
+            ]
+
+    def pad_right(self, count: int, character: str = " ") -> None:
+        """Pad the right with a given character.
+
+        Args:
+            count (int): Number of characters to pad.
+            character (str, optional): Character to pad with. Defaults to " ".
+        """
+        assert len(character) == 1, "Character must be a string of length 1"
+        if count:
+            self.plain = f"{self.plain}{character * count}"
+
+    def align(self, align: AlignMethod, width: int, character: str = " ") -> None:
+        """Align text to a given width.
+
+        Args:
+            align (AlignMethod): One of "left", "center", or "right".
+            width (int): Desired width.
+            character (str, optional): Character to pad with. Defaults to " ".
+        """
+        self.truncate(width)
+        excess_space = width - cell_len(self.plain)
+        if excess_space:
+            if align == "left":
+                self.pad_right(excess_space, character)
+            elif align == "center":
+                left = excess_space // 2
+                self.pad_left(left, character)
+                self.pad_right(excess_space - left, character)
+            else:
+                self.pad_left(excess_space, character)
+
+    def append(
+        self, text: Union["Text", str], style: Optional[Union[str, "Style"]] = None
+    ) -> "Text":
+        """Add text with an optional style.
+
+        Args:
+            text (Union[Text, str]): A str or Text to append.
+            style (str, optional): A style name. Defaults to None.
+
+        Returns:
+            Text: Returns self for chaining.
+        """
+
+        if not isinstance(text, (str, Text)):
+            raise TypeError("Only str or Text can be appended to Text")
+
+        if len(text):
+            if isinstance(text, str):
+                sanitized_text = strip_control_codes(text)
+                self._text.append(sanitized_text)
+                offset = len(self)
+                text_length = len(sanitized_text)
+                if style:
+                    self._spans.append(Span(offset, offset + text_length, style))
+                self._length += text_length
+            elif isinstance(text, Text):
+                _Span = Span
+                if style is not None:
+                    raise ValueError(
+                        "style must not be set when appending Text instance"
+                    )
+                text_length = self._length
+                if text.style:
+                    self._spans.append(
+                        _Span(text_length, text_length + len(text), text.style)
+                    )
+                self._text.append(text.plain)
+                self._spans.extend(
+                    _Span(start + text_length, end + text_length, style)
+                    for start, end, style in text._spans.copy()
+                )
+                self._length += len(text)
+        return self
+
+    def append_text(self, text: "Text") -> "Text":
+        """Append another Text instance. This method is more performant than Text.append, but
+        only works for Text.
+
+        Args:
+            text (Text): The Text instance to append to this instance.
+
+        Returns:
+            Text: Returns self for chaining.
+        """
+        _Span = Span
+        text_length = self._length
+        if text.style:
+            self._spans.append(_Span(text_length, text_length + len(text), text.style))
+        self._text.append(text.plain)
+        self._spans.extend(
+            _Span(start + text_length, end + text_length, style)
+            for start, end, style in text._spans.copy()
+        )
+        self._length += len(text)
+        return self
+
+    def append_tokens(
+        self, tokens: Iterable[Tuple[str, Optional[StyleType]]]
+    ) -> "Text":
+        """Append iterable of str and style. Style may be a Style instance or a str style definition.
+
+        Args:
+            tokens (Iterable[Tuple[str, Optional[StyleType]]]): An iterable of tuples containing str content and style.
+
+        Returns:
+            Text: Returns self for chaining.
+        """
+        append_text = self._text.append
+        append_span = self._spans.append
+        _Span = Span
+        offset = len(self)
+        for content, style in tokens:
+            content = strip_control_codes(content)
+            append_text(content)
+            if style:
+                append_span(_Span(offset, offset + len(content), style))
+            offset += len(content)
+        self._length = offset
+        return self
+
+    def copy_styles(self, text: "Text") -> None:
+        """Copy styles from another Text instance.
+
+        Args:
+            text (Text): A Text instance to copy styles from, must be the same length.
+        """
+        self._spans.extend(text._spans)
+
+    def split(
+        self,
+        separator: str = "\n",
+        *,
+        include_separator: bool = False,
+        allow_blank: bool = False,
+    ) -> Lines:
+        """Split rich text in to lines, preserving styles.
+
+        Args:
+            separator (str, optional): String to split on. Defaults to "\\\\n".
+            include_separator (bool, optional): Include the separator in the lines. Defaults to False.
+            allow_blank (bool, optional): Return a blank line if the text ends with a separator. Defaults to False.
+
+        Returns:
+            List[RichText]: A list of rich text, one per line of the original.
+        """
+        assert separator, "separator must not be empty"
+
+        text = self.plain
+        if separator not in text:
+            return Lines([self.copy()])
+
+        if include_separator:
+            lines = self.divide(
+                match.end() for match in re.finditer(re.escape(separator), text)
+            )
+        else:
+
+            def flatten_spans() -> Iterable[int]:
+                for match in re.finditer(re.escape(separator), text):
+                    start, end = match.span()
+                    yield start
+                    yield end
+
+            lines = Lines(
+                line for line in self.divide(flatten_spans()) if line.plain != separator
+            )
+
+        if not allow_blank and text.endswith(separator):
+            lines.pop()
+
+        return lines
+
+    def divide(self, offsets: Iterable[int]) -> Lines:
+        """Divide text into a number of lines at given offsets.
+
+        Args:
+            offsets (Iterable[int]): Offsets used to divide text.
+
+        Returns:
+            Lines: New RichText instances between offsets.
+        """
+        _offsets = list(offsets)
+
+        if not _offsets:
+            return Lines([self.copy()])
+
+        text = self.plain
+        text_length = len(text)
+        divide_offsets = [0, *_offsets, text_length]
+        line_ranges = list(zip(divide_offsets, divide_offsets[1:]))
+
+        style = self.style
+        justify = self.justify
+        overflow = self.overflow
+        _Text = Text
+        new_lines = Lines(
+            _Text(
+                text[start:end],
+                style=style,
+                justify=justify,
+                overflow=overflow,
+            )
+            for start, end in line_ranges
+        )
+        if not self._spans:
+            return new_lines
+
+        _line_appends = [line._spans.append for line in new_lines._lines]
+        line_count = len(line_ranges)
+        _Span = Span
+
+        for span_start, span_end, style in self._spans:
+            lower_bound = 0
+            upper_bound = line_count
+            start_line_no = (lower_bound + upper_bound) // 2
+
+            while True:
+                line_start, line_end = line_ranges[start_line_no]
+                if span_start < line_start:
+                    upper_bound = start_line_no - 1
+                elif span_start > line_end:
+                    lower_bound = start_line_no + 1
+                else:
+                    break
+                start_line_no = (lower_bound + upper_bound) // 2
+
+            if span_end < line_end:
+                end_line_no = start_line_no
+            else:
+                end_line_no = lower_bound = start_line_no
+                upper_bound = line_count
+
+                while True:
+                    line_start, line_end = line_ranges[end_line_no]
+                    if span_end < line_start:
+                        upper_bound = end_line_no - 1
+                    elif span_end > line_end:
+                        lower_bound = end_line_no + 1
+                    else:
+                        break
+                    end_line_no = (lower_bound + upper_bound) // 2
+
+            for line_no in range(start_line_no, end_line_no + 1):
+                line_start, line_end = line_ranges[line_no]
+                new_start = max(0, span_start - line_start)
+                new_end = min(span_end - line_start, line_end - line_start)
+                if new_end > new_start:
+                    _line_appends[line_no](_Span(new_start, new_end, style))
+
+        return new_lines
+
+    def right_crop(self, amount: int = 1) -> None:
+        """Remove a number of characters from the end of the text."""
+        max_offset = len(self.plain) - amount
+        _Span = Span
+        self._spans[:] = [
+            (
+                span
+                if span.end < max_offset
+                else _Span(span.start, min(max_offset, span.end), span.style)
+            )
+            for span in self._spans
+            if span.start < max_offset
+        ]
+        self._text = [self.plain[:-amount]]
+        self._length -= amount
+
+    def wrap(
+        self,
+        console: "Console",
+        width: int,
+        *,
+        justify: Optional["JustifyMethod"] = None,
+        overflow: Optional["OverflowMethod"] = None,
+        tab_size: int = 8,
+        no_wrap: Optional[bool] = None,
+    ) -> Lines:
+        """Word wrap the text.
+
+        Args:
+            console (Console): Console instance.
+            width (int): Number of cells available per line.
+            justify (str, optional): Justify method: "default", "left", "center", "full", "right". Defaults to "default".
+            overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None.
+            tab_size (int, optional): Default tab size. Defaults to 8.
+            no_wrap (bool, optional): Disable wrapping, Defaults to False.
+
+        Returns:
+            Lines: Number of lines.
+        """
+        wrap_justify = justify or self.justify or DEFAULT_JUSTIFY
+        wrap_overflow = overflow or self.overflow or DEFAULT_OVERFLOW
+
+        no_wrap = pick_bool(no_wrap, self.no_wrap, False) or overflow == "ignore"
+
+        lines = Lines()
+        for line in self.split(allow_blank=True):
+            if "\t" in line:
+                line.expand_tabs(tab_size)
+            if no_wrap:
+                if overflow == "ignore":
+                    lines.append(line)
+                    continue
+                new_lines = Lines([line])
+            else:
+                offsets = divide_line(str(line), width, fold=wrap_overflow == "fold")
+                new_lines = line.divide(offsets)
+                for line in new_lines:
+                    line.rstrip_end(width)
+            if wrap_justify:
+                new_lines.justify(
+                    console, width, justify=wrap_justify, overflow=wrap_overflow
+                )
+            for line in new_lines:
+                line.truncate(width, overflow=wrap_overflow)
+            lines.extend(new_lines)
+        return lines
+
+    def fit(self, width: int) -> Lines:
+        """Fit the text in to given width by chopping in to lines.
+
+        Args:
+            width (int): Maximum characters in a line.
+
+        Returns:
+            Lines: Lines container.
+        """
+        lines: Lines = Lines()
+        append = lines.append
+        for line in self.split():
+            line.set_length(width)
+            append(line)
+        return lines
+
+    def detect_indentation(self) -> int:
+        """Auto-detect indentation of code.
+
+        Returns:
+            int: Number of spaces used to indent code.
+        """
+
+        _indentations = {
+            len(match.group(1))
+            for match in re.finditer(r"^( *)(.*)$", self.plain, flags=re.MULTILINE)
+        }
+
+        try:
+            indentation = (
+                reduce(gcd, [indent for indent in _indentations if not indent % 2]) or 1
+            )
+        except TypeError:
+            indentation = 1
+
+        return indentation
+
+    def with_indent_guides(
+        self,
+        indent_size: Optional[int] = None,
+        *,
+        character: str = "│",
+        style: StyleType = "dim green",
+    ) -> "Text":
+        """Adds indent guide lines to text.
+
+        Args:
+            indent_size (Optional[int]): Size of indentation, or None to auto detect. Defaults to None.
+            character (str, optional): Character to use for indentation. Defaults to "│".
+            style (Union[Style, str], optional): Style of indent guides.
+
+        Returns:
+            Text: New text with indentation guides.
+        """
+
+        _indent_size = self.detect_indentation() if indent_size is None else indent_size
+
+        text = self.copy()
+        text.expand_tabs()
+        indent_line = f"{character}{' ' * (_indent_size - 1)}"
+
+        re_indent = re.compile(r"^( *)(.*)$")
+        new_lines: List[Text] = []
+        add_line = new_lines.append
+        blank_lines = 0
+        for line in text.split(allow_blank=True):
+            match = re_indent.match(line.plain)
+            if not match or not match.group(2):
+                blank_lines += 1
+                continue
+            indent = match.group(1)
+            full_indents, remaining_space = divmod(len(indent), _indent_size)
+            new_indent = f"{indent_line * full_indents}{' ' * remaining_space}"
+            line.plain = new_indent + line.plain[len(new_indent) :]
+            line.stylize(style, 0, len(new_indent))
+            if blank_lines:
+                new_lines.extend([Text(new_indent, style=style)] * blank_lines)
+                blank_lines = 0
+            add_line(line)
+        if blank_lines:
+            new_lines.extend([Text("", style=style)] * blank_lines)
+
+        new_text = text.blank_copy("\n").join(new_lines)
+        return new_text
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich.console import Console
+
+    text = Text(
+        """\nLorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n"""
+    )
+    text.highlight_words(["Lorem"], "bold")
+    text.highlight_words(["ipsum"], "italic")
+
+    console = Console()
+
+    console.rule("justify='left'")
+    console.print(text, style="red")
+    console.print()
+
+    console.rule("justify='center'")
+    console.print(text, style="green", justify="center")
+    console.print()
+
+    console.rule("justify='right'")
+    console.print(text, style="blue", justify="right")
+    console.print()
+
+    console.rule("justify='full'")
+    console.print(text, style="magenta", justify="full")
+    console.print()
diff --git a/lib/rich/theme.py b/lib/rich/theme.py
new file mode 100644
index 0000000..227f1d8
--- /dev/null
+++ b/lib/rich/theme.py
@@ -0,0 +1,115 @@
+import configparser
+from typing import IO, Dict, List, Mapping, Optional
+
+from .default_styles import DEFAULT_STYLES
+from .style import Style, StyleType
+
+
+class Theme:
+    """A container for style information, used by :class:`~rich.console.Console`.
+
+    Args:
+        styles (Dict[str, Style], optional): A mapping of style names on to styles. Defaults to None for a theme with no styles.
+        inherit (bool, optional): Inherit default styles. Defaults to True.
+    """
+
+    styles: Dict[str, Style]
+
+    def __init__(
+        self, styles: Optional[Mapping[str, StyleType]] = None, inherit: bool = True
+    ):
+        self.styles = DEFAULT_STYLES.copy() if inherit else {}
+        if styles is not None:
+            self.styles.update(
+                {
+                    name: style if isinstance(style, Style) else Style.parse(style)
+                    for name, style in styles.items()
+                }
+            )
+
+    @property
+    def config(self) -> str:
+        """Get contents of a config file for this theme."""
+        config = "[styles]\n" + "\n".join(
+            f"{name} = {style}" for name, style in sorted(self.styles.items())
+        )
+        return config
+
+    @classmethod
+    def from_file(
+        cls, config_file: IO[str], source: Optional[str] = None, inherit: bool = True
+    ) -> "Theme":
+        """Load a theme from a text mode file.
+
+        Args:
+            config_file (IO[str]): An open conf file.
+            source (str, optional): The filename of the open file. Defaults to None.
+            inherit (bool, optional): Inherit default styles. Defaults to True.
+
+        Returns:
+            Theme: A New theme instance.
+        """
+        config = configparser.ConfigParser()
+        config.read_file(config_file, source=source)
+        styles = {name: Style.parse(value) for name, value in config.items("styles")}
+        theme = Theme(styles, inherit=inherit)
+        return theme
+
+    @classmethod
+    def read(
+        cls, path: str, inherit: bool = True, encoding: Optional[str] = None
+    ) -> "Theme":
+        """Read a theme from a path.
+
+        Args:
+            path (str): Path to a config file readable by Python configparser module.
+            inherit (bool, optional): Inherit default styles. Defaults to True.
+            encoding (str, optional): Encoding of the config file. Defaults to None.
+
+        Returns:
+            Theme: A new theme instance.
+        """
+        with open(path, encoding=encoding) as config_file:
+            return cls.from_file(config_file, source=path, inherit=inherit)
+
+
+class ThemeStackError(Exception):
+    """Base exception for errors related to the theme stack."""
+
+
+class ThemeStack:
+    """A stack of themes.
+
+    Args:
+        theme (Theme): A theme instance
+    """
+
+    def __init__(self, theme: Theme) -> None:
+        self._entries: List[Dict[str, Style]] = [theme.styles]
+        self.get = self._entries[-1].get
+
+    def push_theme(self, theme: Theme, inherit: bool = True) -> None:
+        """Push a theme on the top of the stack.
+
+        Args:
+            theme (Theme): A Theme instance.
+            inherit (boolean, optional): Inherit styles from current top of stack.
+        """
+        styles: Dict[str, Style]
+        styles = (
+            {**self._entries[-1], **theme.styles} if inherit else theme.styles.copy()
+        )
+        self._entries.append(styles)
+        self.get = self._entries[-1].get
+
+    def pop_theme(self) -> None:
+        """Pop (and discard) the top-most theme."""
+        if len(self._entries) == 1:
+            raise ThemeStackError("Unable to pop base theme")
+        self._entries.pop()
+        self.get = self._entries[-1].get
+
+
+if __name__ == "__main__":  # pragma: no cover
+    theme = Theme()
+    print(theme.config)
diff --git a/lib/rich/themes.py b/lib/rich/themes.py
new file mode 100644
index 0000000..bf6db10
--- /dev/null
+++ b/lib/rich/themes.py
@@ -0,0 +1,5 @@
+from .default_styles import DEFAULT_STYLES
+from .theme import Theme
+
+
+DEFAULT = Theme(DEFAULT_STYLES)
diff --git a/lib/rich/traceback.py b/lib/rich/traceback.py
new file mode 100644
index 0000000..66eaeca
--- /dev/null
+++ b/lib/rich/traceback.py
@@ -0,0 +1,924 @@
+import inspect
+import linecache
+import os
+import sys
+from dataclasses import dataclass, field
+from itertools import islice
+from traceback import walk_tb
+from types import ModuleType, TracebackType
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
+
+from pygments.lexers import guess_lexer_for_filename
+from pygments.token import Comment, Keyword, Name, Number, Operator, String
+from pygments.token import Text as TextToken
+from pygments.token import Token
+from pygments.util import ClassNotFound
+
+from . import pretty
+from ._loop import loop_first_last, loop_last
+from .columns import Columns
+from .console import (
+    Console,
+    ConsoleOptions,
+    ConsoleRenderable,
+    OverflowMethod,
+    Group,
+    RenderResult,
+    group,
+)
+from .constrain import Constrain
+from .highlighter import RegexHighlighter, ReprHighlighter
+from .panel import Panel
+from .scope import render_scope
+from .style import Style
+from .syntax import Syntax, SyntaxPosition
+from .text import Text
+from .theme import Theme
+
+WINDOWS = sys.platform == "win32"
+
+LOCALS_MAX_LENGTH = 10
+LOCALS_MAX_STRING = 80
+
+
+def _iter_syntax_lines(
+    start: SyntaxPosition, end: SyntaxPosition
+) -> Iterable[Tuple[int, int, int]]:
+    """Yield start and end positions per line.
+
+    Args:
+        start: Start position.
+        end: End position.
+
+    Returns:
+        Iterable of (LINE, COLUMN1, COLUMN2).
+    """
+
+    line1, column1 = start
+    line2, column2 = end
+
+    if line1 == line2:
+        yield line1, column1, column2
+    else:
+        for first, last, line_no in loop_first_last(range(line1, line2 + 1)):
+            if first:
+                yield line_no, column1, -1
+            elif last:
+                yield line_no, 0, column2
+            else:
+                yield line_no, 0, -1
+
+
+def install(
+    *,
+    console: Optional[Console] = None,
+    width: Optional[int] = 100,
+    code_width: Optional[int] = 88,
+    extra_lines: int = 3,
+    theme: Optional[str] = None,
+    word_wrap: bool = False,
+    show_locals: bool = False,
+    locals_max_length: int = LOCALS_MAX_LENGTH,
+    locals_max_string: int = LOCALS_MAX_STRING,
+    locals_max_depth: Optional[int] = None,
+    locals_hide_dunder: bool = True,
+    locals_hide_sunder: Optional[bool] = None,
+    locals_overflow: Optional[OverflowMethod] = None,
+    indent_guides: bool = True,
+    suppress: Iterable[Union[str, ModuleType]] = (),
+    max_frames: int = 100,
+) -> Callable[[Type[BaseException], BaseException, Optional[TracebackType]], Any]:
+    """Install a rich traceback handler.
+
+    Once installed, any tracebacks will be printed with syntax highlighting and rich formatting.
+
+
+    Args:
+        console (Optional[Console], optional): Console to write exception to. Default uses internal Console instance.
+        width (Optional[int], optional): Width (in characters) of traceback. Defaults to 100.
+        code_width (Optional[int], optional): Code width (in characters) of traceback. Defaults to 88.
+        extra_lines (int, optional): Extra lines of code. Defaults to 3.
+        theme (Optional[str], optional): Pygments theme to use in traceback. Defaults to ``None`` which will pick
+            a theme appropriate for the platform.
+        word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False.
+        show_locals (bool, optional): Enable display of local variables. Defaults to False.
+        locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to 10.
+        locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80.
+        locals_max_depth (int, optional): Maximum depths of locals before truncating, or None to disable. Defaults to None.
+        locals_hide_dunder (bool, optional): Hide locals prefixed with double underscore. Defaults to True.
+        locals_hide_sunder (bool, optional): Hide locals prefixed with single underscore. Defaults to False.
+        locals_overflow (OverflowMethod, optional): How to handle overflowing locals, or None to disable. Defaults to None.
+        indent_guides (bool, optional): Enable indent guides in code and locals. Defaults to True.
+        suppress (Sequence[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback.
+
+    Returns:
+        Callable: The previous exception handler that was replaced.
+
+    """
+    traceback_console = Console(stderr=True) if console is None else console
+
+    locals_hide_sunder = (
+        True
+        if (traceback_console.is_jupyter and locals_hide_sunder is None)
+        else locals_hide_sunder
+    )
+
+    def excepthook(
+        type_: Type[BaseException],
+        value: BaseException,
+        traceback: Optional[TracebackType],
+    ) -> None:
+        exception_traceback = Traceback.from_exception(
+            type_,
+            value,
+            traceback,
+            width=width,
+            code_width=code_width,
+            extra_lines=extra_lines,
+            theme=theme,
+            word_wrap=word_wrap,
+            show_locals=show_locals,
+            locals_max_length=locals_max_length,
+            locals_max_string=locals_max_string,
+            locals_max_depth=locals_max_depth,
+            locals_hide_dunder=locals_hide_dunder,
+            locals_hide_sunder=bool(locals_hide_sunder),
+            locals_overflow=locals_overflow,
+            indent_guides=indent_guides,
+            suppress=suppress,
+            max_frames=max_frames,
+        )
+        traceback_console.print(exception_traceback)
+
+    def ipy_excepthook_closure(ip: Any) -> None:  # pragma: no cover
+        tb_data = {}  # store information about showtraceback call
+        default_showtraceback = ip.showtraceback  # keep reference of default traceback
+
+        def ipy_show_traceback(*args: Any, **kwargs: Any) -> None:
+            """wrap the default ip.showtraceback to store info for ip._showtraceback"""
+            nonlocal tb_data
+            tb_data = kwargs
+            default_showtraceback(*args, **kwargs)
+
+        def ipy_display_traceback(
+            *args: Any, is_syntax: bool = False, **kwargs: Any
+        ) -> None:
+            """Internally called traceback from ip._showtraceback"""
+            nonlocal tb_data
+            exc_tuple = ip._get_exc_info()
+
+            # do not display trace on syntax error
+            tb: Optional[TracebackType] = None if is_syntax else exc_tuple[2]
+
+            # determine correct tb_offset
+            compiled = tb_data.get("running_compiled_code", False)
+            tb_offset = tb_data.get("tb_offset")
+            if tb_offset is None:
+                tb_offset = 1 if compiled else 0
+            # remove ipython internal frames from trace with tb_offset
+            for _ in range(tb_offset):
+                if tb is None:
+                    break
+                tb = tb.tb_next
+
+            excepthook(exc_tuple[0], exc_tuple[1], tb)
+            tb_data = {}  # clear data upon usage
+
+        # replace _showtraceback instead of showtraceback to allow ipython features such as debugging to work
+        # this is also what the ipython docs recommends to modify when subclassing InteractiveShell
+        ip._showtraceback = ipy_display_traceback
+        # add wrapper to capture tb_data
+        ip.showtraceback = ipy_show_traceback
+        ip.showsyntaxerror = lambda *args, **kwargs: ipy_display_traceback(
+            *args, is_syntax=True, **kwargs
+        )
+
+    try:  # pragma: no cover
+        # if within ipython, use customized traceback
+        ip = get_ipython()  # type: ignore[name-defined]
+        ipy_excepthook_closure(ip)
+        return sys.excepthook
+    except Exception:
+        # otherwise use default system hook
+        old_excepthook = sys.excepthook
+        sys.excepthook = excepthook
+        return old_excepthook
+
+
+@dataclass
+class Frame:
+    filename: str
+    lineno: int
+    name: str
+    line: str = ""
+    locals: Optional[Dict[str, pretty.Node]] = None
+    last_instruction: Optional[Tuple[Tuple[int, int], Tuple[int, int]]] = None
+
+
+@dataclass
+class _SyntaxError:
+    offset: int
+    filename: str
+    line: str
+    lineno: int
+    msg: str
+    notes: List[str] = field(default_factory=list)
+
+
+@dataclass
+class Stack:
+    exc_type: str
+    exc_value: str
+    syntax_error: Optional[_SyntaxError] = None
+    is_cause: bool = False
+    frames: List[Frame] = field(default_factory=list)
+    notes: List[str] = field(default_factory=list)
+    is_group: bool = False
+    exceptions: List["Trace"] = field(default_factory=list)
+
+
+@dataclass
+class Trace:
+    stacks: List[Stack]
+
+
+class PathHighlighter(RegexHighlighter):
+    highlights = [r"(?P.*/)(?P.+)"]
+
+
+class Traceback:
+    """A Console renderable that renders a traceback.
+
+    Args:
+        trace (Trace, optional): A `Trace` object produced from `extract`. Defaults to None, which uses
+            the last exception.
+        width (Optional[int], optional): Number of characters used to traceback. Defaults to 100.
+        code_width (Optional[int], optional): Number of code characters used to traceback. Defaults to 88.
+        extra_lines (int, optional): Additional lines of code to render. Defaults to 3.
+        theme (str, optional): Override pygments theme used in traceback.
+        word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False.
+        show_locals (bool, optional): Enable display of local variables. Defaults to False.
+        indent_guides (bool, optional): Enable indent guides in code and locals. Defaults to True.
+        locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+            Defaults to 10.
+        locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80.
+        locals_max_depth (int, optional): Maximum depths of locals before truncating, or None to disable. Defaults to None.
+        locals_hide_dunder (bool, optional): Hide locals prefixed with double underscore. Defaults to True.
+        locals_hide_sunder (bool, optional): Hide locals prefixed with single underscore. Defaults to False.
+        locals_overflow (OverflowMethod, optional): How to handle overflowing locals, or None to disable. Defaults to None.
+        suppress (Sequence[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback.
+        max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100.
+
+    """
+
+    LEXERS = {
+        "": "text",
+        ".py": "python",
+        ".pxd": "cython",
+        ".pyx": "cython",
+        ".pxi": "pyrex",
+    }
+
+    def __init__(
+        self,
+        trace: Optional[Trace] = None,
+        *,
+        width: Optional[int] = 100,
+        code_width: Optional[int] = 88,
+        extra_lines: int = 3,
+        theme: Optional[str] = None,
+        word_wrap: bool = False,
+        show_locals: bool = False,
+        locals_max_length: int = LOCALS_MAX_LENGTH,
+        locals_max_string: int = LOCALS_MAX_STRING,
+        locals_max_depth: Optional[int] = None,
+        locals_hide_dunder: bool = True,
+        locals_hide_sunder: bool = False,
+        locals_overlow: Optional[OverflowMethod] = None,
+        indent_guides: bool = True,
+        suppress: Iterable[Union[str, ModuleType]] = (),
+        max_frames: int = 100,
+    ):
+        if trace is None:
+            exc_type, exc_value, traceback = sys.exc_info()
+            if exc_type is None or exc_value is None or traceback is None:
+                raise ValueError(
+                    "Value for 'trace' required if not called in except: block"
+                )
+            trace = self.extract(
+                exc_type, exc_value, traceback, show_locals=show_locals
+            )
+        self.trace = trace
+        self.width = width
+        self.code_width = code_width
+        self.extra_lines = extra_lines
+        self.theme = Syntax.get_theme(theme or "ansi_dark")
+        self.word_wrap = word_wrap
+        self.show_locals = show_locals
+        self.indent_guides = indent_guides
+        self.locals_max_length = locals_max_length
+        self.locals_max_string = locals_max_string
+        self.locals_max_depth = locals_max_depth
+        self.locals_hide_dunder = locals_hide_dunder
+        self.locals_hide_sunder = locals_hide_sunder
+        self.locals_overflow = locals_overlow
+
+        self.suppress: Sequence[str] = []
+        for suppress_entity in suppress:
+            if not isinstance(suppress_entity, str):
+                assert (
+                    suppress_entity.__file__ is not None
+                ), f"{suppress_entity!r} must be a module with '__file__' attribute"
+                path = os.path.dirname(suppress_entity.__file__)
+            else:
+                path = suppress_entity
+            path = os.path.normpath(os.path.abspath(path))
+            self.suppress.append(path)
+        self.max_frames = max(4, max_frames) if max_frames > 0 else 0
+
+    @classmethod
+    def from_exception(
+        cls,
+        exc_type: Type[Any],
+        exc_value: BaseException,
+        traceback: Optional[TracebackType],
+        *,
+        width: Optional[int] = 100,
+        code_width: Optional[int] = 88,
+        extra_lines: int = 3,
+        theme: Optional[str] = None,
+        word_wrap: bool = False,
+        show_locals: bool = False,
+        locals_max_length: int = LOCALS_MAX_LENGTH,
+        locals_max_string: int = LOCALS_MAX_STRING,
+        locals_max_depth: Optional[int] = None,
+        locals_hide_dunder: bool = True,
+        locals_hide_sunder: bool = False,
+        locals_overflow: Optional[OverflowMethod] = None,
+        indent_guides: bool = True,
+        suppress: Iterable[Union[str, ModuleType]] = (),
+        max_frames: int = 100,
+    ) -> "Traceback":
+        """Create a traceback from exception info
+
+        Args:
+            exc_type (Type[BaseException]): Exception type.
+            exc_value (BaseException): Exception value.
+            traceback (TracebackType): Python Traceback object.
+            width (Optional[int], optional): Number of characters used to traceback. Defaults to 100.
+            code_width (Optional[int], optional): Number of code characters used to traceback. Defaults to 88.
+            extra_lines (int, optional): Additional lines of code to render. Defaults to 3.
+            theme (str, optional): Override pygments theme used in traceback.
+            word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False.
+            show_locals (bool, optional): Enable display of local variables. Defaults to False.
+            indent_guides (bool, optional): Enable indent guides in code and locals. Defaults to True.
+            locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+                Defaults to 10.
+            locals_max_depth (int, optional): Maximum depths of locals before truncating, or None to disable. Defaults to None.
+            locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80.
+            locals_hide_dunder (bool, optional): Hide locals prefixed with double underscore. Defaults to True.
+            locals_hide_sunder (bool, optional): Hide locals prefixed with single underscore. Defaults to False.
+            locals_overflow (OverflowMethod, optional): How to handle overflowing locals, or None to disable. Defaults to None.
+            suppress (Iterable[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback.
+            max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100.
+
+        Returns:
+            Traceback: A Traceback instance that may be printed.
+        """
+        rich_traceback = cls.extract(
+            exc_type,
+            exc_value,
+            traceback,
+            show_locals=show_locals,
+            locals_max_length=locals_max_length,
+            locals_max_string=locals_max_string,
+            locals_max_depth=locals_max_depth,
+            locals_hide_dunder=locals_hide_dunder,
+            locals_hide_sunder=locals_hide_sunder,
+        )
+
+        return cls(
+            rich_traceback,
+            width=width,
+            code_width=code_width,
+            extra_lines=extra_lines,
+            theme=theme,
+            word_wrap=word_wrap,
+            show_locals=show_locals,
+            indent_guides=indent_guides,
+            locals_max_length=locals_max_length,
+            locals_max_string=locals_max_string,
+            locals_max_depth=locals_max_depth,
+            locals_hide_dunder=locals_hide_dunder,
+            locals_hide_sunder=locals_hide_sunder,
+            locals_overlow=locals_overflow,
+            suppress=suppress,
+            max_frames=max_frames,
+        )
+
+    @classmethod
+    def extract(
+        cls,
+        exc_type: Type[BaseException],
+        exc_value: BaseException,
+        traceback: Optional[TracebackType],
+        *,
+        show_locals: bool = False,
+        locals_max_length: int = LOCALS_MAX_LENGTH,
+        locals_max_string: int = LOCALS_MAX_STRING,
+        locals_max_depth: Optional[int] = None,
+        locals_hide_dunder: bool = True,
+        locals_hide_sunder: bool = False,
+        _visited_exceptions: Optional[Set[BaseException]] = None,
+    ) -> Trace:
+        """Extract traceback information.
+
+        Args:
+            exc_type (Type[BaseException]): Exception type.
+            exc_value (BaseException): Exception value.
+            traceback (TracebackType): Python Traceback object.
+            show_locals (bool, optional): Enable display of local variables. Defaults to False.
+            locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation.
+                Defaults to 10.
+            locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80.
+            locals_max_depth (int, optional): Maximum depths of locals before truncating, or None to disable. Defaults to None.
+            locals_hide_dunder (bool, optional): Hide locals prefixed with double underscore. Defaults to True.
+            locals_hide_sunder (bool, optional): Hide locals prefixed with single underscore. Defaults to False.
+
+        Returns:
+            Trace: A Trace instance which you can use to construct a `Traceback`.
+        """
+
+        stacks: List[Stack] = []
+        is_cause = False
+
+        from rich import _IMPORT_CWD
+
+        notes: List[str] = getattr(exc_value, "__notes__", None) or []
+
+        grouped_exceptions: Set[BaseException] = (
+            set() if _visited_exceptions is None else _visited_exceptions
+        )
+
+        def safe_str(_object: Any) -> str:
+            """Don't allow exceptions from __str__ to propagate."""
+            try:
+                return str(_object)
+            except Exception:
+                return ""
+
+        while True:
+            stack = Stack(
+                exc_type=safe_str(exc_type.__name__),
+                exc_value=safe_str(exc_value),
+                is_cause=is_cause,
+                notes=notes,
+            )
+
+            if sys.version_info >= (3, 11):
+                if isinstance(exc_value, (BaseExceptionGroup, ExceptionGroup)):
+                    stack.is_group = True
+                    for exception in exc_value.exceptions:
+                        if exception in grouped_exceptions:
+                            continue
+                        grouped_exceptions.add(exception)
+                        stack.exceptions.append(
+                            Traceback.extract(
+                                type(exception),
+                                exception,
+                                exception.__traceback__,
+                                show_locals=show_locals,
+                                locals_max_length=locals_max_length,
+                                locals_hide_dunder=locals_hide_dunder,
+                                locals_hide_sunder=locals_hide_sunder,
+                                _visited_exceptions=grouped_exceptions,
+                            )
+                        )
+
+            if isinstance(exc_value, SyntaxError):
+                stack.syntax_error = _SyntaxError(
+                    offset=exc_value.offset or 0,
+                    filename=exc_value.filename or "?",
+                    lineno=exc_value.lineno or 0,
+                    line=exc_value.text or "",
+                    msg=exc_value.msg,
+                    notes=notes,
+                )
+
+            stacks.append(stack)
+            append = stack.frames.append
+
+            def get_locals(
+                iter_locals: Iterable[Tuple[str, object]],
+            ) -> Iterable[Tuple[str, object]]:
+                """Extract locals from an iterator of key pairs."""
+                if not (locals_hide_dunder or locals_hide_sunder):
+                    yield from iter_locals
+                    return
+                for key, value in iter_locals:
+                    if locals_hide_dunder and key.startswith("__"):
+                        continue
+                    if locals_hide_sunder and key.startswith("_"):
+                        continue
+                    yield key, value
+
+            for frame_summary, line_no in walk_tb(traceback):
+                filename = frame_summary.f_code.co_filename
+
+                last_instruction: Optional[Tuple[Tuple[int, int], Tuple[int, int]]]
+                last_instruction = None
+                if sys.version_info >= (3, 11):
+                    instruction_index = frame_summary.f_lasti // 2
+                    instruction_position = next(
+                        islice(
+                            frame_summary.f_code.co_positions(),
+                            instruction_index,
+                            instruction_index + 1,
+                        )
+                    )
+                    (
+                        start_line,
+                        end_line,
+                        start_column,
+                        end_column,
+                    ) = instruction_position
+                    if (
+                        start_line is not None
+                        and end_line is not None
+                        and start_column is not None
+                        and end_column is not None
+                    ):
+                        last_instruction = (
+                            (start_line, start_column),
+                            (end_line, end_column),
+                        )
+
+                if filename and not filename.startswith("<"):
+                    if not os.path.isabs(filename):
+                        filename = os.path.join(_IMPORT_CWD, filename)
+                if frame_summary.f_locals.get("_rich_traceback_omit", False):
+                    continue
+
+                frame = Frame(
+                    filename=filename or "?",
+                    lineno=line_no,
+                    name=frame_summary.f_code.co_name,
+                    locals=(
+                        {
+                            key: pretty.traverse(
+                                value,
+                                max_length=locals_max_length,
+                                max_string=locals_max_string,
+                                max_depth=locals_max_depth,
+                            )
+                            for key, value in get_locals(frame_summary.f_locals.items())
+                            if not (inspect.isfunction(value) or inspect.isclass(value))
+                        }
+                        if show_locals
+                        else None
+                    ),
+                    last_instruction=last_instruction,
+                )
+                append(frame)
+                if frame_summary.f_locals.get("_rich_traceback_guard", False):
+                    del stack.frames[:]
+
+            if not grouped_exceptions:
+                cause = getattr(exc_value, "__cause__", None)
+                if cause is not None and cause is not exc_value:
+                    exc_type = cause.__class__
+                    exc_value = cause
+                    # __traceback__ can be None, e.g. for exceptions raised by the
+                    # 'multiprocessing' module
+                    traceback = cause.__traceback__
+                    is_cause = True
+                    continue
+
+                cause = exc_value.__context__
+                if cause is not None and not getattr(
+                    exc_value, "__suppress_context__", False
+                ):
+                    exc_type = cause.__class__
+                    exc_value = cause
+                    traceback = cause.__traceback__
+                    is_cause = False
+                    continue
+            # No cover, code is reached but coverage doesn't recognize it.
+            break  # pragma: no cover
+
+        trace = Trace(stacks=stacks)
+
+        return trace
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        theme = self.theme
+        background_style = theme.get_background_style()
+        token_style = theme.get_style_for_token
+
+        traceback_theme = Theme(
+            {
+                "pretty": token_style(TextToken),
+                "pygments.text": token_style(Token),
+                "pygments.string": token_style(String),
+                "pygments.function": token_style(Name.Function),
+                "pygments.number": token_style(Number),
+                "repr.indent": token_style(Comment) + Style(dim=True),
+                "repr.str": token_style(String),
+                "repr.brace": token_style(TextToken) + Style(bold=True),
+                "repr.number": token_style(Number),
+                "repr.bool_true": token_style(Keyword.Constant),
+                "repr.bool_false": token_style(Keyword.Constant),
+                "repr.none": token_style(Keyword.Constant),
+                "scope.border": token_style(String.Delimiter),
+                "scope.equals": token_style(Operator),
+                "scope.key": token_style(Name),
+                "scope.key.special": token_style(Name.Constant) + Style(dim=True),
+            },
+            inherit=False,
+        )
+
+        highlighter = ReprHighlighter()
+
+        @group()
+        def render_stack(stack: Stack, last: bool) -> RenderResult:
+            if stack.frames:
+                stack_renderable: ConsoleRenderable = Panel(
+                    self._render_stack(stack),
+                    title="[traceback.title]Traceback [dim](most recent call last)",
+                    style=background_style,
+                    border_style="traceback.border",
+                    expand=True,
+                    padding=(0, 1),
+                )
+                stack_renderable = Constrain(stack_renderable, self.width)
+                with console.use_theme(traceback_theme):
+                    yield stack_renderable
+
+            if stack.syntax_error is not None:
+                with console.use_theme(traceback_theme):
+                    yield Constrain(
+                        Panel(
+                            self._render_syntax_error(stack.syntax_error),
+                            style=background_style,
+                            border_style="traceback.border.syntax_error",
+                            expand=True,
+                            padding=(0, 1),
+                            width=self.width,
+                        ),
+                        self.width,
+                    )
+                yield Text.assemble(
+                    (f"{stack.exc_type}: ", "traceback.exc_type"),
+                    highlighter(stack.syntax_error.msg),
+                )
+            elif stack.exc_value:
+                yield Text.assemble(
+                    (f"{stack.exc_type}: ", "traceback.exc_type"),
+                    highlighter(stack.exc_value),
+                )
+            else:
+                yield Text.assemble((f"{stack.exc_type}", "traceback.exc_type"))
+
+            for note in stack.notes:
+                yield Text.assemble(("[NOTE] ", "traceback.note"), highlighter(note))
+
+            if stack.is_group:
+                for group_no, group_exception in enumerate(stack.exceptions, 1):
+                    grouped_exceptions: List[Group] = []
+                    for group_last, group_stack in loop_last(group_exception.stacks):
+                        grouped_exceptions.append(render_stack(group_stack, group_last))
+                    yield ""
+                    yield Constrain(
+                        Panel(
+                            Group(*grouped_exceptions),
+                            title=f"Sub-exception #{group_no}",
+                            border_style="traceback.group.border",
+                        ),
+                        self.width,
+                    )
+
+            if not last:
+                if stack.is_cause:
+                    yield Text.from_markup(
+                        "\n[i]The above exception was the direct cause of the following exception:\n",
+                    )
+                else:
+                    yield Text.from_markup(
+                        "\n[i]During handling of the above exception, another exception occurred:\n",
+                    )
+
+        for last, stack in loop_last(reversed(self.trace.stacks)):
+            yield render_stack(stack, last)
+
+    @group()
+    def _render_syntax_error(self, syntax_error: _SyntaxError) -> RenderResult:
+        highlighter = ReprHighlighter()
+        path_highlighter = PathHighlighter()
+        if syntax_error.filename != "":
+            if os.path.exists(syntax_error.filename):
+                text = Text.assemble(
+                    (f" {syntax_error.filename}", "pygments.string"),
+                    (":", "pygments.text"),
+                    (str(syntax_error.lineno), "pygments.number"),
+                    style="pygments.text",
+                )
+                yield path_highlighter(text)
+        syntax_error_text = highlighter(syntax_error.line.rstrip())
+        syntax_error_text.no_wrap = True
+        offset = min(syntax_error.offset - 1, len(syntax_error_text))
+        syntax_error_text.stylize("bold underline", offset, offset)
+        syntax_error_text += Text.from_markup(
+            "\n" + " " * offset + "[traceback.offset]▲[/]",
+            style="pygments.text",
+        )
+        yield syntax_error_text
+
+    @classmethod
+    def _guess_lexer(cls, filename: str, code: str) -> str:
+        ext = os.path.splitext(filename)[-1]
+        if not ext:
+            # No extension, look at first line to see if it is a hashbang
+            # Note, this is an educated guess and not a guarantee
+            # If it fails, the only downside is that the code is highlighted strangely
+            new_line_index = code.index("\n")
+            first_line = code[:new_line_index] if new_line_index != -1 else code
+            if first_line.startswith("#!") and "python" in first_line.lower():
+                return "python"
+        try:
+            return cls.LEXERS.get(ext) or guess_lexer_for_filename(filename, code).name
+        except ClassNotFound:
+            return "text"
+
+    @group()
+    def _render_stack(self, stack: Stack) -> RenderResult:
+        path_highlighter = PathHighlighter()
+        theme = self.theme
+
+        def render_locals(frame: Frame) -> Iterable[ConsoleRenderable]:
+            if frame.locals:
+                yield render_scope(
+                    frame.locals,
+                    title="locals",
+                    indent_guides=self.indent_guides,
+                    max_length=self.locals_max_length,
+                    max_string=self.locals_max_string,
+                    max_depth=self.locals_max_depth,
+                    overflow=self.locals_overflow,
+                )
+
+        exclude_frames: Optional[range] = None
+        if self.max_frames != 0:
+            exclude_frames = range(
+                self.max_frames // 2,
+                len(stack.frames) - self.max_frames // 2,
+            )
+
+        excluded = False
+        for frame_index, frame in enumerate(stack.frames):
+            if exclude_frames and frame_index in exclude_frames:
+                excluded = True
+                continue
+
+            if excluded:
+                assert exclude_frames is not None
+                yield Text(
+                    f"\n... {len(exclude_frames)} frames hidden ...",
+                    justify="center",
+                    style="traceback.error",
+                )
+                excluded = False
+
+            first = frame_index == 0
+            frame_filename = frame.filename
+            suppressed = any(frame_filename.startswith(path) for path in self.suppress)
+
+            if os.path.exists(frame.filename):
+                text = Text.assemble(
+                    path_highlighter(Text(frame.filename, style="pygments.string")),
+                    (":", "pygments.text"),
+                    (str(frame.lineno), "pygments.number"),
+                    " in ",
+                    (frame.name, "pygments.function"),
+                    style="pygments.text",
+                )
+            else:
+                text = Text.assemble(
+                    "in ",
+                    (frame.name, "pygments.function"),
+                    (":", "pygments.text"),
+                    (str(frame.lineno), "pygments.number"),
+                    style="pygments.text",
+                )
+            if not frame.filename.startswith("<") and not first:
+                yield ""
+            yield text
+            if frame.filename.startswith("<"):
+                yield from render_locals(frame)
+                continue
+            if not suppressed:
+                try:
+                    code_lines = linecache.getlines(frame.filename)
+                    code = "".join(code_lines)
+                    if not code:
+                        # code may be an empty string if the file doesn't exist, OR
+                        # if the traceback filename is generated dynamically
+                        continue
+                    lexer_name = self._guess_lexer(frame.filename, code)
+                    syntax = Syntax(
+                        code,
+                        lexer_name,
+                        theme=theme,
+                        line_numbers=True,
+                        line_range=(
+                            frame.lineno - self.extra_lines,
+                            frame.lineno + self.extra_lines,
+                        ),
+                        highlight_lines={frame.lineno},
+                        word_wrap=self.word_wrap,
+                        code_width=self.code_width,
+                        indent_guides=self.indent_guides,
+                        dedent=False,
+                    )
+                    yield ""
+                except Exception as error:
+                    yield Text.assemble(
+                        (f"\n{error}", "traceback.error"),
+                    )
+                else:
+                    if frame.last_instruction is not None:
+                        start, end = frame.last_instruction
+
+                        # Stylize a line at a time
+                        # So that indentation isn't underlined (which looks bad)
+                        for line1, column1, column2 in _iter_syntax_lines(start, end):
+                            try:
+                                if column1 == 0:
+                                    line = code_lines[line1 - 1]
+                                    column1 = len(line) - len(line.lstrip())
+                                if column2 == -1:
+                                    column2 = len(code_lines[line1 - 1])
+                            except IndexError:
+                                # Being defensive here
+                                # If last_instruction reports a line out-of-bounds, we don't want to crash
+                                continue
+
+                            syntax.stylize_range(
+                                style="traceback.error_range",
+                                start=(line1, column1),
+                                end=(line1, column2),
+                            )
+                    yield (
+                        Columns(
+                            [
+                                syntax,
+                                *render_locals(frame),
+                            ],
+                            padding=1,
+                        )
+                        if frame.locals
+                        else syntax
+                    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    install(show_locals=True)
+    import sys
+
+    def bar(
+        a: Any,
+    ) -> None:  # 这是对亚洲语言支持的测试。面对模棱两可的想法，拒绝猜测的诱惑
+        one = 1
+        print(one / a)
+
+    def foo(a: Any) -> None:
+        _rich_traceback_guard = True
+        zed = {
+            "characters": {
+                "Paul Atreides",
+                "Vladimir Harkonnen",
+                "Thufir Hawat",
+                "Duncan Idaho",
+            },
+            "atomic_types": (None, False, True),
+        }
+        bar(a)
+
+    def error() -> None:
+        foo(0)
+
+    error()
diff --git a/lib/rich/tree.py b/lib/rich/tree.py
new file mode 100644
index 0000000..9a87d60
--- /dev/null
+++ b/lib/rich/tree.py
@@ -0,0 +1,257 @@
+from typing import Iterator, List, Optional, Tuple
+
+from ._loop import loop_first, loop_last
+from .console import Console, ConsoleOptions, RenderableType, RenderResult
+from .jupyter import JupyterMixin
+from .measure import Measurement
+from .segment import Segment
+from .style import Style, StyleStack, StyleType
+from .styled import Styled
+
+GuideType = Tuple[str, str, str, str]
+
+
+class Tree(JupyterMixin):
+    """A renderable for a tree structure.
+
+    Attributes:
+        ASCII_GUIDES (GuideType): Guide lines used when Console.ascii_only is True.
+        TREE_GUIDES (List[GuideType, GuideType, GuideType]): Default guide lines.
+
+    Args:
+        label (RenderableType): The renderable or str for the tree label.
+        style (StyleType, optional): Style of this tree. Defaults to "tree".
+        guide_style (StyleType, optional): Style of the guide lines. Defaults to "tree.line".
+        expanded (bool, optional): Also display children. Defaults to True.
+        highlight (bool, optional): Highlight renderable (if str). Defaults to False.
+        hide_root (bool, optional): Hide the root node. Defaults to False.
+    """
+
+    ASCII_GUIDES = ("    ", "|   ", "+-- ", "`-- ")
+    TREE_GUIDES = [
+        ("    ", "│   ", "├── ", "└── "),
+        ("    ", "┃   ", "┣━━ ", "┗━━ "),
+        ("    ", "║   ", "╠══ ", "╚══ "),
+    ]
+
+    def __init__(
+        self,
+        label: RenderableType,
+        *,
+        style: StyleType = "tree",
+        guide_style: StyleType = "tree.line",
+        expanded: bool = True,
+        highlight: bool = False,
+        hide_root: bool = False,
+    ) -> None:
+        self.label = label
+        self.style = style
+        self.guide_style = guide_style
+        self.children: List[Tree] = []
+        self.expanded = expanded
+        self.highlight = highlight
+        self.hide_root = hide_root
+
+    def add(
+        self,
+        label: RenderableType,
+        *,
+        style: Optional[StyleType] = None,
+        guide_style: Optional[StyleType] = None,
+        expanded: bool = True,
+        highlight: Optional[bool] = False,
+    ) -> "Tree":
+        """Add a child tree.
+
+        Args:
+            label (RenderableType): The renderable or str for the tree label.
+            style (StyleType, optional): Style of this tree. Defaults to "tree".
+            guide_style (StyleType, optional): Style of the guide lines. Defaults to "tree.line".
+            expanded (bool, optional): Also display children. Defaults to True.
+            highlight (Optional[bool], optional): Highlight renderable (if str). Defaults to False.
+
+        Returns:
+            Tree: A new child Tree, which may be further modified.
+        """
+        node = Tree(
+            label,
+            style=self.style if style is None else style,
+            guide_style=self.guide_style if guide_style is None else guide_style,
+            expanded=expanded,
+            highlight=self.highlight if highlight is None else highlight,
+        )
+        self.children.append(node)
+        return node
+
+    def __rich_console__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "RenderResult":
+        stack: List[Iterator[Tuple[bool, Tree]]] = []
+        pop = stack.pop
+        push = stack.append
+        new_line = Segment.line()
+
+        get_style = console.get_style
+        null_style = Style.null()
+        guide_style = get_style(self.guide_style, default="") or null_style
+        SPACE, CONTINUE, FORK, END = range(4)
+
+        _Segment = Segment
+
+        def make_guide(index: int, style: Style) -> Segment:
+            """Make a Segment for a level of the guide lines."""
+            if options.ascii_only:
+                line = self.ASCII_GUIDES[index]
+            else:
+                guide = 1 if style.bold else (2 if style.underline2 else 0)
+                line = self.TREE_GUIDES[0 if options.legacy_windows else guide][index]
+            return _Segment(line, style)
+
+        levels: List[Segment] = [make_guide(CONTINUE, guide_style)]
+        push(iter(loop_last([self])))
+
+        guide_style_stack = StyleStack(get_style(self.guide_style))
+        style_stack = StyleStack(get_style(self.style))
+        remove_guide_styles = Style(bold=False, underline2=False)
+
+        depth = 0
+
+        while stack:
+            stack_node = pop()
+            try:
+                last, node = next(stack_node)
+            except StopIteration:
+                levels.pop()
+                if levels:
+                    guide_style = levels[-1].style or null_style
+                    levels[-1] = make_guide(FORK, guide_style)
+                    guide_style_stack.pop()
+                    style_stack.pop()
+                continue
+            push(stack_node)
+            if last:
+                levels[-1] = make_guide(END, levels[-1].style or null_style)
+
+            guide_style = guide_style_stack.current + get_style(node.guide_style)
+            style = style_stack.current + get_style(node.style)
+            prefix = levels[(2 if self.hide_root else 1) :]
+            renderable_lines = console.render_lines(
+                Styled(node.label, style),
+                options.update(
+                    width=options.max_width
+                    - sum(level.cell_length for level in prefix),
+                    highlight=self.highlight,
+                    height=None,
+                ),
+                pad=options.justify is not None,
+            )
+
+            if not (depth == 0 and self.hide_root):
+                for first, line in loop_first(renderable_lines):
+                    if prefix:
+                        yield from _Segment.apply_style(
+                            prefix,
+                            style.background_style,
+                            post_style=remove_guide_styles,
+                        )
+                    yield from line
+                    yield new_line
+                    if first and prefix:
+                        prefix[-1] = make_guide(
+                            SPACE if last else CONTINUE, prefix[-1].style or null_style
+                        )
+
+            if node.expanded and node.children:
+                levels[-1] = make_guide(
+                    SPACE if last else CONTINUE, levels[-1].style or null_style
+                )
+                levels.append(
+                    make_guide(END if len(node.children) == 1 else FORK, guide_style)
+                )
+                style_stack.push(get_style(node.style))
+                guide_style_stack.push(get_style(node.guide_style))
+                push(iter(loop_last(node.children)))
+                depth += 1
+
+    def __rich_measure__(
+        self, console: "Console", options: "ConsoleOptions"
+    ) -> "Measurement":
+        stack: List[Iterator[Tree]] = [iter([self])]
+        pop = stack.pop
+        push = stack.append
+        minimum = 0
+        maximum = 0
+        measure = Measurement.get
+        level = 0
+        while stack:
+            iter_tree = pop()
+            try:
+                tree = next(iter_tree)
+            except StopIteration:
+                level -= 1
+                continue
+            push(iter_tree)
+            min_measure, max_measure = measure(console, options, tree.label)
+            indent = level * 4
+            minimum = max(min_measure + indent, minimum)
+            maximum = max(max_measure + indent, maximum)
+            if tree.expanded and tree.children:
+                push(iter(tree.children))
+                level += 1
+        return Measurement(minimum, maximum)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from rich.console import Group
+    from rich.markdown import Markdown
+    from rich.panel import Panel
+    from rich.syntax import Syntax
+    from rich.table import Table
+
+    table = Table(row_styles=["", "dim"])
+
+    table.add_column("Released", style="cyan", no_wrap=True)
+    table.add_column("Title", style="magenta")
+    table.add_column("Box Office", justify="right", style="green")
+
+    table.add_row("Dec 20, 2019", "Star Wars: The Rise of Skywalker", "$952,110,690")
+    table.add_row("May 25, 2018", "Solo: A Star Wars Story", "$393,151,347")
+    table.add_row("Dec 15, 2017", "Star Wars Ep. V111: The Last Jedi", "$1,332,539,889")
+    table.add_row("Dec 16, 2016", "Rogue One: A Star Wars Story", "$1,332,439,889")
+
+    code = """\
+class Segment(NamedTuple):
+    text: str = ""
+    style: Optional[Style] = None
+    is_control: bool = False
+"""
+    syntax = Syntax(code, "python", theme="monokai", line_numbers=True)
+
+    markdown = Markdown(
+        """\
+### example.md
+> Hello, World!
+>
+> Markdown _all_ the things
+"""
+    )
+
+    root = Tree("🌲 [b green]Rich Tree", highlight=True, hide_root=True)
+
+    node = root.add(":file_folder: Renderables", guide_style="red")
+    simple_node = node.add(":file_folder: [bold yellow]Atomic", guide_style="uu green")
+    simple_node.add(Group("📄 Syntax", syntax))
+    simple_node.add(Group("📄 Markdown", Panel(markdown, border_style="green")))
+
+    containers_node = node.add(
+        ":file_folder: [bold magenta]Containers", guide_style="bold magenta"
+    )
+    containers_node.expanded = True
+    panel = Panel.fit("Just a panel", border_style="red")
+    containers_node.add(Group("📄 Panels", panel))
+
+    containers_node.add(Group("📄 [b magenta]Table", table))
+
+    console = Console()
+
+    console.print(root)
diff --git a/lib/wcwidth-0.6.0.dist-info/INSTALLER b/lib/wcwidth-0.6.0.dist-info/INSTALLER
new file mode 100644
index 0000000..a1b589e
--- /dev/null
+++ b/lib/wcwidth-0.6.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/wcwidth-0.6.0.dist-info/METADATA b/lib/wcwidth-0.6.0.dist-info/METADATA
new file mode 100644
index 0000000..f6f0235
--- /dev/null
+++ b/lib/wcwidth-0.6.0.dist-info/METADATA
@@ -0,0 +1,776 @@
+Metadata-Version: 2.4
+Name: wcwidth
+Version: 0.6.0
+Summary: Measures the displayed width of unicode strings in a terminal
+Project-URL: Homepage, https://github.com/jquast/wcwidth
+Author-email: Jeff Quast 
+License-Expression: MIT
+License-File: LICENSE
+Keywords: cjk,combining,console,eastasian,emoji,emulator,terminal,unicode,wcswidth,wcwidth,xterm
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Internationalization
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Localization
+Classifier: Topic :: Terminals
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Description-Content-Type: text/x-rst
+
+|pypi_downloads| |codecov| |license|
+
+============
+Introduction
+============
+
+This library is mainly for CLI/TUI programs that carefully produce output for Terminals.
+
+Installation
+------------
+
+The stable version of this package is maintained on pypi, install or upgrade, using pip::
+
+    pip install --upgrade wcwidth
+
+Problem
+-------
+
+All Python string-formatting functions, `textwrap.wrap()`_, `str.ljust()`_, `str.rjust()`_, and
+`str.center()`_ **incorrectly** measure the displayed width of a string as equal to the number of
+their codepoints.
+
+Some examples of **incorrect results**:
+
+.. code-block:: python
+
+    >>> # result consumes 16 total cells, 11 expected,
+    >>> 'コンニチハ'.rjust(11, 'X')
+    'XXXXXXコンニチハ'
+
+    >>> # result consumes 5 total cells, 6 expected,
+    >>> 'café'.center(6, 'X')
+    'caféX'
+
+Solution
+--------
+
+The lowest-level functions in this library are the POSIX.1-2001 and POSIX.1-2008 `wcwidth(3)`_ and
+`wcswidth(3)`_, which this library precisely copies by interface as `wcwidth()`_ and `wcswidth()`_.
+These functions return -1 when C0 and C1 control codes are present.
+
+An easy-to-use `width()`_ function is provided as a wrapper of `wcswidth()`_ that is also capable of
+measuring most terminal control codes and sequences, like colors, bold, tabstops, and horizontal
+cursor movement.
+
+Text-justification is solved by the grapheme and sequence-aware functions `ljust()`_,
+`rjust()`_, `center()`_, and `wrap()`_, serving as drop-in replacements to python standard functions
+of the same names.
+
+The iterator functions `iter_graphemes()`_ and `iter_sequences()`_ allow for careful navigation of
+grapheme and terminal control sequence boundaries.  `iter_graphemes_reverse()`_, and
+`grapheme_boundary_before()`_ are useful for editing and searching of complex unicode.  The
+`clip()`_ function extracts substrings by display column positions, and `strip_sequences()`_ removes
+terminal escape sequences from text altogether.
+
+Discrepancies
+-------------
+
+You may find that support *varies* for complex unicode sequences or codepoints.
+
+A companion utility, `jquast/ucs-detect`_ was authored to gather and publish the results of Wide
+character, language/grapheme clustering and complex script support, emojis and zero-width joiner,
+variations, and regional indicator (flags) as a `General Tabulated Summary`_ by terminal emulator
+software and version.
+
+========
+Overview
+========
+
+wcwidth()
+---------
+
+Use function ``wcwidth()`` to determine the length of a *single unicode
+codepoint*.
+
+A brief overview, through examples, for all of the public API functions.
+
+Full API Documentation at https://wcwidth.readthedocs.io/en/latest/api.html
+
+wcwidth()
+---------
+
+Measures width of a single codepoint,
+
+.. code-block:: python
+
+    >>> # '♀' narrow emoji
+    >>> wcwidth.wcwidth('\u2640')
+    1
+
+Use function `wcwidth()`_ to determine the length of a *single unicode character*.
+
+See specification_ of character measurements. Note that ``-1`` is returned for control codes.
+
+wcswidth()
+----------
+
+Measures width of a string, returns -1 for control codes.
+
+.. code-block:: python
+
+    >>> # '♀️' emoji w/vs-16
+    >>> wcwidth.wcswidth('\u2640\ufe0f')
+    2
+
+Use function `wcswidth()`_ to determine the length of many, a *string of unicode characters*.
+
+See specification_ of character measurements. Note that ``-1`` is returned if control codes occurs
+anywhere in the string.
+
+width()
+-------
+
+Use function `width()`_ to measure a string with improved handling of ``control_codes``.
+
+.. code-block:: python
+
+    >>> # same support as wcswidth(), eg. regional indicator flag:
+    >>> wcwidth.width('\U0001F1FF\U0001F1FC')
+    2
+    >>> # but also supports SGR colored text, 'WARN', followed by SGR reset
+    >>> wcwidth.width('\x1b[38;2;255;150;100mWARN\x1b[0m')
+    4
+    >>> # tabs,
+    >>> wcwidth.width('\t', tabsize=4)
+    4
+    >>> # or, tab and all other control characters can be ignored
+    >>> wcwidth.width('\t', control_codes='ignore')
+    0
+    >>> # "vertical" control characters are ignored
+    >>> wcwidth.width('\n')
+    0
+    >>> # as well as sequences with "indeterminate" effects like Home + Clear
+    >>> wcwidth.width('\x1b[H\x1b[2J')
+    0
+    >>> # or, raise ValueError for "indeterminate" effects using control_codes='strict'
+    >>> wcwidth.width('\n', control_codes='strict')
+    Traceback (most recent call last):
+    ...
+    ValueError: Vertical movement character 0xa at position 0
+
+Use ``control_codes='ignore'`` when the input is known not to contain any control characters or
+terminal sequences for slightly improved performance. Note that TAB (``'\t'``) is a control
+character and is also ignored, you may want to use `str.expandtabs()`_, first.
+
+iter_sequences()
+----------------
+
+Iterates through text, segmented by terminal sequence,
+
+.. code-block:: python
+
+    >>> list(wcwidth.iter_sequences('hello'))
+    [('hello', False)]
+    >>> list(wcwidth.iter_sequences('\x1b[31mred\x1b[0m'))
+    [('\x1b[31m', True), ('red', False), ('\x1b[0m', True)]
+
+Use `iter_sequences()`_ to split text into segments of plain text and escape sequences. Each tuple
+contains the segment string and a boolean indicating whether it is an escape sequence (``True``) or
+text (``False``).
+
+iter_graphemes()
+----------------
+
+Use `iter_graphemes()`_ to iterate over *grapheme clusters* of a string.
+
+.. code-block:: python
+
+    >>> from wcwidth import iter_graphemes
+    >>> # ok + Regional Indicator 'Z', 'W' (Zimbabwe)
+    >>> list(wcwidth.iter_graphemes('ok\U0001F1FF\U0001F1FC'))
+    ['o', 'k', '🇿🇼']
+
+    >>> # cafe + combining acute accent
+    >>> list(wcwidth.iter_graphemes('cafe\u0301'))
+    ['c', 'a', 'f', 'é']
+
+    >>> # ok + Emoji Man + ZWJ + Woman + ZWJ + Girl
+    >>> list(wcwidth.iter_graphemes('ok\U0001F468\u200D\U0001F469\u200D\U0001F467'))
+    ['o', 'k', '👨\u200d👩\u200d👧']
+
+A grapheme cluster is what a user perceives as a single character, even if it is composed of
+multiple Unicode codepoints. This function implements `Unicode Standard Annex #29`_ grapheme cluster
+boundary rules.
+
+ljust()
+-------
+
+Use `ljust()`_ as replacement of `str.ljust()`_:
+
+.. code-block:: python
+
+    >>> 'コンニチハ'.ljust(11, '*')             # don't do this
+    'コンニチハ******'
+    >>> wcwidth.ljust('コンニチハ', 11, '*')    # do this!
+    'コンニチハ*'
+
+rjust()
+-------
+
+Use `rjust()`_ as replacement of `str.rjust()`_:
+
+.. code-block:: python
+
+    >>> 'コンニチハ'.rjust(11, '*')             # don't do this
+    '******コンニチハ'
+    >>> wcwidth.rjust('コンニチハ', 11, '*')    # do this!
+    '*コンニチハ'
+
+center()
+--------
+
+Use `center()`_ as replacement of `str.center()`_:
+
+.. code-block:: python
+
+    >>> 'cafe\u0301'.center(6, '*')             # don't do this
+    'café*'
+    >>> wcwidth.center('cafe\u0301', 6, '*')
+    '*café*'                                    # do this!
+
+wrap()
+------
+
+Use function `wrap()`_ to wrap text containing terminal sequences, Unicode grapheme
+clusters, and wide characters to a given display width.
+
+.. code-block:: python
+
+    >>> from wcwidth import wrap
+    >>> # Basic wrapping
+    >>> wrap('hello world', 5)
+    ['hello', 'world']
+
+    >>> # Wrapping CJK text (each character is 2 cells wide)
+    >>> wrap('コンニチハ', 4)
+    ['コン', 'ニチ', 'ハ']
+
+    >>> # Text with ANSI color sequences - SGR codes are propagated by default
+    >>> # Each line ends with reset, next line starts with restored style
+    >>> wrap('\x1b[1;31mhello world\x1b[0m', 5)
+    ['\x1b[1;31mhello\x1b[0m', '\x1b[1;31mworld\x1b[0m']
+
+clip()
+------
+
+Use `clip()`_ to extract a substring by column positions, preserving terminal sequences.
+
+.. code-block:: python
+
+    >>> from wcwidth import clip
+    >>> # Wide characters split to Narrow boundaries using fillchar=' '
+    >>> clip('中文字', 0, 3)
+    '中 '
+    >>> clip('中文字', 1, 5, fillchar='.')
+    '.文.'
+
+    >>> # SGR codes are propagated by default - result begins with active style
+    >>> # and ends with reset if styles are active
+    >>> clip('\x1b[1;31mHello world\x1b[0m', 6, 11)
+    '\x1b[1;31mworld\x1b[0m'
+
+    >>> # Disable SGR propagation to preserve original sequences as-is
+    >>> clip('\x1b[31m中文\x1b[0m', 0, 3, propagate_sgr=False)
+    '\x1b[31m中 \x1b[0m'
+
+strip_sequences()
+-----------------
+
+Use `strip_sequences()`_ to remove all terminal escape sequences from text.
+
+.. code-block:: python
+
+    >>> from wcwidth import strip_sequences
+    >>> strip_sequences('\x1b[31mred\x1b[0m')
+    'red'
+
+.. _ambiguous_width:
+
+ambiguous_width
+---------------
+
+Some Unicode characters have "East Asian Ambiguous" (A) width. These characters display as 1 cell by
+default, matching Western terminal contexts, but many CJK (Chinese, Japanese, Korean) environments
+may have a preference for 2 cells.  This is often found as boolean option, "Ambiguous width as wide"
+in Terminal Emulator software preferences.
+
+By default, wcwidth treats ambiguous characters as narrow (width 1). For CJK environments where your
+terminal is configured to display ambiguous characters as double-width, pass ``ambiguous_width=2``:
+
+.. code-block:: python
+
+    >>> # CIRCLED DIGIT ONE - ambiguous width
+    >>> wcwidth.width('\u2460')
+    1
+    >>> wcwidth.width('\u2460', ambiguous_width=2)
+    2
+
+The ``ambiguous_width`` parameter is available on all width-measuring functions: `wcwidth()`_,
+`wcswidth()`_, `width()`_, `ljust()`_, `rjust()`_, `center()`_, `wrap()`_, and `clip()`_.
+
+**Terminal Detection**
+
+The most reliable method to detect whether a terminal profile is set for "Ambiguous width as wide"
+mode is to display an ambiguous character surrounded by a pair of Cursor Position Report (CPR)
+queries with a terminal in cooked or raw mode, and to parse the responses for their ``(y, x)``
+locations and measure the difference ``x``.
+
+This code should also be careful check whether it is attached to a terminal and be careful of
+possible timeout, slow network, or non-response when working with "dumb terminals" like a CI build.
+
+`jquast/blessed`_ library provides such a helping `Terminal.detect_ambiguous_width()`_ method:
+
+.. code-block:: python
+
+    >>> import blessed, functools
+    >>> # Detect terminal ambiguous width as wide (2) or narrow (1)
+    >>> ambiguous_width = blessed.Terminal().detect_ambiguous_width()
+    >>> # Define a new 'width' function with this argument
+    >>> awidth = functools.partial(wcwidth.width, ambiguous_width=ambiguous_width)
+    >>> # result depends on attached terminal mode
+    >>> awidth('\u2460')
+    1
+
+==========
+Developing
+==========
+
+Install wcwidth in editable mode::
+
+   pip install -e .
+
+Execute all code generation, autoformatters, linters and unit tests using tox::
+
+   tox
+
+Or execute individual tasks, see ``tox -lv`` for all available targets::
+
+   tox -e pylint,py36,py314
+
+To run tests with detailed coverage reporting showing missing lines::
+
+   tox -epy314 -- --cov-report=term-missing
+
+Updating Unicode Version
+------------------------
+
+Regenerate python code tables from latest Unicode Specification data files::
+
+   tox -e update
+
+The script is located at ``bin/update-tables.py``, requires Python 3.9 or
+later. It is recommended but not necessary to run this script with the newest
+Python, because the newest Python has the latest ``unicodedata`` for generating
+comments.
+
+Building Documentation
+----------------------
+
+This project is using `sphinx`_ 4.5 to build documentation::
+
+   tox -e sphinx
+
+The output will be in ``docs/_build/html/``.
+
+Updating Requirements
+---------------------
+
+This project is using `pip-tools`_ to manage requirements.
+
+To upgrade requirements for updating unicode version, run::
+
+   tox -e update_requirements_update
+
+To upgrade requirements for testing, run::
+
+   tox -e update_requirements38,update_requirements39
+
+To upgrade requirements for building documentation, run::
+
+   tox -e update_requirements_docs
+
+Utilities
+---------
+
+Supplementary tools for browsing and testing terminals for wide unicode
+characters are found in the `bin/`_ of this project's source code.  Just ensure
+to first ``pip install -r requirements-develop.txt`` from this projects main
+folder. For example, an interactive browser for testing::
+
+  python ./bin/wcwidth-browser.py
+
+====
+Uses
+====
+
+This library is used in:
+
+- `jquast/blessed`_: a thin, practical wrapper around terminal capabilities in
+  Python.
+
+- `prompt-toolkit/python-prompt-toolkit`_: a Library for building powerful
+  interactive command lines in Python.
+
+- `dbcli/pgcli`_: Postgres CLI with autocompletion and syntax highlighting.
+
+- `thomasballinger/curtsies`_: a Curses-like terminal wrapper with a display
+  based on compositing 2d arrays of text.
+
+- `selectel/pyte`_: Simple VTXXX-compatible linux terminal emulator.
+
+- `astanin/python-tabulate`_: Pretty-print tabular data in Python, a library
+  and a command-line utility.
+
+- `rspeer/python-ftfy`_: Fixes mojibake and other glitches in Unicode
+  text.
+
+- `nbedos/termtosvg`_: Terminal recorder that renders sessions as SVG
+  animations.
+
+- `peterbrittain/asciimatics`_: Package to help people create full-screen text
+  UIs.
+
+- `python-cmd2/cmd2`_: A tool for building interactive command line apps
+
+- `stratis-storage/stratis-cli`_: CLI for the Stratis project
+
+- `ihabunek/toot`_: A Mastodon CLI/TUI client
+
+- `saulpw/visidata`_: Terminal spreadsheet multitool for discovering and
+  arranging data
+
+- `jquast/ucs-detect`_: Utility for unicode support detection.
+
+===============
+Other Languages
+===============
+
+There are similar implementations of the `wcwidth()`_ and `wcswidth()`_ functions in other
+languages.
+
+- `timoxley/wcwidth`_: JavaScript
+- `janlelis/unicode-display_width`_: Ruby
+- `alecrabbit/php-wcwidth`_: PHP
+- `Text::CharWidth`_: Perl
+- `bluebear94/Terminal-WCWidth`_: Perl 6
+- `mattn/go-runewidth`_: Go
+- `grepsuzette/wcwidth`_: Haxe
+- `aperezdc/lua-wcwidth`_: Lua
+- `joachimschmidt557/zig-wcwidth`_: Zig
+- `fumiyas/wcwidth-cjk`_: `LD_PRELOAD` override
+- `joshuarubin/wcwidth9`_: Unicode version 9 in C
+- `spectreconsole/wcwidth`_: C#
+
+=======
+History
+=======
+
+0.6.0 *2026-02-06*
+  * **New** Parameters ``expand_tabs``, ``replace_whitespace``, ``fix_sentence_endings``,
+    ``drop_whitespace``, ``max_lines``, and ``placeholder`` for `wrap()`_, completing stdlib
+    `textwrap.wrap()`_ compatibility.
+
+0.5.3 *2026-01-30*
+  * **Bugfix** Brahmic using Virama conjunct formation. `Issue #155`_, `PR #204`_.
+
+0.5.2 *2026-01-29*
+  * **Bugfix** Measurement of category ``Mc`` (`Spacing Combining Mark`_), approx.  443, has a more
+    nuanced specification_, and may be categorized as either zero or wide. `PR #200`_.
+  * **Bugfix** Measurement of "standalone" modifiers and regional indicators, `PR #202`_.
+  * **Updated** Data files used in some automatic tests are no longer distributed. `PR #199`_
+
+0.5.1 *2026-01-27*
+  * **Updated** generated zero and wide code tables to length of 1 to complete the previously
+    announced removal of historical wide and zero tables. `PR #196`_.
+
+0.5.0 *2026-01-26*
+  * **Drop Support** of many historical versions of wide and zero unicode tables.  Only the latest
+    Unicode version (17.0.0) is now shipped. The related ``unicode_version='auto'`` keyword of the
+    `wcwidth()`_ family of functions are ignored. `list_versions()`_ always returns a tuple of only
+    a single element of the only unicode version supported. `PR #195`_.
+  * **Performance** improvement of most common call without version or ambiguous_width specified by
+    20%. `PR #195`_.
+  * **New** Function `propagate_sgr()`_ for applying SGR state propagation to a list of lines.
+    `PR #194`_.
+  * **Improved** `wrap()`_ and `clip()`_ with ``propagate_sgr=True``. `PR #194`_.
+  * **Bugfix** `clip()`_ zero-width characters at clipping boundaries. `PR #194`_.
+  * **Bugfix** OSC Hyperlinks when broken mid-text by `wrap()`_. `PR #193`_.
+
+0.4.0 *2026-01-25*
+  * **New** Functions `iter_graphemes_reverse()`_, `grapheme_boundary_before()`_. `PR #192`_.
+  * **Bugfix** OSC Hyperlinks should not be broken by `wrap()`_. `PR #191`_.
+
+0.3.5 *2026-01-24*
+  * **Bugfix** packaging of 0.3.4 contains a failing test.
+
+0.3.4 *2026-01-24*
+  * **Bugfix** `center()`_ should match the eccentric `parity padding`_.
+    of `str.center()`_. `PR #188`_.
+
+0.3.3 *2026-01-24*
+  * **Performance** improvement in `width()`_. `PR #185`_.
+  * **Bugfix** missing ``py.typed``, ``Typing :: Typed``. `PR #184`_.
+
+0.3.2 *2026-01-23*
+  * **Updated** type hinting for full ``mympy --strict`` compliance. `PR #183`_.
+
+0.3.1 *2026-01-22*
+  * **Performance** improvement up to 30% in `width()_`. `PR #181`_.
+
+0.3.0 *2026-01-21*
+  * **Drop Support** for Python 3.6 and 3.7. `PR #156`_.
+  * **New** Function `iter_graphemes()`_. `PR #165`_.
+  * **New** Functions `width()`_ and `iter_sequences()`_. `PR #166`_.
+  * **New** Functions `ljust()`_, `rjust()`_, `center()`_. `PR #168`_.
+  * **New** Function `wrap()`_. `PR #169`_.
+  * **Performance** improvement in `wcswidth()`_. `PR #171`_.
+  * **New** argument ``ambiguous_width`` to all functions. `PR #172`_.
+  * **New** Functions `clip()`_ and `strip_sequences()`_. `PR #173`_.
+  * **Bugfix** Characters with ``Default_Ignorable_Code_Point`` property now
+    return width 0. `PR #174`_.
+  * **Bugfix** Characters with ``Prepended_Concatenation_Mark`` property now
+    return width 1. `PR #175`_.
+
+0.2.14 *2025-09-22*
+  * **Drop Support** for Python 2.7 and 3.5. `PR #117`_.
+  * **Update** tables to include Unicode Specifications 16.0.0 and 17.0.0.
+    `PR #146`_.
+  * **Bugfix** U+00AD SOFT HYPHEN should measure as 1, versions 0.2.9 through
+    0.2.13 measured as 0. `PR #149`_.
+
+0.2.13 *2024-01-06*
+  * **Bugfix** zero-width support for Hangul Jamo (Korean)
+
+0.2.12 *2023-11-21*
+  * **Bugfix** Re-release to remove `.pyi` files misplaced in wheel `Issue #101`_.
+
+0.2.11 *2023-11-20*
+  * **Updated** Include tests files in the source distribution (`PR #98`_, `PR #100`_).
+
+0.2.10 *2023-11-13*
+  * **Bugfix** accounting of some kinds of emoji sequences using U+FE0F
+    Variation Selector 16 (`PR #97`_).
+  * **Updated** specification_.
+
+0.2.9 *2023-10-30*
+  * **Bugfix** zero-width characters used in Emoji ZWJ sequences, Balinese,
+    Jamo, Devanagari, Tamil, Kannada and others (`PR #91`_).
+  * **Updated** to include specification_ of character measurements.
+
+0.2.8 *2023-09-30*
+  * Include requirements files in the source distribution (`PR #82`_).
+
+0.2.7 *2023-09-28*
+  * **Updated** tables to include Unicode Specification 15.1.0.
+  * Include ``bin``, ``docs``, and ``tox.ini`` in the source distribution
+
+0.2.6 *2023-01-14*
+  * **Updated** tables to include Unicode Specification 14.0.0 and 15.0.0.
+  * **Changed** developer tools to use pip-compile, and to use jinja2 templates
+    for code generation in `bin/update-tables.py` to prepare for possible
+    compiler optimization release.
+
+0.2.1 .. 0.2.5 *2020-06-23*
+  * **Repository** changes to update tests and packaging issues, and
+    begin tagging repository with matching release versions.
+
+0.2.0 *2020-06-01*
+  * **Enhancement**: Unicode version may be selected by exporting the
+    Environment variable ``UNICODE_VERSION``, such as ``13.0``, or ``6.3.0``.
+    See the `jquast/ucs-detect`_ CLI utility for automatic detection.
+  * **Enhancement**:
+    API Documentation is published to readthedocs.io.
+  * **Updated** tables for *all* Unicode Specifications with files
+    published in a programmatically consumable format, versions 4.1.0
+    through 13.0
+
+0.1.9 *2020-03-22*
+  * **Performance** optimization by `Avram Lubkin`_, `PR #35`_.
+  * **Updated** tables to Unicode Specification 13.0.0.
+
+0.1.8 *2020-01-01*
+  * **Updated** tables to Unicode Specification 12.0.0. (`PR #30`_).
+
+0.1.7 *2016-07-01*
+  * **Updated** tables to Unicode Specification 9.0.0. (`PR #18`_).
+
+0.1.6 *2016-01-08 Production/Stable*
+  * ``LICENSE`` file now included with distribution.
+
+0.1.5 *2015-09-13 Alpha*
+  * **Bugfix**:
+    Resolution of "combining_ character width" issue, most especially
+    those that previously returned -1 now often (correctly) return 0.
+    resolved by `Philip Craig`_ via `PR #11`_.
+  * **Deprecated**:
+    The module path ``wcwidth.table_comb`` is no longer available,
+    it has been superseded by module path ``wcwidth.table_zero``.
+
+0.1.4 *2014-11-20 Pre-Alpha*
+  * **Feature**: ``wcswidth()`` now determines printable length
+    for (most) combining_ characters.  The developer's tool
+    `bin/wcwidth-browser.py`_ is improved to display combining_
+    characters when provided the ``--combining`` option
+    (`Thomas Ballinger`_ and `Leta Montopoli`_ `PR #5`_).
+  * **Feature**: added static analysis (prospector_) to testing
+    framework.
+
+0.1.3 *2014-10-29 Pre-Alpha*
+  * **Bugfix**: 2nd parameter of wcswidth was not honored.
+    (`Thomas Ballinger`_, `PR #4`_).
+
+0.1.2 *2014-10-28 Pre-Alpha*
+  * **Updated** tables to Unicode Specification 7.0.0.
+    (`Thomas Ballinger`_, `PR #3`_).
+
+0.1.1 *2014-05-14 Pre-Alpha*
+  * Initial release to pypi, Based on Unicode Specification 6.3.0
+
+This code was originally derived directly from C code of the same name,
+whose latest version is available at
+https://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c::
+
+ * Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+ *
+ * Permission to use, copy, modify, and distribute this software
+ * for any purpose and without fee is hereby granted. The author
+ * disclaims all warranties with regard to this software.
+
+.. _`Spacing Combining Mark`: https://www.unicode.org/versions/latest/ch04.pdf#G134153
+.. _`specification`: https://wcwidth.readthedocs.io/en/latest/specs.html
+.. _`tox`: https://tox.wiki/en/latest/
+.. _`prospector`: https://github.com/landscapeio/prospector
+.. _`combining`: https://en.wikipedia.org/wiki/Combining_character
+.. _`bin/`: https://github.com/jquast/wcwidth/tree/master/bin
+.. _`bin/wcwidth-browser.py`: https://github.com/jquast/wcwidth/blob/master/bin/wcwidth-browser.py
+.. _`Thomas Ballinger`: https://github.com/thomasballinger
+.. _`Leta Montopoli`: https://github.com/lmontopo
+.. _`Philip Craig`: https://github.com/philipc
+.. _`PR #3`: https://github.com/jquast/wcwidth/pull/3
+.. _`PR #4`: https://github.com/jquast/wcwidth/pull/4
+.. _`PR #5`: https://github.com/jquast/wcwidth/pull/5
+.. _`PR #11`: https://github.com/jquast/wcwidth/pull/11
+.. _`PR #18`: https://github.com/jquast/wcwidth/pull/18
+.. _`PR #30`: https://github.com/jquast/wcwidth/pull/30
+.. _`PR #35`: https://github.com/jquast/wcwidth/pull/35
+.. _`PR #82`: https://github.com/jquast/wcwidth/pull/82
+.. _`PR #91`: https://github.com/jquast/wcwidth/pull/91
+.. _`PR #97`: https://github.com/jquast/wcwidth/pull/97
+.. _`PR #98`: https://github.com/jquast/wcwidth/pull/98
+.. _`PR #100`: https://github.com/jquast/wcwidth/pull/100
+.. _`PR #117`: https://github.com/jquast/wcwidth/pull/117
+.. _`PR #146`: https://github.com/jquast/wcwidth/pull/146
+.. _`PR #149`: https://github.com/jquast/wcwidth/pull/149
+.. _`PR #156`: https://github.com/jquast/wcwidth/pull/156
+.. _`PR #165`: https://github.com/jquast/wcwidth/pull/165
+.. _`PR #166`: https://github.com/jquast/wcwidth/pull/166
+.. _`PR #168`: https://github.com/jquast/wcwidth/pull/168
+.. _`PR #169`: https://github.com/jquast/wcwidth/pull/169
+.. _`PR #171`: https://github.com/jquast/wcwidth/pull/171
+.. _`PR #172`: https://github.com/jquast/wcwidth/pull/172
+.. _`PR #173`: https://github.com/jquast/wcwidth/pull/173
+.. _`PR #174`: https://github.com/jquast/wcwidth/pull/174
+.. _`PR #175`: https://github.com/jquast/wcwidth/pull/175
+.. _`PR #181`: https://github.com/jquast/wcwidth/pull/181
+.. _`PR #183`: https://github.com/jquast/wcwidth/pull/183
+.. _`PR #184`: https://github.com/jquast/wcwidth/pull/184
+.. _`PR #185`: https://github.com/jquast/wcwidth/pull/185
+.. _`PR #188`: https://github.com/jquast/wcwidth/pull/188
+.. _`PR #191`: https://github.com/jquast/wcwidth/pull/191
+.. _`PR #192`: https://github.com/jquast/wcwidth/pull/192
+.. _`PR #193`: https://github.com/jquast/wcwidth/pull/193
+.. _`PR #194`: https://github.com/jquast/wcwidth/pull/194
+.. _`PR #195`: https://github.com/jquast/wcwidth/pull/195
+.. _`PR #196`: https://github.com/jquast/wcwidth/pull/196
+.. _`PR #199`: https://github.com/jquast/wcwidth/pull/199
+.. _`PR #200`: https://github.com/jquast/wcwidth/pull/200
+.. _`PR #202`: https://github.com/jquast/wcwidth/pull/202
+.. _`PR #204`: https://github.com/jquast/wcwidth/pull/204
+.. _`Issue #101`: https://github.com/jquast/wcwidth/issues/101
+.. _`Issue #155`: https://github.com/jquast/wcwidth/issues/155
+.. _`Issue #190`: https://github.com/jquast/wcwidth/issues/190
+.. _`jquast/blessed`: https://github.com/jquast/blessed
+.. _`selectel/pyte`: https://github.com/selectel/pyte
+.. _`thomasballinger/curtsies`: https://github.com/thomasballinger/curtsies
+.. _`dbcli/pgcli`: https://github.com/dbcli/pgcli
+.. _`prompt-toolkit/python-prompt-toolkit`: https://github.com/prompt-toolkit/python-prompt-toolkit
+.. _`timoxley/wcwidth`: https://github.com/timoxley/wcwidth
+.. _`wcwidth(3)`:  https://man7.org/linux/man-pages/man3/wcwidth.3.html
+.. _`wcswidth(3)`: https://man7.org/linux/man-pages/man3/wcswidth.3.html
+.. _`astanin/python-tabulate`: https://github.com/astanin/python-tabulate
+.. _`janlelis/unicode-display_width`: https://github.com/janlelis/unicode-display_width
+.. _`rspeer/python-ftfy`: https://github.com/rspeer/python-ftfy
+.. _`alecrabbit/php-wcwidth`: https://github.com/alecrabbit/php-wcwidth
+.. _`Text::CharWidth`: https://metacpan.org/pod/Text::CharWidth
+.. _`bluebear94/Terminal-WCWidth`: https://github.com/bluebear94/Terminal-WCWidth
+.. _`mattn/go-runewidth`: https://github.com/mattn/go-runewidth
+.. _`grepsuzette/wcwidth`: https://github.com/grepsuzette/wcwidth
+.. _`jquast/ucs-detect`: https://github.com/jquast/ucs-detect
+.. _`Avram Lubkin`: https://github.com/avylove
+.. _`nbedos/termtosvg`: https://github.com/nbedos/termtosvg
+.. _`peterbrittain/asciimatics`: https://github.com/peterbrittain/asciimatics
+.. _`aperezdc/lua-wcwidth`: https://github.com/aperezdc/lua-wcwidth
+.. _`joachimschmidt557/zig-wcwidth`: https://github.com/joachimschmidt557/zig-wcwidth
+.. _`fumiyas/wcwidth-cjk`: https://github.com/fumiyas/wcwidth-cjk
+.. _`joshuarubin/wcwidth9`: https://github.com/joshuarubin/wcwidth9
+.. _`spectreconsole/wcwidth`: https://github.com/spectreconsole/wcwidth
+.. _`python-cmd2/cmd2`: https://github.com/python-cmd2/cmd2
+.. _`stratis-storage/stratis-cli`: https://github.com/stratis-storage/stratis-cli
+.. _`ihabunek/toot`: https://github.com/ihabunek/toot
+.. _`saulpw/visidata`: https://github.com/saulpw/visidata
+.. _`pip-tools`: https://pip-tools.readthedocs.io/
+.. _`sphinx`: https://www.sphinx-doc.org/
+.. _`textwrap.wrap()`: https://docs.python.org/3/library/textwrap.html#textwrap.wrap
+.. _`str.ljust()`: https://docs.python.org/3/library/stdtypes.html#str.ljust
+.. _`str.rjust()`: https://docs.python.org/3/library/stdtypes.html#str.rjust
+.. _`str.center()`: https://docs.python.org/3/library/stdtypes.html#str.center
+.. _`str.expandtabs()`: https://docs.python.org/3/library/stdtypes.html#str.expandtabs
+.. _`General Tabulated Summary`: https://ucs-detect.readthedocs.io/results.html#tabulated-results
+.. _`wcwidth()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.wcwidth
+.. _`wcswidth()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.wcswidth
+.. _`width()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.width
+.. _`iter_graphemes()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.iter_graphemes
+.. _`iter_graphemes_reverse()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.iter_graphemes_reverse
+.. _`grapheme_boundary_before()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.grapheme_boundary_before
+.. _`ljust()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.ljust
+.. _`rjust()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.rjust
+.. _`center()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.center
+.. _`wrap()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.wrap
+.. _`clip()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.clip
+.. _`strip_sequences()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.strip_sequences
+.. _`propagate_sgr()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.propagate_sgr
+.. _`iter_sequences()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.iter_sequences
+.. _`list_versions()`: https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.list_versions
+.. _`Unicode Standard Annex #29`: https://www.unicode.org/reports/tr29/
+.. _`Terminal.detect_ambiguous_width()`: https://blessed.readthedocs.io/en/latest/api/terminal.html#blessed.terminal.Terminal.detect_ambiguous_width
+.. _`parity padding`: https://jazcap53.github.io/pythons-eccentric-strcenter.html
+.. |pypi_downloads| image:: https://img.shields.io/pypi/dm/wcwidth.svg?logo=pypi
+    :alt: Downloads
+    :target: https://pypi.org/project/wcwidth/
+.. |codecov| image:: https://codecov.io/gh/jquast/wcwidth/branch/master/graph/badge.svg
+    :alt: codecov.io Code Coverage
+    :target: https://app.codecov.io/gh/jquast/wcwidth/
+.. |license| image:: https://img.shields.io/pypi/l/wcwidth.svg
+    :target: https://pypi.org/project/wcwidth/
+    :alt: MIT License
diff --git a/lib/wcwidth-0.6.0.dist-info/RECORD b/lib/wcwidth-0.6.0.dist-info/RECORD
new file mode 100644
index 0000000..6bedd3e
--- /dev/null
+++ b/lib/wcwidth-0.6.0.dist-info/RECORD
@@ -0,0 +1,36 @@
+wcwidth-0.6.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+wcwidth-0.6.0.dist-info/METADATA,sha256=3_N4a4-6lVqDxdMDTOApuKrCDjOJ2ttiNNJUhQDVHnI,30525
+wcwidth-0.6.0.dist-info/RECORD,,
+wcwidth-0.6.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+wcwidth-0.6.0.dist-info/licenses/LICENSE,sha256=cLmKlaIUTrcK-AF_qMbZXOJH5AhnQ26LxknhN_4T0ho,1322
+wcwidth/__init__.py,sha256=UXJ2nUqbNHUpZvcW_pMfl7qQJr52Kww52u8LKFbJqF4,1793
+wcwidth/__pycache__/__init__.cpython-314.pyc,,
+wcwidth/__pycache__/bisearch.cpython-314.pyc,,
+wcwidth/__pycache__/control_codes.cpython-314.pyc,,
+wcwidth/__pycache__/escape_sequences.cpython-314.pyc,,
+wcwidth/__pycache__/grapheme.cpython-314.pyc,,
+wcwidth/__pycache__/sgr_state.cpython-314.pyc,,
+wcwidth/__pycache__/table_ambiguous.cpython-314.pyc,,
+wcwidth/__pycache__/table_grapheme.cpython-314.pyc,,
+wcwidth/__pycache__/table_mc.cpython-314.pyc,,
+wcwidth/__pycache__/table_vs16.cpython-314.pyc,,
+wcwidth/__pycache__/table_wide.cpython-314.pyc,,
+wcwidth/__pycache__/table_zero.cpython-314.pyc,,
+wcwidth/__pycache__/textwrap.cpython-314.pyc,,
+wcwidth/__pycache__/unicode_versions.cpython-314.pyc,,
+wcwidth/__pycache__/wcwidth.cpython-314.pyc,,
+wcwidth/bisearch.py,sha256=oF4Wn7JPkkUQlFXMGMcBPDL8VCkAirjXUsGj95BTh7o,812
+wcwidth/control_codes.py,sha256=NGkPsXNA_MSNH_G1xQjvDgX4mSj-kp1TLfLP-GIUz08,1579
+wcwidth/escape_sequences.py,sha256=OmHUFj3f7UcywR0iQMmaD6x9nQXhQhY4bBDkjXdrjJQ,3097
+wcwidth/grapheme.py,sha256=O-0Fn1vZcQ77vrKBusRTCdWWnybmL2RH1QHXqfsb5O4,13382
+wcwidth/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wcwidth/sgr_state.py,sha256=4hUWKl2znPJ13uA1OipcmmQjOO7LBR-W-6y4hPBROAs,12429
+wcwidth/table_ambiguous.py,sha256=6zyqWUQqAswfpF3JesLjrHNeQhI30pfP66GfHm-jmD8,11900
+wcwidth/table_grapheme.py,sha256=yTwC8Yr1_H0GNIhTQMaO4jl-4cc4qHPIyIkoNxh3aGk,142899
+wcwidth/table_mc.py,sha256=KCR35VpvTQ_lv6XtuhukBhdnH18-TurFUrbABDwLzYA,13993
+wcwidth/table_vs16.py,sha256=TWc_Hgen926MO-EW_nFdyAsOrXVmP09N0msHdTXibX8,6890
+wcwidth/table_wide.py,sha256=9JNk7kRfG5awpY7wLIztZddHHA9JGc6TAuxHdBZ-Zrk,8974
+wcwidth/table_zero.py,sha256=096xl4g7GJUVFHfUO0gzJxNzTtBRMgin6-Jf--2fCzM,25642
+wcwidth/textwrap.py,sha256=n1_Bgs3f7t1SMnM1GE2w40jlN01fiDJZlLQvpMZK21A,28997
+wcwidth/unicode_versions.py,sha256=U5NM8ficYOVH_I1nL6Ap0reWhozB5rLIMZ7y8XU8uFU,541
+wcwidth/wcwidth.py,sha256=o5cH_RmQQwLBexguI62HPHrT6yYtxOk5P3--jXDlelI,41127
diff --git a/lib/wcwidth-0.6.0.dist-info/WHEEL b/lib/wcwidth-0.6.0.dist-info/WHEEL
new file mode 100644
index 0000000..ae8ec1b
--- /dev/null
+++ b/lib/wcwidth-0.6.0.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any
diff --git a/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE b/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE
new file mode 100644
index 0000000..a44c075
--- /dev/null
+++ b/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE
@@ -0,0 +1,27 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Jeff Quast 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+
+Permission to use, copy, modify, and distribute this software
+for any purpose and without fee is hereby granted. The author
+disclaims all warranties with regard to this software.
diff --git a/lib/wcwidth/__init__.py b/lib/wcwidth/__init__.py
new file mode 100644
index 0000000..400c8a6
--- /dev/null
+++ b/lib/wcwidth/__init__.py
@@ -0,0 +1,43 @@
+"""
+Wcwidth module.
+
+https://github.com/jquast/wcwidth
+"""
+# re-export all functions & definitions, even private ones, from top-level
+# module path, to allow for 'from wcwidth import _private_func'.  Of course,
+# user beware that any _private functions or variables not exported by __all__
+# may disappear or change signature at any future version.
+
+# local
+from .wcwidth import ZERO_WIDTH  # noqa
+from .wcwidth import (WIDE_EASTASIAN,
+                      AMBIGUOUS_EASTASIAN,
+                      VS16_NARROW_TO_WIDE,
+                      clip,
+                      ljust,
+                      rjust,
+                      width,
+                      center,
+                      wcwidth,
+                      wcswidth,
+                      list_versions,
+                      iter_sequences,
+                      strip_sequences,
+                      _wcmatch_version,
+                      _wcversion_value)
+from .bisearch import bisearch as _bisearch
+from .grapheme import grapheme_boundary_before  # noqa
+from .grapheme import iter_graphemes, iter_graphemes_reverse
+from .textwrap import SequenceTextWrapper, wrap
+from .sgr_state import propagate_sgr
+
+# The __all__ attribute defines the items exported from statement,
+# 'from wcwidth import *', but also to say, "This is the public API".
+__all__ = ('wcwidth', 'wcswidth', 'width', 'iter_sequences', 'iter_graphemes',
+           'iter_graphemes_reverse', 'grapheme_boundary_before',
+           'ljust', 'rjust', 'center', 'wrap', 'clip', 'strip_sequences',
+           'list_versions', 'propagate_sgr')
+
+# Using 'hatchling', it does not seem to provide the pyproject.toml nicety, "dynamic = ['version']"
+# like flit_core, maybe there is some better way but for now we have to duplicate it in both places
+__version__ = '0.6.0'
diff --git a/lib/wcwidth/__pycache__/__init__.cpython-314.pyc b/lib/wcwidth/__pycache__/__init__.cpython-314.pyc
new file mode 100644
index 0000000..291c50a
Binary files /dev/null and b/lib/wcwidth/__pycache__/__init__.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc b/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc
new file mode 100644
index 0000000..5d33c65
Binary files /dev/null and b/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc b/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc
new file mode 100644
index 0000000..36e8f41
Binary files /dev/null and b/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc b/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc
new file mode 100644
index 0000000..3b4397e
Binary files /dev/null and b/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc b/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc
new file mode 100644
index 0000000..2478f0a
Binary files /dev/null and b/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc b/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc
new file mode 100644
index 0000000..a20ecfb
Binary files /dev/null and b/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc b/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc
new file mode 100644
index 0000000..4dcbcc0
Binary files /dev/null and b/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc b/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc
new file mode 100644
index 0000000..d4d015d
Binary files /dev/null and b/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc b/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc
new file mode 100644
index 0000000..b72c676
Binary files /dev/null and b/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc b/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc
new file mode 100644
index 0000000..3687ee1
Binary files /dev/null and b/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc b/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc
new file mode 100644
index 0000000..c381163
Binary files /dev/null and b/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc b/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc
new file mode 100644
index 0000000..948aa03
Binary files /dev/null and b/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc b/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc
new file mode 100644
index 0000000..0120abc
Binary files /dev/null and b/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc b/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc
new file mode 100644
index 0000000..945b501
Binary files /dev/null and b/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc differ
diff --git a/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc b/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc
new file mode 100644
index 0000000..c92536d
Binary files /dev/null and b/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc differ
diff --git a/lib/wcwidth/bisearch.py b/lib/wcwidth/bisearch.py
new file mode 100644
index 0000000..becfe86
--- /dev/null
+++ b/lib/wcwidth/bisearch.py
@@ -0,0 +1,29 @@
+"""Binary search function for Unicode interval tables."""
+from __future__ import annotations
+
+
+def bisearch(ucs: int, table: tuple[tuple[int, int], ...]) -> int:
+    """
+    Binary search in interval table.
+
+    :param ucs: Ordinal value of unicode character.
+    :param table: Tuple of starting and ending ranges of ordinal values,
+        in form of ``((start, end), ...)``.
+    :returns: 1 if ordinal value ucs is found within lookup table, else 0.
+    """
+    lbound = 0
+    ubound = len(table) - 1
+
+    if ucs < table[0][0] or ucs > table[ubound][1]:
+        return 0
+
+    while ubound >= lbound:
+        mid = (lbound + ubound) // 2
+        if ucs > table[mid][1]:
+            lbound = mid + 1
+        elif ucs < table[mid][0]:
+            ubound = mid - 1
+        else:
+            return 1
+
+    return 0
diff --git a/lib/wcwidth/control_codes.py b/lib/wcwidth/control_codes.py
new file mode 100644
index 0000000..3a6fff7
--- /dev/null
+++ b/lib/wcwidth/control_codes.py
@@ -0,0 +1,46 @@
+"""
+Control character sets for terminal handling.
+
+This module provides the control character sets used by the width() function to handle terminal
+control characters.
+"""
+
+# Illegal C0/C1 control characters.
+# These raise ValueError in 'strict' mode.
+ILLEGAL_CTRL = frozenset(
+    chr(c) for c in (
+        list(range(0x01, 0x07)) +    # SOH, STX, ETX (^C), EOT (^D), ENQ, ACK
+        list(range(0x10, 0x1b)) +    # DLE through SUB (^Z)
+        list(range(0x1c, 0x20)) +    # FS, GS, RS, US
+        [0x7f] +                      # DEL
+        list(range(0x80, 0xa0))       # C1 control characters
+    )
+)
+
+# Vertical movement control characters.
+# These raise ValueError in 'strict' mode (indeterminate horizontal position).
+VERTICAL_CTRL = frozenset({
+    '\x0a',  # LF (line feed)
+    '\x0b',  # VT (vertical tab)
+    '\x0c',  # FF (form feed)
+})
+
+# Horizontal movement control characters.
+# These affect cursor position and are tracked in 'strict' and 'parse' modes.
+HORIZONTAL_CTRL = frozenset({
+    '\x08',  # BS (backspace) - cursor left 1
+    '\x09',  # HT (horizontal tab) - advance to next tab stop
+    '\x0d',  # CR (carriage return) - cursor to column 0
+})
+
+# Terminal-valid zero-width control characters.
+# These are allowed in all modes (zero-width, no movement).
+ZERO_WIDTH_CTRL = frozenset({
+    '\x00',  # NUL
+    '\x07',  # BEL (bell)
+    '\x0e',  # SO (shift out)
+    '\x0f',  # SI (shift in)
+})
+
+# All control characters that need special handling (not regular printable).
+ALL_CTRL = ILLEGAL_CTRL | VERTICAL_CTRL | HORIZONTAL_CTRL | ZERO_WIDTH_CTRL | {'\x1b'}
diff --git a/lib/wcwidth/escape_sequences.py b/lib/wcwidth/escape_sequences.py
new file mode 100644
index 0000000..d4ac6cc
--- /dev/null
+++ b/lib/wcwidth/escape_sequences.py
@@ -0,0 +1,69 @@
+r"""
+Terminal escape sequence patterns.
+
+This module provides regex patterns for matching terminal escape sequences. All patterns match
+sequences that begin with ESC (``\x1b``). Before calling re.match with these patterns, callers
+should first check that the character at the current position is ESC for optimal performance.
+"""
+# std imports
+import re
+
+# Zero-width escape sequences (SGR, OSC, CSI, etc.). This table, like INDETERMINATE_EFFECT_SEQUENCE,
+# originated from the 'blessed' library.
+ZERO_WIDTH_PATTERN = re.compile(
+    # CSI sequences
+    r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]|'
+    # OSC sequences
+    r'\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|'
+    # APC sequences
+    r'\x1b_[^\x1b\x07]*(?:\x07|\x1b\\)|'
+    # DCS sequences
+    r'\x1bP[^\x1b\x07]*(?:\x07|\x1b\\)|'
+    # PM sequences
+    r'\x1b\^[^\x1b\x07]*(?:\x07|\x1b\\)|'
+    # Character set designation
+    r'\x1b[()].|'
+    # Fe sequences
+    r'\x1b[\x40-\x5f]|'
+    # Fp sequences
+    r'\x1b[78=>g]'
+)
+
+# Cursor right movement: CSI [n] C, parameter may be parsed by width()
+CURSOR_RIGHT_SEQUENCE = re.compile(r'\x1b\[(\d*)C')
+
+# Cursor left movement: CSI [n] D, parameter may be parsed by width()
+CURSOR_LEFT_SEQUENCE = re.compile(r'\x1b\[(\d*)D')
+
+# Indeterminate effect sequences - raise ValueError in 'strict' mode. The effects of these sequences
+# are likely to be undesirable, moving the cursor vertically or to any unknown position, and
+# otherwise not managed by the 'width' method of this library.
+#
+# This table was created initially with code generation by extraction of termcap library with
+# techniques used at 'blessed' library runtime for 'xterm', 'alacritty', 'kitty', ghostty',
+# 'screen', 'tmux', and others. Then, these common capabilities were merged into the list below.
+INDETERMINATE_EFFECT_SEQUENCE = re.compile(
+    '|'.join(f'(?:{_pattern})' for _pattern in (
+        r'\x1b\[\d+;\d+r',           # change_scroll_region
+        r'\x1b\[\d*K',               # erase_in_line (clr_eol, clr_bol)
+        r'\x1b\[\d*J',               # erase_in_display (clr_eos, erase_display)
+        r'\x1b\[\d*G',               # column_address
+        r'\x1b\[\d+;\d+H',           # cursor_address
+        r'\x1b\[\d*H',               # cursor_home
+        r'\x1b\[\d*A',               # cursor_up
+        r'\x1b\[\d*B',               # cursor_down
+        r'\x1b\[\d*P',               # delete_character
+        r'\x1b\[\d*M',               # delete_line
+        r'\x1b\[\d*L',               # insert_line
+        r'\x1b\[\d*@',               # insert_character
+        r'\x1b\[\d+X',               # erase_chars
+        r'\x1b\[\d*S',               # scroll_up (parm_index)
+        r'\x1b\[\d*T',               # scroll_down (parm_rindex)
+        r'\x1b\[\d*d',               # row_address
+        r'\x1b\[\?1049[hl]',         # alternate screen buffer
+        r'\x1b\[\?47[hl]',           # alternate screen (legacy)
+        r'\x1b8',                    # restore_cursor
+        r'\x1bD',                    # scroll_forward (index)
+        r'\x1bM',                    # scroll_reverse (reverse index)
+    ))
+)
diff --git a/lib/wcwidth/grapheme.py b/lib/wcwidth/grapheme.py
new file mode 100644
index 0000000..7befc92
--- /dev/null
+++ b/lib/wcwidth/grapheme.py
@@ -0,0 +1,428 @@
+"""
+Grapheme cluster segmentation following Unicode Standard Annex #29.
+
+This module provides pure-Python implementation of the grapheme cluster boundary algorithm as
+defined in UAX #29: Unicode Text Segmentation.
+
+https://www.unicode.org/reports/tr29/
+"""
+
+from __future__ import annotations
+
+# std imports
+from enum import IntEnum
+from functools import lru_cache
+
+from typing import TYPE_CHECKING, NamedTuple
+
+# local
+from .bisearch import bisearch as _bisearch
+from .table_grapheme import (GRAPHEME_L,
+                             GRAPHEME_T,
+                             GRAPHEME_V,
+                             GRAPHEME_LV,
+                             INCB_EXTEND,
+                             INCB_LINKER,
+                             GRAPHEME_LVT,
+                             INCB_CONSONANT,
+                             GRAPHEME_EXTEND,
+                             GRAPHEME_CONTROL,
+                             GRAPHEME_PREPEND,
+                             GRAPHEME_SPACINGMARK,
+                             EXTENDED_PICTOGRAPHIC,
+                             GRAPHEME_REGIONAL_INDICATOR)
+
+if TYPE_CHECKING:  # pragma: no cover
+    # std imports
+    from collections.abc import Iterator
+
+# Maximum backward scan distance when finding grapheme cluster boundaries.
+# Covers all known Unicode grapheme clusters with margin; longer sequences are pathological.
+MAX_GRAPHEME_SCAN = 32
+
+
+class GCB(IntEnum):
+    """Grapheme Cluster Break property values."""
+
+    OTHER = 0
+    CR = 1
+    LF = 2
+    CONTROL = 3
+    EXTEND = 4
+    ZWJ = 5
+    REGIONAL_INDICATOR = 6
+    PREPEND = 7
+    SPACING_MARK = 8
+    L = 9
+    V = 10
+    T = 11
+    LV = 12
+    LVT = 13
+
+
+# All lru_cache sizes in this file use maxsize=1024, chosen by benchmarking UDHR data (500+
+# languages) and considering typical process-long sessions: western scripts need ~64 unique
+# codepoints, but CJK could reach ~2000 -- but likely not.
+@lru_cache(maxsize=1024)
+def _grapheme_cluster_break(ucs: int) -> GCB:
+    # pylint: disable=too-many-branches,too-complex
+    """Return the Grapheme_Cluster_Break property for a codepoint."""
+    # Single codepoint matches
+    if ucs == 0x000d:
+        return GCB.CR
+    if ucs == 0x000a:
+        return GCB.LF
+    if ucs == 0x200d:
+        return GCB.ZWJ
+    # Matching by codepoint ranges, requiring binary search
+    if _bisearch(ucs, GRAPHEME_CONTROL):
+        return GCB.CONTROL
+    if _bisearch(ucs, GRAPHEME_EXTEND):
+        return GCB.EXTEND
+    if _bisearch(ucs, GRAPHEME_REGIONAL_INDICATOR):
+        return GCB.REGIONAL_INDICATOR
+    if _bisearch(ucs, GRAPHEME_PREPEND):
+        return GCB.PREPEND
+    if _bisearch(ucs, GRAPHEME_SPACINGMARK):
+        return GCB.SPACING_MARK
+    if _bisearch(ucs, GRAPHEME_L):
+        return GCB.L
+    if _bisearch(ucs, GRAPHEME_V):
+        return GCB.V
+    if _bisearch(ucs, GRAPHEME_T):
+        return GCB.T
+    if _bisearch(ucs, GRAPHEME_LV):
+        return GCB.LV
+    if _bisearch(ucs, GRAPHEME_LVT):
+        return GCB.LVT
+    return GCB.OTHER
+
+
+@lru_cache(maxsize=1024)
+def _is_extended_pictographic(ucs: int) -> bool:
+    """Check if codepoint has Extended_Pictographic property."""
+    return bool(_bisearch(ucs, EXTENDED_PICTOGRAPHIC))
+
+
+@lru_cache(maxsize=1024)
+def _is_incb_linker(ucs: int) -> bool:
+    """Check if codepoint has InCB=Linker property."""
+    return bool(_bisearch(ucs, INCB_LINKER))
+
+
+@lru_cache(maxsize=1024)
+def _is_incb_consonant(ucs: int) -> bool:
+    """Check if codepoint has InCB=Consonant property."""
+    return bool(_bisearch(ucs, INCB_CONSONANT))
+
+
+@lru_cache(maxsize=1024)
+def _is_incb_extend(ucs: int) -> bool:
+    """Check if codepoint has InCB=Extend property."""
+    return bool(_bisearch(ucs, INCB_EXTEND))
+
+
+class BreakResult(NamedTuple):
+    """Result of grapheme cluster break decision."""
+
+    should_break: bool
+    ri_count: int
+
+
+@lru_cache(maxsize=1024)
+def _simple_break_check(prev_gcb: GCB, curr_gcb: GCB) -> BreakResult | None:
+    """
+    Check simple GCB-pair-based break rules (cacheable).
+
+    Returns BreakResult for rules that can be determined from GCB properties alone, or None if
+    complex lookback rules (GB9c, GB11) need to be checked.
+    """
+    # GB3: CR x LF
+    if prev_gcb == GCB.CR and curr_gcb == GCB.LF:
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB4: (Control|CR|LF) ÷
+    if prev_gcb in (GCB.CONTROL, GCB.CR, GCB.LF):
+        return BreakResult(should_break=True, ri_count=0)
+
+    # GB5: ÷ (Control|CR|LF)
+    if curr_gcb in (GCB.CONTROL, GCB.CR, GCB.LF):
+        return BreakResult(should_break=True, ri_count=0)
+
+    # GB6: L x (L|V|LV|LVT)
+    if prev_gcb == GCB.L and curr_gcb in (GCB.L, GCB.V, GCB.LV, GCB.LVT):
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB7: (LV|V) x (V|T)
+    if prev_gcb in (GCB.LV, GCB.V) and curr_gcb in (GCB.V, GCB.T):
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB8: (LVT|T) x T
+    if prev_gcb in (GCB.LVT, GCB.T) and curr_gcb == GCB.T:
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB9: x (Extend|ZWJ) - but ZWJ needs GB11 check, so only handle Extend here
+    if curr_gcb == GCB.EXTEND:
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB9a: x SpacingMark
+    if curr_gcb == GCB.SPACING_MARK:
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB9b: Prepend x
+    if prev_gcb == GCB.PREPEND:
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB9c and GB11 need lookback - return None to signal complex check needed
+    # GB12/13 (RI pairs) need ri_count state - also handled in main function
+    return None
+
+
+def _should_break(
+    prev_gcb: GCB,
+    curr_gcb: GCB,
+    text: str,
+    curr_idx: int,
+    ri_count: int,
+) -> BreakResult:
+    # pylint: disable=too-many-branches,too-complex
+    """
+    Determine if there should be a grapheme cluster break between prev and curr.
+
+    Implements UAX #29 grapheme cluster boundary rules.
+    """
+    # Try cached simple rules first
+    result = _simple_break_check(prev_gcb, curr_gcb)
+    if result is not None:
+        return result
+
+    # GB9: x ZWJ (not cached because GB11 needs lookback when prev is ZWJ)
+    if curr_gcb == GCB.ZWJ:
+        return BreakResult(should_break=False, ri_count=0)
+
+    # GB9c: Indic conjunct cluster
+    # \p{InCB=Consonant} [\p{InCB=Extend}\p{InCB=Linker}]* \p{InCB=Linker}
+    #     [\p{InCB=Extend}\p{InCB=Linker}]* x \p{InCB=Consonant}
+    curr_ucs = ord(text[curr_idx])
+    if _is_incb_consonant(curr_ucs):
+        has_linker = False
+        i = curr_idx - 1
+        while i >= 0:
+            prev_ucs = ord(text[i])
+            if _is_incb_linker(prev_ucs):
+                has_linker = True
+                i -= 1
+            elif _is_incb_extend(prev_ucs):
+                i -= 1
+            elif _is_incb_consonant(prev_ucs):
+                if has_linker:
+                    return BreakResult(should_break=False, ri_count=0)
+                break
+            else:
+                break
+
+    # GB11: ExtPict Extend* ZWJ x ExtPict
+    if prev_gcb == GCB.ZWJ and _is_extended_pictographic(curr_ucs):
+        i = curr_idx - 2  # Skip the ZWJ at curr_idx - 1
+        while i >= 0:
+            prev_ucs = ord(text[i])
+            prev_prop = _grapheme_cluster_break(prev_ucs)
+            if prev_prop == GCB.EXTEND:
+                i -= 1
+            elif _is_extended_pictographic(prev_ucs):
+                return BreakResult(should_break=False, ri_count=0)
+            else:
+                break
+
+    # GB12/GB13: RI x RI (pair matching)
+    if prev_gcb == GCB.REGIONAL_INDICATOR and curr_gcb == GCB.REGIONAL_INDICATOR:
+        if ri_count % 2 == 1:
+            return BreakResult(should_break=False, ri_count=ri_count + 1)
+        return BreakResult(should_break=True, ri_count=1)
+
+    # GB999: Any ÷ Any
+    ri_count = 1 if curr_gcb == GCB.REGIONAL_INDICATOR else 0
+    return BreakResult(should_break=True, ri_count=ri_count)
+
+
+def iter_graphemes(
+    unistr: str,
+    start: int = 0,
+    end: int | None = None,
+) -> Iterator[str]:
+    r"""
+    Iterate over grapheme clusters in a Unicode string.
+
+    Grapheme clusters are "user-perceived characters" - what a user would
+    consider a single character, which may consist of multiple Unicode
+    codepoints (e.g., a base character with combining marks, emoji sequences).
+
+    :param unistr: The Unicode string to segment.
+    :param start: Starting index (default 0).
+    :param end: Ending index (default len(unistr)).
+    :yields: Grapheme cluster substrings.
+
+    Example::
+
+        >>> list(iter_graphemes('cafe\u0301'))
+        ['c', 'a', 'f', 'e\u0301']
+        >>> list(iter_graphemes('\U0001F468\u200D\U0001F469\u200D\U0001F467'))
+        ['o', 'k', '\U0001F468\u200D\U0001F469\u200D\U0001F467']
+        >>> list(iter_graphemes('\U0001F1FA\U0001F1F8'))
+        ['o', 'k', '\U0001F1FA\U0001F1F8']
+
+    .. versionadded:: 0.3.0
+    """
+    if not unistr:
+        return
+
+    length = len(unistr)
+
+    if end is None:
+        end = length
+
+    if start >= end or start >= length:
+        return
+
+    end = min(end, length)
+
+    # Track state for grapheme cluster boundaries
+    cluster_start = start
+    ri_count = 0
+
+    # Get GCB for first character
+    prev_gcb = _grapheme_cluster_break(ord(unistr[start]))
+
+    # Handle Regional Indicator count initialization
+    if prev_gcb == GCB.REGIONAL_INDICATOR:
+        ri_count = 1
+
+    for idx in range(start + 1, end):
+        curr_gcb = _grapheme_cluster_break(ord(unistr[idx]))
+
+        result = _should_break(prev_gcb, curr_gcb, unistr, idx, ri_count)
+        ri_count = result.ri_count
+
+        if result.should_break:
+            yield unistr[cluster_start:idx]
+            cluster_start = idx
+
+        prev_gcb = curr_gcb
+
+    # Yield the final cluster
+    yield unistr[cluster_start:end]
+
+
+def _find_cluster_start(text: str, pos: int) -> int:
+    """
+    Find the start of the grapheme cluster containing the character before pos.
+
+    Scans backwards from pos to find a safe starting point, then iterates forward using standard
+    break rules to find the actual cluster boundary.
+
+    :param text: The Unicode string.
+    :param pos: Position to search before (exclusive).
+    :returns: Start position of the grapheme cluster.
+    """
+    target_cp = ord(text[pos - 1])
+
+    # GB3: CR x LF - LF after CR is part of same cluster
+    if target_cp == 0x0A and pos >= 2 and text[pos - 2] == '\r':
+        return pos - 2
+
+    # Fast path: ASCII (except LF) starts its own cluster
+    if target_cp < 0x80:
+        # GB9b: Check for preceding PREPEND (rare: Arabic/Brahmic)
+        if pos >= 2 and target_cp >= 0x20:
+            prev_cp = ord(text[pos - 2])
+            if prev_cp >= 0x80 and _grapheme_cluster_break(prev_cp) == GCB.PREPEND:
+                return _find_cluster_start(text, pos - 1)
+        return pos - 1
+
+    # Scan backward to find a safe starting point
+    safe_start = pos - 1
+    while safe_start > 0 and (pos - safe_start) < MAX_GRAPHEME_SCAN:
+        cp = ord(text[safe_start])
+        if 0x20 <= cp < 0x80:  # ASCII always starts a cluster
+            break
+        if _grapheme_cluster_break(cp) == GCB.CONTROL:  # GB4
+            break
+        safe_start -= 1
+
+    # Verify forward to find the actual cluster boundary
+    cluster_start = safe_start
+    left_gcb = _grapheme_cluster_break(ord(text[safe_start]))
+    ri_count = 1 if left_gcb == GCB.REGIONAL_INDICATOR else 0
+
+    for i in range(safe_start + 1, pos):
+        right_gcb = _grapheme_cluster_break(ord(text[i]))
+        result = _should_break(left_gcb, right_gcb, text, i, ri_count)
+        ri_count = result.ri_count
+        if result.should_break:
+            cluster_start = i
+        left_gcb = right_gcb
+
+    return cluster_start
+
+
+def grapheme_boundary_before(unistr: str, pos: int) -> int:
+    r"""
+    Find the grapheme cluster boundary immediately before a position.
+
+    :param unistr: The Unicode string to search.
+    :param pos: Position in the string (0 < pos <= len(unistr)).
+    :returns: Start index of the grapheme cluster containing the character at pos-1.
+
+    Example::
+
+        >>> grapheme_boundary_before('Hello \U0001F44B\U0001F3FB', 8)
+        6
+        >>> grapheme_boundary_before('a\r\nb', 3)
+        1
+
+    .. versionadded:: 0.3.6
+    """
+    if pos <= 0:
+        return 0
+    return _find_cluster_start(unistr, min(pos, len(unistr)))
+
+
+def iter_graphemes_reverse(
+    unistr: str,
+    start: int = 0,
+    end: int | None = None,
+) -> Iterator[str]:
+    r"""
+    Iterate over grapheme clusters in reverse order (last to first).
+
+    :param unistr: The Unicode string to segment.
+    :param start: Starting index (default 0).
+    :param end: Ending index (default len(unistr)).
+    :yields: Grapheme cluster substrings in reverse order.
+
+    Example::
+
+        >>> list(iter_graphemes_reverse('cafe\u0301'))
+        ['e\u0301', 'f', 'a', 'c']
+
+    .. versionadded:: 0.3.6
+    """
+    if not unistr:
+        return
+
+    length = len(unistr)
+
+    end = length if end is None else min(end, length)
+    start = max(start, 0)
+
+    if start >= end or start >= length:
+        return
+
+    pos = end
+    while pos > start:
+        cluster_start = _find_cluster_start(unistr, pos)
+        # Don't yield partial graphemes that extend before start
+        if cluster_start < start:
+            break
+        yield unistr[cluster_start:pos]
+        pos = cluster_start
diff --git a/lib/wcwidth/py.typed b/lib/wcwidth/py.typed
new file mode 100644
index 0000000..e69de29
diff --git a/lib/wcwidth/sgr_state.py b/lib/wcwidth/sgr_state.py
new file mode 100644
index 0000000..b0c8648
--- /dev/null
+++ b/lib/wcwidth/sgr_state.py
@@ -0,0 +1,338 @@
+"""
+SGR (Select Graphic Rendition) state tracking for terminal escape sequences.
+
+This module provides functions for tracking and propagating terminal styling (bold, italic, colors,
+etc.) via public API propagate_sgr(), and its dependent functions, cut() and wrap(). It only has
+attributes necessary to perform its functions, eg 'RED' and 'BLUE' attributes are not defined.
+"""
+from __future__ import annotations
+
+# std imports
+import re
+from enum import IntEnum
+
+from typing import TYPE_CHECKING, Iterator, NamedTuple
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Sequence
+
+
+class _SGR(IntEnum):
+    """
+    SGR (Select Graphic Rendition) parameter codes.
+
+    References:
+    - https://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+    - https://github.com/tehmaze/ansi/tree/master/ansi/colour
+    """
+
+    RESET = 0
+    BOLD = 1
+    DIM = 2
+    ITALIC = 3
+    UNDERLINE = 4
+    BLINK = 5
+    RAPID_BLINK = 6
+    INVERSE = 7
+    HIDDEN = 8
+    STRIKETHROUGH = 9
+    DOUBLE_UNDERLINE = 21
+    BOLD_DIM_OFF = 22
+    ITALIC_OFF = 23
+    UNDERLINE_OFF = 24
+    BLINK_OFF = 25
+    INVERSE_OFF = 27
+    HIDDEN_OFF = 28
+    STRIKETHROUGH_OFF = 29
+    FG_BLACK = 30
+    FG_WHITE = 37
+    FG_EXTENDED = 38
+    FG_DEFAULT = 39
+    BG_BLACK = 40
+    BG_WHITE = 47
+    BG_EXTENDED = 48
+    BG_DEFAULT = 49
+    FG_BRIGHT_BLACK = 90
+    FG_BRIGHT_WHITE = 97
+    BG_BRIGHT_BLACK = 100
+    BG_BRIGHT_WHITE = 107
+
+
+# SGR sequence pattern: CSI followed by params (digits, semicolons, colons) ending with 'm'
+# Colons are used in ITU T.416 (ISO 8613-6) extended color format: 38:2::R:G:B
+# This colon format is less common than semicolon (38;2;R;G;B) but supported by kitty,
+# iTerm2, and newer VTE-based terminals.
+_SGR_PATTERN = re.compile(r'\x1b\[([\d;:]*)m')
+
+# Fast path: quick check if any SGR sequence exists
+_SGR_QUICK_CHECK = re.compile(r'\x1b\[[\d;:]*m')
+
+# Reset sequence
+_SGR_RESET = '\x1b[0m'
+
+
+class _SGRState(NamedTuple):
+    """
+    Track active SGR terminal attributes by category (immutable).
+
+    :param bold: Bold attribute (SGR 1).
+    :param dim: Dim/faint attribute (SGR 2).
+    :param italic: Italic attribute (SGR 3).
+    :param underline: Underline attribute (SGR 4).
+    :param blink: Slow blink attribute (SGR 5).
+    :param rapid_blink: Rapid blink attribute (SGR 6).
+    :param inverse: Inverse/reverse attribute (SGR 7).
+    :param hidden: Hidden/invisible attribute (SGR 8).
+    :param strikethrough: Strikethrough attribute (SGR 9).
+    :param double_underline: Double underline attribute (SGR 21).
+    :param foreground: Foreground color as tuple of SGR params, or None for default.
+    :param background: Background color as tuple of SGR params, or None for default.
+    """
+
+    bold: bool = False
+    dim: bool = False
+    italic: bool = False
+    underline: bool = False
+    blink: bool = False
+    rapid_blink: bool = False
+    inverse: bool = False
+    hidden: bool = False
+    strikethrough: bool = False
+    double_underline: bool = False
+    foreground: tuple[int, ...] | None = None
+    background: tuple[int, ...] | None = None
+
+
+# Default state with no attributes set
+_SGR_STATE_DEFAULT = _SGRState()
+
+
+def _sgr_state_is_active(state: _SGRState) -> bool:
+    """
+    Return True if any attributes are set.
+
+    :param state: The SGR state to check.
+    :returns: True if any attribute differs from default.
+    """
+    return (state.bold or state.dim or state.italic or state.underline
+            or state.blink or state.rapid_blink or state.inverse or state.hidden
+            or state.strikethrough or state.double_underline
+            or state.foreground is not None or state.background is not None)
+
+
+def _sgr_state_to_sequence(state: _SGRState) -> str:
+    """
+    Generate minimal SGR sequence to restore this state from reset.
+
+    :param state: The SGR state to convert.
+    :returns: SGR escape sequence string, or empty string if no attributes set.
+    """
+    if not _sgr_state_is_active(state):
+        return ''
+
+    # Map boolean attributes to their SGR codes
+    bool_attrs = [
+        (state.bold, '1'), (state.dim, '2'), (state.italic, '3'),
+        (state.underline, '4'), (state.blink, '5'), (state.rapid_blink, '6'),
+        (state.inverse, '7'), (state.hidden, '8'), (state.strikethrough, '9'),
+        (state.double_underline, '21'),
+    ]
+    params = [code for active, code in bool_attrs if active]
+
+    # Add color params (already formatted as tuples)
+    if state.foreground is not None:
+        params.append(';'.join(str(p) for p in state.foreground))
+    if state.background is not None:
+        params.append(';'.join(str(p) for p in state.background))
+
+    return f'\x1b[{";".join(params)}m'
+
+
+def _parse_sgr_params(sequence: str) -> list[int | tuple[int, ...]]:
+    r"""
+    Parse SGR sequence and return list of parameter values.
+
+    Handles compound sequences like ``\x1b[1;31;4m`` -> [1, 31, 4].
+    Empty params (e.g., ``\x1b[m``) are treated as [0] (reset).
+    Colon-separated extended colors like ``\x1b[38:2::255:0:0m`` are returned
+    as tuples: [(38, 2, 255, 0, 0)].
+
+    :param sequence: SGR escape sequence string.
+    :returns: List of integer parameters or tuples for colon-separated colors.
+    """
+    match = _SGR_PATTERN.match(sequence)
+    if not match:
+        return []
+    params_str = match.group(1)
+    if not params_str:
+        return [0]  # \x1b[m is equivalent to \x1b[0m
+    result: list[int | tuple[int, ...]] = []
+    for param in params_str.split(';'):
+        if ':' in param:
+            # Colon-separated extended color (ITU T.416 format)
+            # e.g., "38:2::255:0:0" or "38:2:1:255:0:0" (with colorspace)
+            parts = [int(p) if p else 0 for p in param.split(':')]
+            result.append(tuple(parts))
+        else:
+            result.append(int(param) if param else 0)
+    return result
+
+
+def _parse_extended_color(
+    params: Iterator[int | tuple[int, ...]], base: int
+) -> tuple[int, ...] | None:
+    """
+    Parse extended color (256-color or RGB) from parameter iterator.
+
+    :param params: Iterator of remaining SGR parameters (semicolon-separated format).
+    :param base: Base code (38 for foreground, 48 for background).
+    :returns: Color tuple like (38, 5, N) or (38, 2, R, G, B), or None if malformed.
+    """
+    try:
+        mode = next(params)
+        if isinstance(mode, tuple):
+            return None  # Unexpected tuple, colon format handled separately
+        if mode == 5:  # 256-color
+            n = next(params)
+            if isinstance(n, tuple):
+                return None
+            return (int(base), 5, n)
+        if mode == 2:  # RGB
+            r, g, b = next(params), next(params), next(params)
+            if isinstance(r, tuple) or isinstance(g, tuple) or isinstance(b, tuple):
+                return None
+            return (int(base), 2, r, g, b)
+    except StopIteration:
+        pass
+    return None
+
+
+def _sgr_state_update(state: _SGRState, sequence: str) -> _SGRState:
+    # pylint: disable=too-many-branches,too-complex,too-many-statements
+    # NOTE: When minimum Python version is 3.10+, this can be simplified using match/case.
+    """
+    Parse SGR sequence and return new state with updates applied.
+
+    :param state: Current SGR state.
+    :param sequence: SGR escape sequence string.
+    :returns: New SGRState with updates applied.
+    """
+    params_list = _parse_sgr_params(sequence)
+    params = iter(params_list)
+    for p in params:
+        # Handle colon-separated extended colors (ITU T.416 format)
+        if isinstance(p, tuple):
+            if len(p) >= 2 and p[0] == _SGR.FG_EXTENDED:
+                # Foreground: (38, 2, [colorspace,] R, G, B) or (38, 5, N)
+                state = state._replace(foreground=p)
+            elif len(p) >= 2 and p[0] == _SGR.BG_EXTENDED:
+                # Background: (48, 2, [colorspace,] R, G, B) or (48, 5, N)
+                state = state._replace(background=p)
+            continue
+        if p == _SGR.RESET:
+            state = _SGR_STATE_DEFAULT
+        # Attribute ON codes
+        elif p == _SGR.BOLD:
+            state = state._replace(bold=True)
+        elif p == _SGR.DIM:
+            state = state._replace(dim=True)
+        elif p == _SGR.ITALIC:
+            state = state._replace(italic=True)
+        elif p == _SGR.UNDERLINE:
+            state = state._replace(underline=True)
+        elif p == _SGR.BLINK:
+            state = state._replace(blink=True)
+        elif p == _SGR.RAPID_BLINK:
+            state = state._replace(rapid_blink=True)
+        elif p == _SGR.INVERSE:
+            state = state._replace(inverse=True)
+        elif p == _SGR.HIDDEN:
+            state = state._replace(hidden=True)
+        elif p == _SGR.STRIKETHROUGH:
+            state = state._replace(strikethrough=True)
+        elif p == _SGR.DOUBLE_UNDERLINE:
+            state = state._replace(double_underline=True)
+        # Attribute OFF codes
+        elif p == _SGR.BOLD_DIM_OFF:
+            state = state._replace(bold=False, dim=False)
+        elif p == _SGR.ITALIC_OFF:
+            state = state._replace(italic=False)
+        elif p == _SGR.UNDERLINE_OFF:
+            state = state._replace(underline=False, double_underline=False)
+        elif p == _SGR.BLINK_OFF:
+            state = state._replace(blink=False, rapid_blink=False)
+        elif p == _SGR.INVERSE_OFF:
+            state = state._replace(inverse=False)
+        elif p == _SGR.HIDDEN_OFF:
+            state = state._replace(hidden=False)
+        elif p == _SGR.STRIKETHROUGH_OFF:
+            state = state._replace(strikethrough=False)
+        # Basic colors (30-37, 40-47 standard; 90-97, 100-107 bright)
+        elif (_SGR.FG_BLACK <= p <= _SGR.FG_WHITE
+              or _SGR.FG_BRIGHT_BLACK <= p <= _SGR.FG_BRIGHT_WHITE):
+            state = state._replace(foreground=(p,))
+        elif (_SGR.BG_BLACK <= p <= _SGR.BG_WHITE
+              or _SGR.BG_BRIGHT_BLACK <= p <= _SGR.BG_BRIGHT_WHITE):
+            state = state._replace(background=(p,))
+        elif p == _SGR.FG_DEFAULT:
+            state = state._replace(foreground=None)
+        elif p == _SGR.BG_DEFAULT:
+            state = state._replace(background=None)
+        # Extended colors (semicolon-separated format)
+        elif p == _SGR.FG_EXTENDED:
+            if color := _parse_extended_color(params, _SGR.FG_EXTENDED):
+                state = state._replace(foreground=color)
+        elif p == _SGR.BG_EXTENDED:
+            if color := _parse_extended_color(params, _SGR.BG_EXTENDED):
+                state = state._replace(background=color)
+    return state
+
+
+def propagate_sgr(lines: Sequence[str]) -> list[str]:
+    r"""
+    Propagate SGR codes across wrapped lines.
+
+    When text with SGR styling is wrapped across multiple lines, each line
+    needs to be self-contained for proper display. This function:
+
+    - Ends each line with ``\x1b[0m`` if styles are active (prevents bleeding)
+    - Starts each subsequent line with the active style restored
+
+    :param lines: List of text lines, possibly containing SGR sequences.
+    :returns: List of lines with SGR codes propagated.
+
+    Example::
+
+        >>> propagate_sgr(['\x1b[31mhello', 'world\x1b[0m'])
+        ['\x1b[31mhello\x1b[0m', '\x1b[31mworld\x1b[0m']
+
+    This is useful in cases of making special editors and viewers, and is used for the
+    default modes (propagate_sgr=True) of :func:`wcwidth.width` and :func:`wcwidth.clip`.
+
+    When wrapping and clipping text containing SGR sequences, maybe a previous line enabled the BLUE
+    color--if we are viewing *only* the line following, we would want the carry over the BLUE color,
+    and all lines with sequences should end with terminating reset (``\x1b[0m``).
+    """
+    # Fast path: check if any line contains SGR sequences
+    if not any(_SGR_QUICK_CHECK.search(line) for line in lines) or not lines:
+        return list(lines)
+
+    result: list[str] = []
+    state = _SGR_STATE_DEFAULT
+
+    for line in lines:
+        # Prefix with restoration sequence if state is active
+        prefix = _sgr_state_to_sequence(state)
+
+        # Update state by processing all SGR sequences in this line
+        for match in _SGR_PATTERN.finditer(line):
+            state = _sgr_state_update(state, match.group())
+
+        # Build output line
+        output_line = prefix + line if prefix else line
+        if _sgr_state_is_active(state):
+            output_line = output_line + _SGR_RESET
+
+        result.append(output_line)
+
+    return result
diff --git a/lib/wcwidth/table_ambiguous.py b/lib/wcwidth/table_ambiguous.py
new file mode 100644
index 0000000..e3dc0b1
--- /dev/null
+++ b/lib/wcwidth/table_ambiguous.py
@@ -0,0 +1,189 @@
+"""
+Exports AMBIGUOUS_EASTASIAN table keyed by supporting unicode version level.
+
+This code generated by wcwidth/bin/update-tables.py on 2026-01-18 23:27:15 UTC.
+"""
+# pylint: disable=duplicate-code
+AMBIGUOUS_EASTASIAN = {
+    '17.0.0': (
+        # Source: EastAsianWidth-17.0.0.txt
+        # Date: 2025-07-24, 00:12:54 GMT
+        #
+        (0x000a1, 0x000a1,),  # Inverted Exclamation Mark
+        (0x000a4, 0x000a4,),  # Currency Sign
+        (0x000a7, 0x000a8,),  # Section Sign            ..Diaeresis
+        (0x000aa, 0x000aa,),  # Feminine Ordinal Indicator
+        (0x000ad, 0x000ae,),  # Soft Hyphen             ..Registered Sign
+        (0x000b0, 0x000b4,),  # Degree Sign             ..Acute Accent
+        (0x000b6, 0x000ba,),  # Pilcrow Sign            ..Masculine Ordinal Indica
+        (0x000bc, 0x000bf,),  # Vulgar Fraction One Quar..Inverted Question Mark
+        (0x000c6, 0x000c6,),  # Latin Capital Letter Ae
+        (0x000d0, 0x000d0,),  # Latin Capital Letter Eth
+        (0x000d7, 0x000d8,),  # Multiplication Sign     ..Latin Capital Letter O W
+        (0x000de, 0x000e1,),  # Latin Capital Letter Tho..Latin Small Letter A Wit
+        (0x000e6, 0x000e6,),  # Latin Small Letter Ae
+        (0x000e8, 0x000ea,),  # Latin Small Letter E Wit..Latin Small Letter E Wit
+        (0x000ec, 0x000ed,),  # Latin Small Letter I Wit..Latin Small Letter I Wit
+        (0x000f0, 0x000f0,),  # Latin Small Letter Eth
+        (0x000f2, 0x000f3,),  # Latin Small Letter O Wit..Latin Small Letter O Wit
+        (0x000f7, 0x000fa,),  # Division Sign           ..Latin Small Letter U Wit
+        (0x000fc, 0x000fc,),  # Latin Small Letter U With Diaeresis
+        (0x000fe, 0x000fe,),  # Latin Small Letter Thorn
+        (0x00101, 0x00101,),  # Latin Small Letter A With Macron
+        (0x00111, 0x00111,),  # Latin Small Letter D With Stroke
+        (0x00113, 0x00113,),  # Latin Small Letter E With Macron
+        (0x0011b, 0x0011b,),  # Latin Small Letter E With Caron
+        (0x00126, 0x00127,),  # Latin Capital Letter H W..Latin Small Letter H Wit
+        (0x0012b, 0x0012b,),  # Latin Small Letter I With Macron
+        (0x00131, 0x00133,),  # Latin Small Letter Dotle..Latin Small Ligature Ij
+        (0x00138, 0x00138,),  # Latin Small Letter Kra
+        (0x0013f, 0x00142,),  # Latin Capital Letter L W..Latin Small Letter L Wit
+        (0x00144, 0x00144,),  # Latin Small Letter N With Acute
+        (0x00148, 0x0014b,),  # Latin Small Letter N Wit..Latin Small Letter Eng
+        (0x0014d, 0x0014d,),  # Latin Small Letter O With Macron
+        (0x00152, 0x00153,),  # Latin Capital Ligature O..Latin Small Ligature Oe
+        (0x00166, 0x00167,),  # Latin Capital Letter T W..Latin Small Letter T Wit
+        (0x0016b, 0x0016b,),  # Latin Small Letter U With Macron
+        (0x001ce, 0x001ce,),  # Latin Small Letter A With Caron
+        (0x001d0, 0x001d0,),  # Latin Small Letter I With Caron
+        (0x001d2, 0x001d2,),  # Latin Small Letter O With Caron
+        (0x001d4, 0x001d4,),  # Latin Small Letter U With Caron
+        (0x001d6, 0x001d6,),  # Latin Small Letter U With Diaeresis And Macron
+        (0x001d8, 0x001d8,),  # Latin Small Letter U With Diaeresis And Acute
+        (0x001da, 0x001da,),  # Latin Small Letter U With Diaeresis And Caron
+        (0x001dc, 0x001dc,),  # Latin Small Letter U With Diaeresis And Grave
+        (0x00251, 0x00251,),  # Latin Small Letter Alpha
+        (0x00261, 0x00261,),  # Latin Small Letter Script G
+        (0x002c4, 0x002c4,),  # Modifier Letter Up Arrowhead
+        (0x002c7, 0x002c7,),  # Caron
+        (0x002c9, 0x002cb,),  # Modifier Letter Macron  ..Modifier Letter Grave Ac
+        (0x002cd, 0x002cd,),  # Modifier Letter Low Macron
+        (0x002d0, 0x002d0,),  # Modifier Letter Triangular Colon
+        (0x002d8, 0x002db,),  # Breve                   ..Ogonek
+        (0x002dd, 0x002dd,),  # Double Acute Accent
+        (0x002df, 0x002df,),  # Modifier Letter Cross Accent
+        (0x00391, 0x003a1,),  # Greek Capital Letter Alp..Greek Capital Letter Rho
+        (0x003a3, 0x003a9,),  # Greek Capital Letter Sig..Greek Capital Letter Ome
+        (0x003b1, 0x003c1,),  # Greek Small Letter Alpha..Greek Small Letter Rho
+        (0x003c3, 0x003c9,),  # Greek Small Letter Sigma..Greek Small Letter Omega
+        (0x00401, 0x00401,),  # Cyrillic Capital Letter Io
+        (0x00410, 0x0044f,),  # Cyrillic Capital Letter ..Cyrillic Small Letter Ya
+        (0x00451, 0x00451,),  # Cyrillic Small Letter Io
+        (0x02010, 0x02010,),  # Hyphen
+        (0x02013, 0x02016,),  # En Dash                 ..Double Vertical Line
+        (0x02018, 0x02019,),  # Left Single Quotation Ma..Right Single Quotation M
+        (0x0201c, 0x0201d,),  # Left Double Quotation Ma..Right Double Quotation M
+        (0x02020, 0x02022,),  # Dagger                  ..Bullet
+        (0x02024, 0x02027,),  # One Dot Leader          ..Hyphenation Point
+        (0x02030, 0x02030,),  # Per Mille Sign
+        (0x02032, 0x02033,),  # Prime                   ..Double Prime
+        (0x02035, 0x02035,),  # Reversed Prime
+        (0x0203b, 0x0203b,),  # Reference Mark
+        (0x0203e, 0x0203e,),  # Overline
+        (0x02074, 0x02074,),  # Superscript Four
+        (0x0207f, 0x0207f,),  # Superscript Latin Small Letter N
+        (0x02081, 0x02084,),  # Subscript One           ..Subscript Four
+        (0x020ac, 0x020ac,),  # Euro Sign
+        (0x02103, 0x02103,),  # Degree Celsius
+        (0x02105, 0x02105,),  # Care Of
+        (0x02109, 0x02109,),  # Degree Fahrenheit
+        (0x02113, 0x02113,),  # Script Small L
+        (0x02116, 0x02116,),  # Numero Sign
+        (0x02121, 0x02122,),  # Telephone Sign          ..Trade Mark Sign
+        (0x02126, 0x02126,),  # Ohm Sign
+        (0x0212b, 0x0212b,),  # Angstrom Sign
+        (0x02153, 0x02154,),  # Vulgar Fraction One Thir..Vulgar Fraction Two Thir
+        (0x0215b, 0x0215e,),  # Vulgar Fraction One Eigh..Vulgar Fraction Seven Ei
+        (0x02160, 0x0216b,),  # Roman Numeral One       ..Roman Numeral Twelve
+        (0x02170, 0x02179,),  # Small Roman Numeral One ..Small Roman Numeral Ten
+        (0x02189, 0x02189,),  # Vulgar Fraction Zero Thirds
+        (0x02190, 0x02199,),  # Leftwards Arrow         ..South West Arrow
+        (0x021b8, 0x021b9,),  # North West Arrow To Long..Leftwards Arrow To Bar O
+        (0x021d2, 0x021d2,),  # Rightwards Double Arrow
+        (0x021d4, 0x021d4,),  # Left Right Double Arrow
+        (0x021e7, 0x021e7,),  # Upwards White Arrow
+        (0x02200, 0x02200,),  # For All
+        (0x02202, 0x02203,),  # Partial Differential    ..There Exists
+        (0x02207, 0x02208,),  # Nabla                   ..Element Of
+        (0x0220b, 0x0220b,),  # Contains As Member
+        (0x0220f, 0x0220f,),  # N-ary Product
+        (0x02211, 0x02211,),  # N-ary Summation
+        (0x02215, 0x02215,),  # Division Slash
+        (0x0221a, 0x0221a,),  # Square Root
+        (0x0221d, 0x02220,),  # Proportional To         ..Angle
+        (0x02223, 0x02223,),  # Divides
+        (0x02225, 0x02225,),  # Parallel To
+        (0x02227, 0x0222c,),  # Logical And             ..Double Integral
+        (0x0222e, 0x0222e,),  # Contour Integral
+        (0x02234, 0x02237,),  # Therefore               ..Proportion
+        (0x0223c, 0x0223d,),  # Tilde Operator          ..Reversed Tilde
+        (0x02248, 0x02248,),  # Almost Equal To
+        (0x0224c, 0x0224c,),  # All Equal To
+        (0x02252, 0x02252,),  # Approximately Equal To Or The Image Of
+        (0x02260, 0x02261,),  # Not Equal To            ..Identical To
+        (0x02264, 0x02267,),  # Less-than Or Equal To   ..Greater-than Over Equal
+        (0x0226a, 0x0226b,),  # Much Less-than          ..Much Greater-than
+        (0x0226e, 0x0226f,),  # Not Less-than           ..Not Greater-than
+        (0x02282, 0x02283,),  # Subset Of               ..Superset Of
+        (0x02286, 0x02287,),  # Subset Of Or Equal To   ..Superset Of Or Equal To
+        (0x02295, 0x02295,),  # Circled Plus
+        (0x02299, 0x02299,),  # Circled Dot Operator
+        (0x022a5, 0x022a5,),  # Up Tack
+        (0x022bf, 0x022bf,),  # Right Triangle
+        (0x02312, 0x02312,),  # Arc
+        (0x02460, 0x024e9,),  # Circled Digit One       ..Circled Latin Small Lett
+        (0x024eb, 0x0254b,),  # Negative Circled Number ..Box Drawings Heavy Verti
+        (0x02550, 0x02573,),  # Box Drawings Double Hori..Box Drawings Light Diago
+        (0x02580, 0x0258f,),  # Upper Half Block        ..Left One Eighth Block
+        (0x02592, 0x02595,),  # Medium Shade            ..Right One Eighth Block
+        (0x025a0, 0x025a1,),  # Black Square            ..White Square
+        (0x025a3, 0x025a9,),  # White Square Containing ..Square With Diagonal Cro
+        (0x025b2, 0x025b3,),  # Black Up-pointing Triang..White Up-pointing Triang
+        (0x025b6, 0x025b7,),  # Black Right-pointing Tri..White Right-pointing Tri
+        (0x025bc, 0x025bd,),  # Black Down-pointing Tria..White Down-pointing Tria
+        (0x025c0, 0x025c1,),  # Black Left-pointing Tria..White Left-pointing Tria
+        (0x025c6, 0x025c8,),  # Black Diamond           ..White Diamond Containing
+        (0x025cb, 0x025cb,),  # White Circle
+        (0x025ce, 0x025d1,),  # Bullseye                ..Circle With Right Half B
+        (0x025e2, 0x025e5,),  # Black Lower Right Triang..Black Upper Right Triang
+        (0x025ef, 0x025ef,),  # Large Circle
+        (0x02605, 0x02606,),  # Black Star              ..White Star
+        (0x02609, 0x02609,),  # Sun
+        (0x0260e, 0x0260f,),  # Black Telephone         ..White Telephone
+        (0x0261c, 0x0261c,),  # White Left Pointing Index
+        (0x0261e, 0x0261e,),  # White Right Pointing Index
+        (0x02640, 0x02640,),  # Female Sign
+        (0x02642, 0x02642,),  # Male Sign
+        (0x02660, 0x02661,),  # Black Spade Suit        ..White Heart Suit
+        (0x02663, 0x02665,),  # Black Club Suit         ..Black Heart Suit
+        (0x02667, 0x0266a,),  # White Club Suit         ..Eighth Note
+        (0x0266c, 0x0266d,),  # Beamed Sixteenth Notes  ..Music Flat Sign
+        (0x0266f, 0x0266f,),  # Music Sharp Sign
+        (0x0269e, 0x0269f,),  # Three Lines Converging R..Three Lines Converging L
+        (0x026bf, 0x026bf,),  # Squared Key
+        (0x026c6, 0x026cd,),  # Rain                    ..Disabled Car
+        (0x026cf, 0x026d3,),  # Pick                    ..Chains
+        (0x026d5, 0x026e1,),  # Alternate One-way Left W..Restricted Left Entry-2
+        (0x026e3, 0x026e3,),  # Heavy Circle With Stroke And Two Dots Above
+        (0x026e8, 0x026e9,),  # Black Cross On Shield   ..Shinto Shrine
+        (0x026eb, 0x026f1,),  # Castle                  ..Umbrella On Ground
+        (0x026f4, 0x026f4,),  # Ferry
+        (0x026f6, 0x026f9,),  # Square Four Corners     ..Person With Ball
+        (0x026fb, 0x026fc,),  # Japanese Bank Symbol    ..Headstone Graveyard Symb
+        (0x026fe, 0x026ff,),  # Cup On Black Square     ..White Flag With Horizont
+        (0x0273d, 0x0273d,),  # Heavy Teardrop-spoked Asterisk
+        (0x02776, 0x0277f,),  # Dingbat Negative Circled..Dingbat Negative Circled
+        (0x02b56, 0x02b59,),  # Heavy Oval With Oval Ins..Heavy Circled Saltire
+        (0x03248, 0x0324f,),  # Circled Number Ten On Bl..Circled Number Eighty On
+        (0x0e000, 0x0f8ff,),  # (nil)
+        (0x0fffd, 0x0fffd,),  # Replacement Character
+        (0x1f100, 0x1f10a,),  # Digit Zero Full Stop    ..Digit Nine Comma
+        (0x1f110, 0x1f12d,),  # Parenthesized Latin Capi..Circled Cd
+        (0x1f130, 0x1f169,),  # Squared Latin Capital Le..Negative Circled Latin C
+        (0x1f170, 0x1f18d,),  # Negative Squared Latin C..Negative Squared Sa
+        (0x1f18f, 0x1f190,),  # Negative Squared Wc     ..Square Dj
+        (0x1f19b, 0x1f1ac,),  # Squared Three D         ..Squared Vod
+        (0xf0000, 0xffffd,),  # (nil)
+        (0x100000, 0x10fffd,),  # (nil)
+    ),
+}
diff --git a/lib/wcwidth/table_grapheme.py b/lib/wcwidth/table_grapheme.py
new file mode 100644
index 0000000..42fd19e
--- /dev/null
+++ b/lib/wcwidth/table_grapheme.py
@@ -0,0 +1,2294 @@
+"""
+Exports grapheme cluster break property tables for Unicode version 17.0.0.
+
+This module provides lookup tables for Unicode grapheme cluster break properties as defined in UAX
+#29: Unicode Text Segmentation.
+
+This code generated by wcwidth/bin/update-tables.py on 2026-01-29 23:33:42 UTC.
+"""
+# pylint: disable=duplicate-code
+
+GRAPHEME_CR = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x0000d, 0x0000d,),  # (nil)
+)
+
+GRAPHEME_LF = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x0000a, 0x0000a,),  # (nil)
+)
+
+GRAPHEME_CONTROL = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x00000, 0x00009,),  # (nil)
+    (0x0000b, 0x0000c,),  # (nil)
+    (0x0000e, 0x0001f,),  # (nil)
+    (0x0007f, 0x0009f,),  # (nil)
+    (0x000ad, 0x000ad,),  # Soft Hyphen
+    (0x0061c, 0x0061c,),  # Arabic Letter Mark
+    (0x0180e, 0x0180e,),  # Mongolian Vowel Separator
+    (0x0200b, 0x0200b,),  # Zero Width Space
+    (0x0200e, 0x0200f,),  # Left-to-right Mark      ..Right-to-left Mark
+    (0x02028, 0x0202e,),  # Line Separator          ..Right-to-left Override
+    (0x02060, 0x0206f,),  # Word Joiner             ..Nominal Digit Shapes
+    (0x0feff, 0x0feff,),  # Zero Width No-break Space
+    (0x0fff0, 0x0fffb,),  # (nil)                   ..Interlinear Annotation T
+    (0x13430, 0x1343f,),  # Egyptian Hieroglyph Vert..Egyptian Hieroglyph End
+    (0x1bca0, 0x1bca3,),  # Shorthand Format Letter ..Shorthand Format Up Step
+    (0x1d173, 0x1d17a,),  # Musical Symbol Begin Bea..Musical Symbol End Phras
+    (0xe0000, 0xe001f,),  # (nil)
+    (0xe0080, 0xe00ff,),  # (nil)
+    (0xe01f0, 0xe0fff,),  # (nil)
+)
+
+GRAPHEME_EXTEND = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x00300, 0x0036f,),  # Combining Grave Accent  ..Combining Latin Small Le
+    (0x00483, 0x00489,),  # Combining Cyrillic Titlo..Combining Cyrillic Milli
+    (0x00591, 0x005bd,),  # Hebrew Accent Etnahta   ..Hebrew Point Meteg
+    (0x005bf, 0x005bf,),  # Hebrew Point Rafe
+    (0x005c1, 0x005c2,),  # Hebrew Point Shin Dot   ..Hebrew Point Sin Dot
+    (0x005c4, 0x005c5,),  # Hebrew Mark Upper Dot   ..Hebrew Mark Lower Dot
+    (0x005c7, 0x005c7,),  # Hebrew Point Qamats Qatan
+    (0x00610, 0x0061a,),  # Arabic Sign Sallallahou ..Arabic Small Kasra
+    (0x0064b, 0x0065f,),  # Arabic Fathatan         ..Arabic Wavy Hamza Below
+    (0x00670, 0x00670,),  # Arabic Letter Superscript Alef
+    (0x006d6, 0x006dc,),  # Arabic Small High Ligatu..Arabic Small High Seen
+    (0x006df, 0x006e4,),  # Arabic Small High Rounde..Arabic Small High Madda
+    (0x006e7, 0x006e8,),  # Arabic Small High Yeh   ..Arabic Small High Noon
+    (0x006ea, 0x006ed,),  # Arabic Empty Centre Low ..Arabic Small Low Meem
+    (0x00711, 0x00711,),  # Syriac Letter Superscript Alaph
+    (0x00730, 0x0074a,),  # Syriac Pthaha Above     ..Syriac Barrekh
+    (0x007a6, 0x007b0,),  # Thaana Abafili          ..Thaana Sukun
+    (0x007eb, 0x007f3,),  # Nko Combining Short High..Nko Combining Double Dot
+    (0x007fd, 0x007fd,),  # Nko Dantayalan
+    (0x00816, 0x00819,),  # Samaritan Mark In       ..Samaritan Mark Dagesh
+    (0x0081b, 0x00823,),  # Samaritan Mark Epentheti..Samaritan Vowel Sign A
+    (0x00825, 0x00827,),  # Samaritan Vowel Sign Sho..Samaritan Vowel Sign U
+    (0x00829, 0x0082d,),  # Samaritan Vowel Sign Lon..Samaritan Mark Nequdaa
+    (0x00859, 0x0085b,),  # Mandaic Affrication Mark..Mandaic Gemination Mark
+    (0x00897, 0x0089f,),  # Arabic Pepet            ..Arabic Half Madda Over M
+    (0x008ca, 0x008e1,),  # Arabic Small High Farsi ..Arabic Small High Sign S
+    (0x008e3, 0x00902,),  # Arabic Turned Damma Belo..Devanagari Sign Anusvara
+    (0x0093a, 0x0093a,),  # Devanagari Vowel Sign Oe
+    (0x0093c, 0x0093c,),  # Devanagari Sign Nukta
+    (0x00941, 0x00948,),  # Devanagari Vowel Sign U ..Devanagari Vowel Sign Ai
+    (0x0094d, 0x0094d,),  # Devanagari Sign Virama
+    (0x00951, 0x00957,),  # Devanagari Stress Sign U..Devanagari Vowel Sign Uu
+    (0x00962, 0x00963,),  # Devanagari Vowel Sign Vo..Devanagari Vowel Sign Vo
+    (0x00981, 0x00981,),  # Bengali Sign Candrabindu
+    (0x009bc, 0x009bc,),  # Bengali Sign Nukta
+    (0x009be, 0x009be,),  # Bengali Vowel Sign Aa
+    (0x009c1, 0x009c4,),  # Bengali Vowel Sign U    ..Bengali Vowel Sign Vocal
+    (0x009cd, 0x009cd,),  # Bengali Sign Virama
+    (0x009d7, 0x009d7,),  # Bengali Au Length Mark
+    (0x009e2, 0x009e3,),  # Bengali Vowel Sign Vocal..Bengali Vowel Sign Vocal
+    (0x009fe, 0x009fe,),  # Bengali Sandhi Mark
+    (0x00a01, 0x00a02,),  # Gurmukhi Sign Adak Bindi..Gurmukhi Sign Bindi
+    (0x00a3c, 0x00a3c,),  # Gurmukhi Sign Nukta
+    (0x00a41, 0x00a42,),  # Gurmukhi Vowel Sign U   ..Gurmukhi Vowel Sign Uu
+    (0x00a47, 0x00a48,),  # Gurmukhi Vowel Sign Ee  ..Gurmukhi Vowel Sign Ai
+    (0x00a4b, 0x00a4d,),  # Gurmukhi Vowel Sign Oo  ..Gurmukhi Sign Virama
+    (0x00a51, 0x00a51,),  # Gurmukhi Sign Udaat
+    (0x00a70, 0x00a71,),  # Gurmukhi Tippi          ..Gurmukhi Addak
+    (0x00a75, 0x00a75,),  # Gurmukhi Sign Yakash
+    (0x00a81, 0x00a82,),  # Gujarati Sign Candrabind..Gujarati Sign Anusvara
+    (0x00abc, 0x00abc,),  # Gujarati Sign Nukta
+    (0x00ac1, 0x00ac5,),  # Gujarati Vowel Sign U   ..Gujarati Vowel Sign Cand
+    (0x00ac7, 0x00ac8,),  # Gujarati Vowel Sign E   ..Gujarati Vowel Sign Ai
+    (0x00acd, 0x00acd,),  # Gujarati Sign Virama
+    (0x00ae2, 0x00ae3,),  # Gujarati Vowel Sign Voca..Gujarati Vowel Sign Voca
+    (0x00afa, 0x00aff,),  # Gujarati Sign Sukun     ..Gujarati Sign Two-circle
+    (0x00b01, 0x00b01,),  # Oriya Sign Candrabindu
+    (0x00b3c, 0x00b3c,),  # Oriya Sign Nukta
+    (0x00b3e, 0x00b3f,),  # Oriya Vowel Sign Aa     ..Oriya Vowel Sign I
+    (0x00b41, 0x00b44,),  # Oriya Vowel Sign U      ..Oriya Vowel Sign Vocalic
+    (0x00b4d, 0x00b4d,),  # Oriya Sign Virama
+    (0x00b55, 0x00b57,),  # Oriya Sign Overline     ..Oriya Au Length Mark
+    (0x00b62, 0x00b63,),  # Oriya Vowel Sign Vocalic..Oriya Vowel Sign Vocalic
+    (0x00b82, 0x00b82,),  # Tamil Sign Anusvara
+    (0x00bbe, 0x00bbe,),  # Tamil Vowel Sign Aa
+    (0x00bc0, 0x00bc0,),  # Tamil Vowel Sign Ii
+    (0x00bcd, 0x00bcd,),  # Tamil Sign Virama
+    (0x00bd7, 0x00bd7,),  # Tamil Au Length Mark
+    (0x00c00, 0x00c00,),  # Telugu Sign Combining Candrabindu Above
+    (0x00c04, 0x00c04,),  # Telugu Sign Combining Anusvara Above
+    (0x00c3c, 0x00c3c,),  # Telugu Sign Nukta
+    (0x00c3e, 0x00c40,),  # Telugu Vowel Sign Aa    ..Telugu Vowel Sign Ii
+    (0x00c46, 0x00c48,),  # Telugu Vowel Sign E     ..Telugu Vowel Sign Ai
+    (0x00c4a, 0x00c4d,),  # Telugu Vowel Sign O     ..Telugu Sign Virama
+    (0x00c55, 0x00c56,),  # Telugu Length Mark      ..Telugu Ai Length Mark
+    (0x00c62, 0x00c63,),  # Telugu Vowel Sign Vocali..Telugu Vowel Sign Vocali
+    (0x00c81, 0x00c81,),  # Kannada Sign Candrabindu
+    (0x00cbc, 0x00cbc,),  # Kannada Sign Nukta
+    (0x00cbf, 0x00cc0,),  # Kannada Vowel Sign I    ..Kannada Vowel Sign Ii
+    (0x00cc2, 0x00cc2,),  # Kannada Vowel Sign Uu
+    (0x00cc6, 0x00cc8,),  # Kannada Vowel Sign E    ..Kannada Vowel Sign Ai
+    (0x00cca, 0x00ccd,),  # Kannada Vowel Sign O    ..Kannada Sign Virama
+    (0x00cd5, 0x00cd6,),  # Kannada Length Mark     ..Kannada Ai Length Mark
+    (0x00ce2, 0x00ce3,),  # Kannada Vowel Sign Vocal..Kannada Vowel Sign Vocal
+    (0x00d00, 0x00d01,),  # Malayalam Sign Combining..Malayalam Sign Candrabin
+    (0x00d3b, 0x00d3c,),  # Malayalam Sign Vertical ..Malayalam Sign Circular
+    (0x00d3e, 0x00d3e,),  # Malayalam Vowel Sign Aa
+    (0x00d41, 0x00d44,),  # Malayalam Vowel Sign U  ..Malayalam Vowel Sign Voc
+    (0x00d4d, 0x00d4d,),  # Malayalam Sign Virama
+    (0x00d57, 0x00d57,),  # Malayalam Au Length Mark
+    (0x00d62, 0x00d63,),  # Malayalam Vowel Sign Voc..Malayalam Vowel Sign Voc
+    (0x00d81, 0x00d81,),  # Sinhala Sign Candrabindu
+    (0x00dca, 0x00dca,),  # Sinhala Sign Al-lakuna
+    (0x00dcf, 0x00dcf,),  # Sinhala Vowel Sign Aela-pilla
+    (0x00dd2, 0x00dd4,),  # Sinhala Vowel Sign Ketti..Sinhala Vowel Sign Ketti
+    (0x00dd6, 0x00dd6,),  # Sinhala Vowel Sign Diga Paa-pilla
+    (0x00ddf, 0x00ddf,),  # Sinhala Vowel Sign Gayanukitta
+    (0x00e31, 0x00e31,),  # Thai Character Mai Han-akat
+    (0x00e34, 0x00e3a,),  # Thai Character Sara I   ..Thai Character Phinthu
+    (0x00e47, 0x00e4e,),  # Thai Character Maitaikhu..Thai Character Yamakkan
+    (0x00eb1, 0x00eb1,),  # Lao Vowel Sign Mai Kan
+    (0x00eb4, 0x00ebc,),  # Lao Vowel Sign I        ..Lao Semivowel Sign Lo
+    (0x00ec8, 0x00ece,),  # Lao Tone Mai Ek         ..Lao Yamakkan
+    (0x00f18, 0x00f19,),  # Tibetan Astrological Sig..Tibetan Astrological Sig
+    (0x00f35, 0x00f35,),  # Tibetan Mark Ngas Bzung Nyi Zla
+    (0x00f37, 0x00f37,),  # Tibetan Mark Ngas Bzung Sgor Rtags
+    (0x00f39, 0x00f39,),  # Tibetan Mark Tsa -phru
+    (0x00f71, 0x00f7e,),  # Tibetan Vowel Sign Aa   ..Tibetan Sign Rjes Su Nga
+    (0x00f80, 0x00f84,),  # Tibetan Vowel Sign Rever..Tibetan Mark Halanta
+    (0x00f86, 0x00f87,),  # Tibetan Sign Lci Rtags  ..Tibetan Sign Yang Rtags
+    (0x00f8d, 0x00f97,),  # Tibetan Subjoined Sign L..Tibetan Subjoined Letter
+    (0x00f99, 0x00fbc,),  # Tibetan Subjoined Letter..Tibetan Subjoined Letter
+    (0x00fc6, 0x00fc6,),  # Tibetan Symbol Padma Gdan
+    (0x0102d, 0x01030,),  # Myanmar Vowel Sign I    ..Myanmar Vowel Sign Uu
+    (0x01032, 0x01037,),  # Myanmar Vowel Sign Ai   ..Myanmar Sign Dot Below
+    (0x01039, 0x0103a,),  # Myanmar Sign Virama     ..Myanmar Sign Asat
+    (0x0103d, 0x0103e,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+    (0x01058, 0x01059,),  # Myanmar Vowel Sign Vocal..Myanmar Vowel Sign Vocal
+    (0x0105e, 0x01060,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+    (0x01071, 0x01074,),  # Myanmar Vowel Sign Geba ..Myanmar Vowel Sign Kayah
+    (0x01082, 0x01082,),  # Myanmar Consonant Sign Shan Medial Wa
+    (0x01085, 0x01086,),  # Myanmar Vowel Sign Shan ..Myanmar Vowel Sign Shan
+    (0x0108d, 0x0108d,),  # Myanmar Sign Shan Council Emphatic Tone
+    (0x0109d, 0x0109d,),  # Myanmar Vowel Sign Aiton Ai
+    (0x0135d, 0x0135f,),  # Ethiopic Combining Gemin..Ethiopic Combining Gemin
+    (0x01712, 0x01715,),  # Tagalog Vowel Sign I    ..Tagalog Sign Pamudpod
+    (0x01732, 0x01734,),  # Hanunoo Vowel Sign I    ..Hanunoo Sign Pamudpod
+    (0x01752, 0x01753,),  # Buhid Vowel Sign I      ..Buhid Vowel Sign U
+    (0x01772, 0x01773,),  # Tagbanwa Vowel Sign I   ..Tagbanwa Vowel Sign U
+    (0x017b4, 0x017b5,),  # Khmer Vowel Inherent Aq ..Khmer Vowel Inherent Aa
+    (0x017b7, 0x017bd,),  # Khmer Vowel Sign I      ..Khmer Vowel Sign Ua
+    (0x017c6, 0x017c6,),  # Khmer Sign Nikahit
+    (0x017c9, 0x017d3,),  # Khmer Sign Muusikatoan  ..Khmer Sign Bathamasat
+    (0x017dd, 0x017dd,),  # Khmer Sign Atthacan
+    (0x0180b, 0x0180d,),  # Mongolian Free Variation..Mongolian Free Variation
+    (0x0180f, 0x0180f,),  # Mongolian Free Variation Selector Four
+    (0x01885, 0x01886,),  # Mongolian Letter Ali Gal..Mongolian Letter Ali Gal
+    (0x018a9, 0x018a9,),  # Mongolian Letter Ali Gali Dagalga
+    (0x01920, 0x01922,),  # Limbu Vowel Sign A      ..Limbu Vowel Sign U
+    (0x01927, 0x01928,),  # Limbu Vowel Sign E      ..Limbu Vowel Sign O
+    (0x01932, 0x01932,),  # Limbu Small Letter Anusvara
+    (0x01939, 0x0193b,),  # Limbu Sign Mukphreng    ..Limbu Sign Sa-i
+    (0x01a17, 0x01a18,),  # Buginese Vowel Sign I   ..Buginese Vowel Sign U
+    (0x01a1b, 0x01a1b,),  # Buginese Vowel Sign Ae
+    (0x01a56, 0x01a56,),  # Tai Tham Consonant Sign Medial La
+    (0x01a58, 0x01a5e,),  # Tai Tham Sign Mai Kang L..Tai Tham Consonant Sign
+    (0x01a60, 0x01a60,),  # Tai Tham Sign Sakot
+    (0x01a62, 0x01a62,),  # Tai Tham Vowel Sign Mai Sat
+    (0x01a65, 0x01a6c,),  # Tai Tham Vowel Sign I   ..Tai Tham Vowel Sign Oa B
+    (0x01a73, 0x01a7c,),  # Tai Tham Vowel Sign Oa A..Tai Tham Sign Khuen-lue
+    (0x01a7f, 0x01a7f,),  # Tai Tham Combining Cryptogrammic Dot
+    (0x01ab0, 0x01add,),  # Combining Doubled Circum..Combining Dot-and-ring B
+    (0x01ae0, 0x01aeb,),  # Combining Left Tack Abov..Combining Double Rightwa
+    (0x01b00, 0x01b03,),  # Balinese Sign Ulu Ricem ..Balinese Sign Surang
+    (0x01b34, 0x01b3d,),  # Balinese Sign Rerekan   ..Balinese Vowel Sign La L
+    (0x01b42, 0x01b44,),  # Balinese Vowel Sign Pepe..Balinese Adeg Adeg
+    (0x01b6b, 0x01b73,),  # Balinese Musical Symbol ..Balinese Musical Symbol
+    (0x01b80, 0x01b81,),  # Sundanese Sign Panyecek ..Sundanese Sign Panglayar
+    (0x01ba2, 0x01ba5,),  # Sundanese Consonant Sign..Sundanese Vowel Sign Pan
+    (0x01ba8, 0x01bad,),  # Sundanese Vowel Sign Pam..Sundanese Consonant Sign
+    (0x01be6, 0x01be6,),  # Batak Sign Tompi
+    (0x01be8, 0x01be9,),  # Batak Vowel Sign Pakpak ..Batak Vowel Sign Ee
+    (0x01bed, 0x01bed,),  # Batak Vowel Sign Karo O
+    (0x01bef, 0x01bf3,),  # Batak Vowel Sign U For S..Batak Panongonan
+    (0x01c2c, 0x01c33,),  # Lepcha Vowel Sign E     ..Lepcha Consonant Sign T
+    (0x01c36, 0x01c37,),  # Lepcha Sign Ran         ..Lepcha Sign Nukta
+    (0x01cd0, 0x01cd2,),  # Vedic Tone Karshana     ..Vedic Tone Prenkha
+    (0x01cd4, 0x01ce0,),  # Vedic Sign Yajurvedic Mi..Vedic Tone Rigvedic Kash
+    (0x01ce2, 0x01ce8,),  # Vedic Sign Visarga Svari..Vedic Sign Visarga Anuda
+    (0x01ced, 0x01ced,),  # Vedic Sign Tiryak
+    (0x01cf4, 0x01cf4,),  # Vedic Tone Candra Above
+    (0x01cf8, 0x01cf9,),  # Vedic Tone Ring Above   ..Vedic Tone Double Ring A
+    (0x01dc0, 0x01dff,),  # Combining Dotted Grave A..Combining Right Arrowhea
+    (0x0200c, 0x0200c,),  # Zero Width Non-joiner
+    (0x020d0, 0x020f0,),  # Combining Left Harpoon A..Combining Asterisk Above
+    (0x02cef, 0x02cf1,),  # Coptic Combining Ni Abov..Coptic Combining Spiritu
+    (0x02d7f, 0x02d7f,),  # Tifinagh Consonant Joiner
+    (0x02de0, 0x02dff,),  # Combining Cyrillic Lette..Combining Cyrillic Lette
+    (0x0302a, 0x0302f,),  # Ideographic Level Tone M..Hangul Double Dot Tone M
+    (0x03099, 0x0309a,),  # Combining Katakana-hirag..Combining Katakana-hirag
+    (0x0a66f, 0x0a672,),  # Combining Cyrillic Vzmet..Combining Cyrillic Thous
+    (0x0a674, 0x0a67d,),  # Combining Cyrillic Lette..Combining Cyrillic Payer
+    (0x0a69e, 0x0a69f,),  # Combining Cyrillic Lette..Combining Cyrillic Lette
+    (0x0a6f0, 0x0a6f1,),  # Bamum Combining Mark Koq..Bamum Combining Mark Tuk
+    (0x0a802, 0x0a802,),  # Syloti Nagri Sign Dvisvara
+    (0x0a806, 0x0a806,),  # Syloti Nagri Sign Hasanta
+    (0x0a80b, 0x0a80b,),  # Syloti Nagri Sign Anusvara
+    (0x0a825, 0x0a826,),  # Syloti Nagri Vowel Sign ..Syloti Nagri Vowel Sign
+    (0x0a82c, 0x0a82c,),  # Syloti Nagri Sign Alternate Hasanta
+    (0x0a8c4, 0x0a8c5,),  # Saurashtra Sign Virama  ..Saurashtra Sign Candrabi
+    (0x0a8e0, 0x0a8f1,),  # Combining Devanagari Dig..Combining Devanagari Sig
+    (0x0a8ff, 0x0a8ff,),  # Devanagari Vowel Sign Ay
+    (0x0a926, 0x0a92d,),  # Kayah Li Vowel Ue       ..Kayah Li Tone Calya Plop
+    (0x0a947, 0x0a951,),  # Rejang Vowel Sign I     ..Rejang Consonant Sign R
+    (0x0a953, 0x0a953,),  # Rejang Virama
+    (0x0a980, 0x0a982,),  # Javanese Sign Panyangga ..Javanese Sign Layar
+    (0x0a9b3, 0x0a9b3,),  # Javanese Sign Cecak Telu
+    (0x0a9b6, 0x0a9b9,),  # Javanese Vowel Sign Wulu..Javanese Vowel Sign Suku
+    (0x0a9bc, 0x0a9bd,),  # Javanese Vowel Sign Pepe..Javanese Consonant Sign
+    (0x0a9c0, 0x0a9c0,),  # Javanese Pangkon
+    (0x0a9e5, 0x0a9e5,),  # Myanmar Sign Shan Saw
+    (0x0aa29, 0x0aa2e,),  # Cham Vowel Sign Aa      ..Cham Vowel Sign Oe
+    (0x0aa31, 0x0aa32,),  # Cham Vowel Sign Au      ..Cham Vowel Sign Ue
+    (0x0aa35, 0x0aa36,),  # Cham Consonant Sign La  ..Cham Consonant Sign Wa
+    (0x0aa43, 0x0aa43,),  # Cham Consonant Sign Final Ng
+    (0x0aa4c, 0x0aa4c,),  # Cham Consonant Sign Final M
+    (0x0aa7c, 0x0aa7c,),  # Myanmar Sign Tai Laing Tone-2
+    (0x0aab0, 0x0aab0,),  # Tai Viet Mai Kang
+    (0x0aab2, 0x0aab4,),  # Tai Viet Vowel I        ..Tai Viet Vowel U
+    (0x0aab7, 0x0aab8,),  # Tai Viet Mai Khit       ..Tai Viet Vowel Ia
+    (0x0aabe, 0x0aabf,),  # Tai Viet Vowel Am       ..Tai Viet Tone Mai Ek
+    (0x0aac1, 0x0aac1,),  # Tai Viet Tone Mai Tho
+    (0x0aaec, 0x0aaed,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+    (0x0aaf6, 0x0aaf6,),  # Meetei Mayek Virama
+    (0x0abe5, 0x0abe5,),  # Meetei Mayek Vowel Sign Anap
+    (0x0abe8, 0x0abe8,),  # Meetei Mayek Vowel Sign Unap
+    (0x0abed, 0x0abed,),  # Meetei Mayek Apun Iyek
+    (0x0fb1e, 0x0fb1e,),  # Hebrew Point Judeo-spanish Varika
+    (0x0fe00, 0x0fe0f,),  # Variation Selector-1    ..Variation Selector-16
+    (0x0fe20, 0x0fe2f,),  # Combining Ligature Left ..Combining Cyrillic Titlo
+    (0x0ff9e, 0x0ff9f,),  # Halfwidth Katakana Voice..Halfwidth Katakana Semi-
+    (0x101fd, 0x101fd,),  # Phaistos Disc Sign Combining Oblique Stroke
+    (0x102e0, 0x102e0,),  # Coptic Epact Thousands Mark
+    (0x10376, 0x1037a,),  # Combining Old Permic Let..Combining Old Permic Let
+    (0x10a01, 0x10a03,),  # Kharoshthi Vowel Sign I ..Kharoshthi Vowel Sign Vo
+    (0x10a05, 0x10a06,),  # Kharoshthi Vowel Sign E ..Kharoshthi Vowel Sign O
+    (0x10a0c, 0x10a0f,),  # Kharoshthi Vowel Length ..Kharoshthi Sign Visarga
+    (0x10a38, 0x10a3a,),  # Kharoshthi Sign Bar Abov..Kharoshthi Sign Dot Belo
+    (0x10a3f, 0x10a3f,),  # Kharoshthi Virama
+    (0x10ae5, 0x10ae6,),  # Manichaean Abbreviation ..Manichaean Abbreviation
+    (0x10d24, 0x10d27,),  # Hanifi Rohingya Sign Har..Hanifi Rohingya Sign Tas
+    (0x10d69, 0x10d6d,),  # Garay Vowel Sign E      ..Garay Consonant Nasaliza
+    (0x10eab, 0x10eac,),  # Yezidi Combining Hamza M..Yezidi Combining Madda M
+    (0x10efa, 0x10eff,),  # Arabic Double Vertical B..Arabic Small Low Word Ma
+    (0x10f46, 0x10f50,),  # Sogdian Combining Dot Be..Sogdian Combining Stroke
+    (0x10f82, 0x10f85,),  # Old Uyghur Combining Dot..Old Uyghur Combining Two
+    (0x11001, 0x11001,),  # Brahmi Sign Anusvara
+    (0x11038, 0x11046,),  # Brahmi Vowel Sign Aa    ..Brahmi Virama
+    (0x11070, 0x11070,),  # Brahmi Sign Old Tamil Virama
+    (0x11073, 0x11074,),  # Brahmi Vowel Sign Old Ta..Brahmi Vowel Sign Old Ta
+    (0x1107f, 0x11081,),  # Brahmi Number Joiner    ..Kaithi Sign Anusvara
+    (0x110b3, 0x110b6,),  # Kaithi Vowel Sign U     ..Kaithi Vowel Sign Ai
+    (0x110b9, 0x110ba,),  # Kaithi Sign Virama      ..Kaithi Sign Nukta
+    (0x110c2, 0x110c2,),  # Kaithi Vowel Sign Vocalic R
+    (0x11100, 0x11102,),  # Chakma Sign Candrabindu ..Chakma Sign Visarga
+    (0x11127, 0x1112b,),  # Chakma Vowel Sign A     ..Chakma Vowel Sign Uu
+    (0x1112d, 0x11134,),  # Chakma Vowel Sign Ai    ..Chakma Maayyaa
+    (0x11173, 0x11173,),  # Mahajani Sign Nukta
+    (0x11180, 0x11181,),  # Sharada Sign Candrabindu..Sharada Sign Anusvara
+    (0x111b6, 0x111be,),  # Sharada Vowel Sign U    ..Sharada Vowel Sign O
+    (0x111c0, 0x111c0,),  # Sharada Sign Virama
+    (0x111c9, 0x111cc,),  # Sharada Sandhi Mark     ..Sharada Extra Short Vowe
+    (0x111cf, 0x111cf,),  # Sharada Sign Inverted Candrabindu
+    (0x1122f, 0x11231,),  # Khojki Vowel Sign U     ..Khojki Vowel Sign Ai
+    (0x11234, 0x11237,),  # Khojki Sign Anusvara    ..Khojki Sign Shadda
+    (0x1123e, 0x1123e,),  # Khojki Sign Sukun
+    (0x11241, 0x11241,),  # Khojki Vowel Sign Vocalic R
+    (0x112df, 0x112df,),  # Khudawadi Sign Anusvara
+    (0x112e3, 0x112ea,),  # Khudawadi Vowel Sign U  ..Khudawadi Sign Virama
+    (0x11300, 0x11301,),  # Grantha Sign Combining A..Grantha Sign Candrabindu
+    (0x1133b, 0x1133c,),  # Combining Bindu Below   ..Grantha Sign Nukta
+    (0x1133e, 0x1133e,),  # Grantha Vowel Sign Aa
+    (0x11340, 0x11340,),  # Grantha Vowel Sign Ii
+    (0x1134d, 0x1134d,),  # Grantha Sign Virama
+    (0x11357, 0x11357,),  # Grantha Au Length Mark
+    (0x11366, 0x1136c,),  # Combining Grantha Digit ..Combining Grantha Digit
+    (0x11370, 0x11374,),  # Combining Grantha Letter..Combining Grantha Letter
+    (0x113b8, 0x113b8,),  # Tulu-tigalari Vowel Sign Aa
+    (0x113bb, 0x113c0,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Vowel Sign
+    (0x113c2, 0x113c2,),  # Tulu-tigalari Vowel Sign Ee
+    (0x113c5, 0x113c5,),  # Tulu-tigalari Vowel Sign Ai
+    (0x113c7, 0x113c9,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Au Length
+    (0x113ce, 0x113d0,),  # Tulu-tigalari Sign Viram..Tulu-tigalari Conjoiner
+    (0x113d2, 0x113d2,),  # Tulu-tigalari Gemination Mark
+    (0x113e1, 0x113e2,),  # Tulu-tigalari Vedic Tone..Tulu-tigalari Vedic Tone
+    (0x11438, 0x1143f,),  # Newa Vowel Sign U       ..Newa Vowel Sign Ai
+    (0x11442, 0x11444,),  # Newa Sign Virama        ..Newa Sign Anusvara
+    (0x11446, 0x11446,),  # Newa Sign Nukta
+    (0x1145e, 0x1145e,),  # Newa Sandhi Mark
+    (0x114b0, 0x114b0,),  # Tirhuta Vowel Sign Aa
+    (0x114b3, 0x114b8,),  # Tirhuta Vowel Sign U    ..Tirhuta Vowel Sign Vocal
+    (0x114ba, 0x114ba,),  # Tirhuta Vowel Sign Short E
+    (0x114bd, 0x114bd,),  # Tirhuta Vowel Sign Short O
+    (0x114bf, 0x114c0,),  # Tirhuta Sign Candrabindu..Tirhuta Sign Anusvara
+    (0x114c2, 0x114c3,),  # Tirhuta Sign Virama     ..Tirhuta Sign Nukta
+    (0x115af, 0x115af,),  # Siddham Vowel Sign Aa
+    (0x115b2, 0x115b5,),  # Siddham Vowel Sign U    ..Siddham Vowel Sign Vocal
+    (0x115bc, 0x115bd,),  # Siddham Sign Candrabindu..Siddham Sign Anusvara
+    (0x115bf, 0x115c0,),  # Siddham Sign Virama     ..Siddham Sign Nukta
+    (0x115dc, 0x115dd,),  # Siddham Vowel Sign Alter..Siddham Vowel Sign Alter
+    (0x11633, 0x1163a,),  # Modi Vowel Sign U       ..Modi Vowel Sign Ai
+    (0x1163d, 0x1163d,),  # Modi Sign Anusvara
+    (0x1163f, 0x11640,),  # Modi Sign Virama        ..Modi Sign Ardhacandra
+    (0x116ab, 0x116ab,),  # Takri Sign Anusvara
+    (0x116ad, 0x116ad,),  # Takri Vowel Sign Aa
+    (0x116b0, 0x116b7,),  # Takri Vowel Sign U      ..Takri Sign Nukta
+    (0x1171d, 0x1171d,),  # Ahom Consonant Sign Medial La
+    (0x1171f, 0x1171f,),  # Ahom Consonant Sign Medial Ligating Ra
+    (0x11722, 0x11725,),  # Ahom Vowel Sign I       ..Ahom Vowel Sign Uu
+    (0x11727, 0x1172b,),  # Ahom Vowel Sign Aw      ..Ahom Sign Killer
+    (0x1182f, 0x11837,),  # Dogra Vowel Sign U      ..Dogra Sign Anusvara
+    (0x11839, 0x1183a,),  # Dogra Sign Virama       ..Dogra Sign Nukta
+    (0x11930, 0x11930,),  # Dives Akuru Vowel Sign Aa
+    (0x1193b, 0x1193e,),  # Dives Akuru Sign Anusvar..Dives Akuru Virama
+    (0x11943, 0x11943,),  # Dives Akuru Sign Nukta
+    (0x119d4, 0x119d7,),  # Nandinagari Vowel Sign U..Nandinagari Vowel Sign V
+    (0x119da, 0x119db,),  # Nandinagari Vowel Sign E..Nandinagari Vowel Sign A
+    (0x119e0, 0x119e0,),  # Nandinagari Sign Virama
+    (0x11a01, 0x11a0a,),  # Zanabazar Square Vowel S..Zanabazar Square Vowel L
+    (0x11a33, 0x11a38,),  # Zanabazar Square Final C..Zanabazar Square Sign An
+    (0x11a3b, 0x11a3e,),  # Zanabazar Square Cluster..Zanabazar Square Cluster
+    (0x11a47, 0x11a47,),  # Zanabazar Square Subjoiner
+    (0x11a51, 0x11a56,),  # Soyombo Vowel Sign I    ..Soyombo Vowel Sign Oe
+    (0x11a59, 0x11a5b,),  # Soyombo Vowel Sign Vocal..Soyombo Vowel Length Mar
+    (0x11a8a, 0x11a96,),  # Soyombo Final Consonant ..Soyombo Sign Anusvara
+    (0x11a98, 0x11a99,),  # Soyombo Gemination Mark ..Soyombo Subjoiner
+    (0x11b60, 0x11b60,),  # Sharada Vowel Sign Oe
+    (0x11b62, 0x11b64,),  # Sharada Vowel Sign Ue   ..Sharada Vowel Sign Short
+    (0x11b66, 0x11b66,),  # Sharada Vowel Sign Candra E
+    (0x11c30, 0x11c36,),  # Bhaiksuki Vowel Sign I  ..Bhaiksuki Vowel Sign Voc
+    (0x11c38, 0x11c3d,),  # Bhaiksuki Vowel Sign E  ..Bhaiksuki Sign Anusvara
+    (0x11c3f, 0x11c3f,),  # Bhaiksuki Sign Virama
+    (0x11c92, 0x11ca7,),  # Marchen Subjoined Letter..Marchen Subjoined Letter
+    (0x11caa, 0x11cb0,),  # Marchen Subjoined Letter..Marchen Vowel Sign Aa
+    (0x11cb2, 0x11cb3,),  # Marchen Vowel Sign U    ..Marchen Vowel Sign E
+    (0x11cb5, 0x11cb6,),  # Marchen Sign Anusvara   ..Marchen Sign Candrabindu
+    (0x11d31, 0x11d36,),  # Masaram Gondi Vowel Sign..Masaram Gondi Vowel Sign
+    (0x11d3a, 0x11d3a,),  # Masaram Gondi Vowel Sign E
+    (0x11d3c, 0x11d3d,),  # Masaram Gondi Vowel Sign..Masaram Gondi Vowel Sign
+    (0x11d3f, 0x11d45,),  # Masaram Gondi Vowel Sign..Masaram Gondi Virama
+    (0x11d47, 0x11d47,),  # Masaram Gondi Ra-kara
+    (0x11d90, 0x11d91,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+    (0x11d95, 0x11d95,),  # Gunjala Gondi Sign Anusvara
+    (0x11d97, 0x11d97,),  # Gunjala Gondi Virama
+    (0x11ef3, 0x11ef4,),  # Makasar Vowel Sign I    ..Makasar Vowel Sign U
+    (0x11f00, 0x11f01,),  # Kawi Sign Candrabindu   ..Kawi Sign Anusvara
+    (0x11f36, 0x11f3a,),  # Kawi Vowel Sign I       ..Kawi Vowel Sign Vocalic
+    (0x11f40, 0x11f42,),  # Kawi Vowel Sign Eu      ..Kawi Conjoiner
+    (0x11f5a, 0x11f5a,),  # Kawi Sign Nukta
+    (0x13440, 0x13440,),  # Egyptian Hieroglyph Mirror Horizontally
+    (0x13447, 0x13455,),  # Egyptian Hieroglyph Modi..Egyptian Hieroglyph Modi
+    (0x1611e, 0x16129,),  # Gurung Khema Vowel Sign ..Gurung Khema Vowel Lengt
+    (0x1612d, 0x1612f,),  # Gurung Khema Sign Anusva..Gurung Khema Sign Tholho
+    (0x16af0, 0x16af4,),  # Bassa Vah Combining High..Bassa Vah Combining High
+    (0x16b30, 0x16b36,),  # Pahawh Hmong Mark Cim Tu..Pahawh Hmong Mark Cim Ta
+    (0x16f4f, 0x16f4f,),  # Miao Sign Consonant Modifier Bar
+    (0x16f8f, 0x16f92,),  # Miao Tone Right         ..Miao Tone Below
+    (0x16fe4, 0x16fe4,),  # Khitan Small Script Filler
+    (0x16ff0, 0x16ff1,),  # Vietnamese Alternate Rea..Vietnamese Alternate Rea
+    (0x1bc9d, 0x1bc9e,),  # Duployan Thick Letter Se..Duployan Double Mark
+    (0x1cf00, 0x1cf2d,),  # Znamenny Combining Mark ..Znamenny Combining Mark
+    (0x1cf30, 0x1cf46,),  # Znamenny Combining Tonal..Znamenny Priznak Modifie
+    (0x1d165, 0x1d169,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d16d, 0x1d172,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d17b, 0x1d182,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d185, 0x1d18b,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d1aa, 0x1d1ad,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d242, 0x1d244,),  # Combining Greek Musical ..Combining Greek Musical
+    (0x1da00, 0x1da36,),  # Signwriting Head Rim    ..Signwriting Air Sucking
+    (0x1da3b, 0x1da6c,),  # Signwriting Mouth Closed..Signwriting Excitement
+    (0x1da75, 0x1da75,),  # Signwriting Upper Body Tilting From Hip Joints
+    (0x1da84, 0x1da84,),  # Signwriting Location Head Neck
+    (0x1da9b, 0x1da9f,),  # Signwriting Fill Modifie..Signwriting Fill Modifie
+    (0x1daa1, 0x1daaf,),  # Signwriting Rotation Mod..Signwriting Rotation Mod
+    (0x1e000, 0x1e006,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e008, 0x1e018,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e01b, 0x1e021,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e023, 0x1e024,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e026, 0x1e02a,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e08f, 0x1e08f,),  # Combining Cyrillic Small Letter Byelorussian-ukr
+    (0x1e130, 0x1e136,),  # Nyiakeng Puachue Hmong T..Nyiakeng Puachue Hmong T
+    (0x1e2ae, 0x1e2ae,),  # Toto Sign Rising Tone
+    (0x1e2ec, 0x1e2ef,),  # Wancho Tone Tup         ..Wancho Tone Koini
+    (0x1e4ec, 0x1e4ef,),  # Nag Mundari Sign Muhor  ..Nag Mundari Sign Sutuh
+    (0x1e5ee, 0x1e5ef,),  # Ol Onal Sign Mu         ..Ol Onal Sign Ikir
+    (0x1e6e3, 0x1e6e3,),  # Tai Yo Sign Ue
+    (0x1e6e6, 0x1e6e6,),  # Tai Yo Sign Au
+    (0x1e6ee, 0x1e6ef,),  # Tai Yo Sign Ay          ..Tai Yo Sign Ang
+    (0x1e6f5, 0x1e6f5,),  # Tai Yo Sign Om
+    (0x1e8d0, 0x1e8d6,),  # Mende Kikakui Combining ..Mende Kikakui Combining
+    (0x1e944, 0x1e94a,),  # Adlam Alif Lengthener   ..Adlam Nukta
+    (0x1f3fb, 0x1f3ff,),  # Emoji Modifier Fitzpatri..Emoji Modifier Fitzpatri
+    (0xe0020, 0xe007f,),  # Tag Space               ..Cancel Tag
+    (0xe0100, 0xe01ef,),  # Variation Selector-17   ..Variation Selector-256
+)
+
+GRAPHEME_ZWJ = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x0200d, 0x0200d,),  # Zero Width Joiner
+)
+
+GRAPHEME_REGIONAL_INDICATOR = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x1f1e6, 0x1f1ff,),  # Regional Indicator Symbo..Regional Indicator Symbo
+)
+
+GRAPHEME_PREPEND = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x00600, 0x00605,),  # Arabic Number Sign      ..Arabic Number Mark Above
+    (0x006dd, 0x006dd,),  # Arabic End Of Ayah
+    (0x0070f, 0x0070f,),  # Syriac Abbreviation Mark
+    (0x00890, 0x00891,),  # Arabic Pound Mark Above ..Arabic Piastre Mark Abov
+    (0x008e2, 0x008e2,),  # Arabic Disputed End Of Ayah
+    (0x00d4e, 0x00d4e,),  # Malayalam Letter Dot Reph
+    (0x110bd, 0x110bd,),  # Kaithi Number Sign
+    (0x110cd, 0x110cd,),  # Kaithi Number Sign Above
+    (0x111c2, 0x111c3,),  # Sharada Sign Jihvamuliya..Sharada Sign Upadhmaniya
+    (0x113d1, 0x113d1,),  # Tulu-tigalari Repha
+    (0x1193f, 0x1193f,),  # Dives Akuru Prefixed Nasal Sign
+    (0x11941, 0x11941,),  # Dives Akuru Initial Ra
+    (0x11a84, 0x11a89,),  # Soyombo Sign Jihvamuliya..Soyombo Cluster-initial
+    (0x11d46, 0x11d46,),  # Masaram Gondi Repha
+    (0x11f02, 0x11f02,),  # Kawi Sign Repha
+)
+
+GRAPHEME_SPACINGMARK = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x00903, 0x00903,),  # Devanagari Sign Visarga
+    (0x0093b, 0x0093b,),  # Devanagari Vowel Sign Ooe
+    (0x0093e, 0x00940,),  # Devanagari Vowel Sign Aa..Devanagari Vowel Sign Ii
+    (0x00949, 0x0094c,),  # Devanagari Vowel Sign Ca..Devanagari Vowel Sign Au
+    (0x0094e, 0x0094f,),  # Devanagari Vowel Sign Pr..Devanagari Vowel Sign Aw
+    (0x00982, 0x00983,),  # Bengali Sign Anusvara   ..Bengali Sign Visarga
+    (0x009bf, 0x009c0,),  # Bengali Vowel Sign I    ..Bengali Vowel Sign Ii
+    (0x009c7, 0x009c8,),  # Bengali Vowel Sign E    ..Bengali Vowel Sign Ai
+    (0x009cb, 0x009cc,),  # Bengali Vowel Sign O    ..Bengali Vowel Sign Au
+    (0x00a03, 0x00a03,),  # Gurmukhi Sign Visarga
+    (0x00a3e, 0x00a40,),  # Gurmukhi Vowel Sign Aa  ..Gurmukhi Vowel Sign Ii
+    (0x00a83, 0x00a83,),  # Gujarati Sign Visarga
+    (0x00abe, 0x00ac0,),  # Gujarati Vowel Sign Aa  ..Gujarati Vowel Sign Ii
+    (0x00ac9, 0x00ac9,),  # Gujarati Vowel Sign Candra O
+    (0x00acb, 0x00acc,),  # Gujarati Vowel Sign O   ..Gujarati Vowel Sign Au
+    (0x00b02, 0x00b03,),  # Oriya Sign Anusvara     ..Oriya Sign Visarga
+    (0x00b40, 0x00b40,),  # Oriya Vowel Sign Ii
+    (0x00b47, 0x00b48,),  # Oriya Vowel Sign E      ..Oriya Vowel Sign Ai
+    (0x00b4b, 0x00b4c,),  # Oriya Vowel Sign O      ..Oriya Vowel Sign Au
+    (0x00bbf, 0x00bbf,),  # Tamil Vowel Sign I
+    (0x00bc1, 0x00bc2,),  # Tamil Vowel Sign U      ..Tamil Vowel Sign Uu
+    (0x00bc6, 0x00bc8,),  # Tamil Vowel Sign E      ..Tamil Vowel Sign Ai
+    (0x00bca, 0x00bcc,),  # Tamil Vowel Sign O      ..Tamil Vowel Sign Au
+    (0x00c01, 0x00c03,),  # Telugu Sign Candrabindu ..Telugu Sign Visarga
+    (0x00c41, 0x00c44,),  # Telugu Vowel Sign U     ..Telugu Vowel Sign Vocali
+    (0x00c82, 0x00c83,),  # Kannada Sign Anusvara   ..Kannada Sign Visarga
+    (0x00cbe, 0x00cbe,),  # Kannada Vowel Sign Aa
+    (0x00cc1, 0x00cc1,),  # Kannada Vowel Sign U
+    (0x00cc3, 0x00cc4,),  # Kannada Vowel Sign Vocal..Kannada Vowel Sign Vocal
+    (0x00cf3, 0x00cf3,),  # Kannada Sign Combining Anusvara Above Right
+    (0x00d02, 0x00d03,),  # Malayalam Sign Anusvara ..Malayalam Sign Visarga
+    (0x00d3f, 0x00d40,),  # Malayalam Vowel Sign I  ..Malayalam Vowel Sign Ii
+    (0x00d46, 0x00d48,),  # Malayalam Vowel Sign E  ..Malayalam Vowel Sign Ai
+    (0x00d4a, 0x00d4c,),  # Malayalam Vowel Sign O  ..Malayalam Vowel Sign Au
+    (0x00d82, 0x00d83,),  # Sinhala Sign Anusvaraya ..Sinhala Sign Visargaya
+    (0x00dd0, 0x00dd1,),  # Sinhala Vowel Sign Ketti..Sinhala Vowel Sign Diga
+    (0x00dd8, 0x00dde,),  # Sinhala Vowel Sign Gaett..Sinhala Vowel Sign Kombu
+    (0x00df2, 0x00df3,),  # Sinhala Vowel Sign Diga ..Sinhala Vowel Sign Diga
+    (0x00e33, 0x00e33,),  # Thai Character Sara Am
+    (0x00eb3, 0x00eb3,),  # Lao Vowel Sign Am
+    (0x00f3e, 0x00f3f,),  # Tibetan Sign Yar Tshes  ..Tibetan Sign Mar Tshes
+    (0x00f7f, 0x00f7f,),  # Tibetan Sign Rnam Bcad
+    (0x01031, 0x01031,),  # Myanmar Vowel Sign E
+    (0x0103b, 0x0103c,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+    (0x01056, 0x01057,),  # Myanmar Vowel Sign Vocal..Myanmar Vowel Sign Vocal
+    (0x01084, 0x01084,),  # Myanmar Vowel Sign Shan E
+    (0x017b6, 0x017b6,),  # Khmer Vowel Sign Aa
+    (0x017be, 0x017c5,),  # Khmer Vowel Sign Oe     ..Khmer Vowel Sign Au
+    (0x017c7, 0x017c8,),  # Khmer Sign Reahmuk      ..Khmer Sign Yuukaleapintu
+    (0x01923, 0x01926,),  # Limbu Vowel Sign Ee     ..Limbu Vowel Sign Au
+    (0x01929, 0x0192b,),  # Limbu Subjoined Letter Y..Limbu Subjoined Letter W
+    (0x01930, 0x01931,),  # Limbu Small Letter Ka   ..Limbu Small Letter Nga
+    (0x01933, 0x01938,),  # Limbu Small Letter Ta   ..Limbu Small Letter La
+    (0x01a19, 0x01a1a,),  # Buginese Vowel Sign E   ..Buginese Vowel Sign O
+    (0x01a55, 0x01a55,),  # Tai Tham Consonant Sign Medial Ra
+    (0x01a57, 0x01a57,),  # Tai Tham Consonant Sign La Tang Lai
+    (0x01a6d, 0x01a72,),  # Tai Tham Vowel Sign Oy  ..Tai Tham Vowel Sign Tham
+    (0x01b04, 0x01b04,),  # Balinese Sign Bisah
+    (0x01b3e, 0x01b41,),  # Balinese Vowel Sign Tali..Balinese Vowel Sign Tali
+    (0x01b82, 0x01b82,),  # Sundanese Sign Pangwisad
+    (0x01ba1, 0x01ba1,),  # Sundanese Consonant Sign Pamingkal
+    (0x01ba6, 0x01ba7,),  # Sundanese Vowel Sign Pan..Sundanese Vowel Sign Pan
+    (0x01be7, 0x01be7,),  # Batak Vowel Sign E
+    (0x01bea, 0x01bec,),  # Batak Vowel Sign I      ..Batak Vowel Sign O
+    (0x01bee, 0x01bee,),  # Batak Vowel Sign U
+    (0x01c24, 0x01c2b,),  # Lepcha Subjoined Letter ..Lepcha Vowel Sign Uu
+    (0x01c34, 0x01c35,),  # Lepcha Consonant Sign Ny..Lepcha Consonant Sign Ka
+    (0x01ce1, 0x01ce1,),  # Vedic Tone Atharvavedic Independent Svarita
+    (0x01cf7, 0x01cf7,),  # Vedic Sign Atikrama
+    (0x0a823, 0x0a824,),  # Syloti Nagri Vowel Sign ..Syloti Nagri Vowel Sign
+    (0x0a827, 0x0a827,),  # Syloti Nagri Vowel Sign Oo
+    (0x0a880, 0x0a881,),  # Saurashtra Sign Anusvara..Saurashtra Sign Visarga
+    (0x0a8b4, 0x0a8c3,),  # Saurashtra Consonant Sig..Saurashtra Vowel Sign Au
+    (0x0a952, 0x0a952,),  # Rejang Consonant Sign H
+    (0x0a983, 0x0a983,),  # Javanese Sign Wignyan
+    (0x0a9b4, 0x0a9b5,),  # Javanese Vowel Sign Taru..Javanese Vowel Sign Tolo
+    (0x0a9ba, 0x0a9bb,),  # Javanese Vowel Sign Tali..Javanese Vowel Sign Dirg
+    (0x0a9be, 0x0a9bf,),  # Javanese Consonant Sign ..Javanese Consonant Sign
+    (0x0aa2f, 0x0aa30,),  # Cham Vowel Sign O       ..Cham Vowel Sign Ai
+    (0x0aa33, 0x0aa34,),  # Cham Consonant Sign Ya  ..Cham Consonant Sign Ra
+    (0x0aa4d, 0x0aa4d,),  # Cham Consonant Sign Final H
+    (0x0aaeb, 0x0aaeb,),  # Meetei Mayek Vowel Sign Ii
+    (0x0aaee, 0x0aaef,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+    (0x0aaf5, 0x0aaf5,),  # Meetei Mayek Vowel Sign Visarga
+    (0x0abe3, 0x0abe4,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+    (0x0abe6, 0x0abe7,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+    (0x0abe9, 0x0abea,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+    (0x0abec, 0x0abec,),  # Meetei Mayek Lum Iyek
+    (0x11000, 0x11000,),  # Brahmi Sign Candrabindu
+    (0x11002, 0x11002,),  # Brahmi Sign Visarga
+    (0x11082, 0x11082,),  # Kaithi Sign Visarga
+    (0x110b0, 0x110b2,),  # Kaithi Vowel Sign Aa    ..Kaithi Vowel Sign Ii
+    (0x110b7, 0x110b8,),  # Kaithi Vowel Sign O     ..Kaithi Vowel Sign Au
+    (0x1112c, 0x1112c,),  # Chakma Vowel Sign E
+    (0x11145, 0x11146,),  # Chakma Vowel Sign Aa    ..Chakma Vowel Sign Ei
+    (0x11182, 0x11182,),  # Sharada Sign Visarga
+    (0x111b3, 0x111b5,),  # Sharada Vowel Sign Aa   ..Sharada Vowel Sign Ii
+    (0x111bf, 0x111bf,),  # Sharada Vowel Sign Au
+    (0x111ce, 0x111ce,),  # Sharada Vowel Sign Prishthamatra E
+    (0x1122c, 0x1122e,),  # Khojki Vowel Sign Aa    ..Khojki Vowel Sign Ii
+    (0x11232, 0x11233,),  # Khojki Vowel Sign O     ..Khojki Vowel Sign Au
+    (0x112e0, 0x112e2,),  # Khudawadi Vowel Sign Aa ..Khudawadi Vowel Sign Ii
+    (0x11302, 0x11303,),  # Grantha Sign Anusvara   ..Grantha Sign Visarga
+    (0x1133f, 0x1133f,),  # Grantha Vowel Sign I
+    (0x11341, 0x11344,),  # Grantha Vowel Sign U    ..Grantha Vowel Sign Vocal
+    (0x11347, 0x11348,),  # Grantha Vowel Sign Ee   ..Grantha Vowel Sign Ai
+    (0x1134b, 0x1134c,),  # Grantha Vowel Sign Oo   ..Grantha Vowel Sign Au
+    (0x11362, 0x11363,),  # Grantha Vowel Sign Vocal..Grantha Vowel Sign Vocal
+    (0x113b9, 0x113ba,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Vowel Sign
+    (0x113ca, 0x113ca,),  # Tulu-tigalari Sign Candra Anunasika
+    (0x113cc, 0x113cd,),  # Tulu-tigalari Sign Anusv..Tulu-tigalari Sign Visar
+    (0x11435, 0x11437,),  # Newa Vowel Sign Aa      ..Newa Vowel Sign Ii
+    (0x11440, 0x11441,),  # Newa Vowel Sign O       ..Newa Vowel Sign Au
+    (0x11445, 0x11445,),  # Newa Sign Visarga
+    (0x114b1, 0x114b2,),  # Tirhuta Vowel Sign I    ..Tirhuta Vowel Sign Ii
+    (0x114b9, 0x114b9,),  # Tirhuta Vowel Sign E
+    (0x114bb, 0x114bc,),  # Tirhuta Vowel Sign Ai   ..Tirhuta Vowel Sign O
+    (0x114be, 0x114be,),  # Tirhuta Vowel Sign Au
+    (0x114c1, 0x114c1,),  # Tirhuta Sign Visarga
+    (0x115b0, 0x115b1,),  # Siddham Vowel Sign I    ..Siddham Vowel Sign Ii
+    (0x115b8, 0x115bb,),  # Siddham Vowel Sign E    ..Siddham Vowel Sign Au
+    (0x115be, 0x115be,),  # Siddham Sign Visarga
+    (0x11630, 0x11632,),  # Modi Vowel Sign Aa      ..Modi Vowel Sign Ii
+    (0x1163b, 0x1163c,),  # Modi Vowel Sign O       ..Modi Vowel Sign Au
+    (0x1163e, 0x1163e,),  # Modi Sign Visarga
+    (0x116ac, 0x116ac,),  # Takri Sign Visarga
+    (0x116ae, 0x116af,),  # Takri Vowel Sign I      ..Takri Vowel Sign Ii
+    (0x1171e, 0x1171e,),  # Ahom Consonant Sign Medial Ra
+    (0x11726, 0x11726,),  # Ahom Vowel Sign E
+    (0x1182c, 0x1182e,),  # Dogra Vowel Sign Aa     ..Dogra Vowel Sign Ii
+    (0x11838, 0x11838,),  # Dogra Sign Visarga
+    (0x11931, 0x11935,),  # Dives Akuru Vowel Sign I..Dives Akuru Vowel Sign E
+    (0x11937, 0x11938,),  # Dives Akuru Vowel Sign A..Dives Akuru Vowel Sign O
+    (0x11940, 0x11940,),  # Dives Akuru Medial Ya
+    (0x11942, 0x11942,),  # Dives Akuru Medial Ra
+    (0x119d1, 0x119d3,),  # Nandinagari Vowel Sign A..Nandinagari Vowel Sign I
+    (0x119dc, 0x119df,),  # Nandinagari Vowel Sign O..Nandinagari Sign Visarga
+    (0x119e4, 0x119e4,),  # Nandinagari Vowel Sign Prishthamatra E
+    (0x11a39, 0x11a39,),  # Zanabazar Square Sign Visarga
+    (0x11a57, 0x11a58,),  # Soyombo Vowel Sign Ai   ..Soyombo Vowel Sign Au
+    (0x11a97, 0x11a97,),  # Soyombo Sign Visarga
+    (0x11b61, 0x11b61,),  # Sharada Vowel Sign Ooe
+    (0x11b65, 0x11b65,),  # Sharada Vowel Sign Short O
+    (0x11b67, 0x11b67,),  # Sharada Vowel Sign Candra O
+    (0x11c2f, 0x11c2f,),  # Bhaiksuki Vowel Sign Aa
+    (0x11c3e, 0x11c3e,),  # Bhaiksuki Sign Visarga
+    (0x11ca9, 0x11ca9,),  # Marchen Subjoined Letter Ya
+    (0x11cb1, 0x11cb1,),  # Marchen Vowel Sign I
+    (0x11cb4, 0x11cb4,),  # Marchen Vowel Sign O
+    (0x11d8a, 0x11d8e,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+    (0x11d93, 0x11d94,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+    (0x11d96, 0x11d96,),  # Gunjala Gondi Sign Visarga
+    (0x11ef5, 0x11ef6,),  # Makasar Vowel Sign E    ..Makasar Vowel Sign O
+    (0x11f03, 0x11f03,),  # Kawi Sign Visarga
+    (0x11f34, 0x11f35,),  # Kawi Vowel Sign Aa      ..Kawi Vowel Sign Alternat
+    (0x11f3e, 0x11f3f,),  # Kawi Vowel Sign E       ..Kawi Vowel Sign Ai
+    (0x1612a, 0x1612c,),  # Gurung Khema Consonant S..Gurung Khema Consonant S
+    (0x16f51, 0x16f87,),  # Miao Sign Aspiration    ..Miao Vowel Sign Ui
+)
+
+GRAPHEME_L = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x01100, 0x0115f,),  # Hangul Choseong Kiyeok  ..Hangul Choseong Filler
+    (0x0a960, 0x0a97c,),  # Hangul Choseong Tikeut-m..Hangul Choseong Ssangyeo
+)
+
+GRAPHEME_V = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x01160, 0x011a7,),  # Hangul Jungseong Filler ..Hangul Jungseong O-yae
+    (0x0d7b0, 0x0d7c6,),  # Hangul Jungseong O-yeo  ..Hangul Jungseong Araea-e
+    (0x16d63, 0x16d63,),  # Kirat Rai Vowel Sign Aa
+    (0x16d67, 0x16d6a,),  # Kirat Rai Vowel Sign E  ..Kirat Rai Vowel Sign Au
+)
+
+GRAPHEME_T = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x011a8, 0x011ff,),  # Hangul Jongseong Kiyeok ..Hangul Jongseong Ssangni
+    (0x0d7cb, 0x0d7fb,),  # Hangul Jongseong Nieun-r..Hangul Jongseong Phieuph
+)
+
+GRAPHEME_LV = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x0ac00, 0x0ac00,),  # Hangul Syllable Ga
+    (0x0ac1c, 0x0ac1c,),  # Hangul Syllable Gae
+    (0x0ac38, 0x0ac38,),  # Hangul Syllable Gya
+    (0x0ac54, 0x0ac54,),  # Hangul Syllable Gyae
+    (0x0ac70, 0x0ac70,),  # Hangul Syllable Geo
+    (0x0ac8c, 0x0ac8c,),  # Hangul Syllable Ge
+    (0x0aca8, 0x0aca8,),  # Hangul Syllable Gyeo
+    (0x0acc4, 0x0acc4,),  # Hangul Syllable Gye
+    (0x0ace0, 0x0ace0,),  # Hangul Syllable Go
+    (0x0acfc, 0x0acfc,),  # Hangul Syllable Gwa
+    (0x0ad18, 0x0ad18,),  # Hangul Syllable Gwae
+    (0x0ad34, 0x0ad34,),  # Hangul Syllable Goe
+    (0x0ad50, 0x0ad50,),  # Hangul Syllable Gyo
+    (0x0ad6c, 0x0ad6c,),  # Hangul Syllable Gu
+    (0x0ad88, 0x0ad88,),  # Hangul Syllable Gweo
+    (0x0ada4, 0x0ada4,),  # Hangul Syllable Gwe
+    (0x0adc0, 0x0adc0,),  # Hangul Syllable Gwi
+    (0x0addc, 0x0addc,),  # Hangul Syllable Gyu
+    (0x0adf8, 0x0adf8,),  # Hangul Syllable Geu
+    (0x0ae14, 0x0ae14,),  # Hangul Syllable Gyi
+    (0x0ae30, 0x0ae30,),  # Hangul Syllable Gi
+    (0x0ae4c, 0x0ae4c,),  # Hangul Syllable Gga
+    (0x0ae68, 0x0ae68,),  # Hangul Syllable Ggae
+    (0x0ae84, 0x0ae84,),  # Hangul Syllable Ggya
+    (0x0aea0, 0x0aea0,),  # Hangul Syllable Ggyae
+    (0x0aebc, 0x0aebc,),  # Hangul Syllable Ggeo
+    (0x0aed8, 0x0aed8,),  # Hangul Syllable Gge
+    (0x0aef4, 0x0aef4,),  # Hangul Syllable Ggyeo
+    (0x0af10, 0x0af10,),  # Hangul Syllable Ggye
+    (0x0af2c, 0x0af2c,),  # Hangul Syllable Ggo
+    (0x0af48, 0x0af48,),  # Hangul Syllable Ggwa
+    (0x0af64, 0x0af64,),  # Hangul Syllable Ggwae
+    (0x0af80, 0x0af80,),  # Hangul Syllable Ggoe
+    (0x0af9c, 0x0af9c,),  # Hangul Syllable Ggyo
+    (0x0afb8, 0x0afb8,),  # Hangul Syllable Ggu
+    (0x0afd4, 0x0afd4,),  # Hangul Syllable Ggweo
+    (0x0aff0, 0x0aff0,),  # Hangul Syllable Ggwe
+    (0x0b00c, 0x0b00c,),  # Hangul Syllable Ggwi
+    (0x0b028, 0x0b028,),  # Hangul Syllable Ggyu
+    (0x0b044, 0x0b044,),  # Hangul Syllable Ggeu
+    (0x0b060, 0x0b060,),  # Hangul Syllable Ggyi
+    (0x0b07c, 0x0b07c,),  # Hangul Syllable Ggi
+    (0x0b098, 0x0b098,),  # Hangul Syllable Na
+    (0x0b0b4, 0x0b0b4,),  # Hangul Syllable Nae
+    (0x0b0d0, 0x0b0d0,),  # Hangul Syllable Nya
+    (0x0b0ec, 0x0b0ec,),  # Hangul Syllable Nyae
+    (0x0b108, 0x0b108,),  # Hangul Syllable Neo
+    (0x0b124, 0x0b124,),  # Hangul Syllable Ne
+    (0x0b140, 0x0b140,),  # Hangul Syllable Nyeo
+    (0x0b15c, 0x0b15c,),  # Hangul Syllable Nye
+    (0x0b178, 0x0b178,),  # Hangul Syllable No
+    (0x0b194, 0x0b194,),  # Hangul Syllable Nwa
+    (0x0b1b0, 0x0b1b0,),  # Hangul Syllable Nwae
+    (0x0b1cc, 0x0b1cc,),  # Hangul Syllable Noe
+    (0x0b1e8, 0x0b1e8,),  # Hangul Syllable Nyo
+    (0x0b204, 0x0b204,),  # Hangul Syllable Nu
+    (0x0b220, 0x0b220,),  # Hangul Syllable Nweo
+    (0x0b23c, 0x0b23c,),  # Hangul Syllable Nwe
+    (0x0b258, 0x0b258,),  # Hangul Syllable Nwi
+    (0x0b274, 0x0b274,),  # Hangul Syllable Nyu
+    (0x0b290, 0x0b290,),  # Hangul Syllable Neu
+    (0x0b2ac, 0x0b2ac,),  # Hangul Syllable Nyi
+    (0x0b2c8, 0x0b2c8,),  # Hangul Syllable Ni
+    (0x0b2e4, 0x0b2e4,),  # Hangul Syllable Da
+    (0x0b300, 0x0b300,),  # Hangul Syllable Dae
+    (0x0b31c, 0x0b31c,),  # Hangul Syllable Dya
+    (0x0b338, 0x0b338,),  # Hangul Syllable Dyae
+    (0x0b354, 0x0b354,),  # Hangul Syllable Deo
+    (0x0b370, 0x0b370,),  # Hangul Syllable De
+    (0x0b38c, 0x0b38c,),  # Hangul Syllable Dyeo
+    (0x0b3a8, 0x0b3a8,),  # Hangul Syllable Dye
+    (0x0b3c4, 0x0b3c4,),  # Hangul Syllable Do
+    (0x0b3e0, 0x0b3e0,),  # Hangul Syllable Dwa
+    (0x0b3fc, 0x0b3fc,),  # Hangul Syllable Dwae
+    (0x0b418, 0x0b418,),  # Hangul Syllable Doe
+    (0x0b434, 0x0b434,),  # Hangul Syllable Dyo
+    (0x0b450, 0x0b450,),  # Hangul Syllable Du
+    (0x0b46c, 0x0b46c,),  # Hangul Syllable Dweo
+    (0x0b488, 0x0b488,),  # Hangul Syllable Dwe
+    (0x0b4a4, 0x0b4a4,),  # Hangul Syllable Dwi
+    (0x0b4c0, 0x0b4c0,),  # Hangul Syllable Dyu
+    (0x0b4dc, 0x0b4dc,),  # Hangul Syllable Deu
+    (0x0b4f8, 0x0b4f8,),  # Hangul Syllable Dyi
+    (0x0b514, 0x0b514,),  # Hangul Syllable Di
+    (0x0b530, 0x0b530,),  # Hangul Syllable Dda
+    (0x0b54c, 0x0b54c,),  # Hangul Syllable Ddae
+    (0x0b568, 0x0b568,),  # Hangul Syllable Ddya
+    (0x0b584, 0x0b584,),  # Hangul Syllable Ddyae
+    (0x0b5a0, 0x0b5a0,),  # Hangul Syllable Ddeo
+    (0x0b5bc, 0x0b5bc,),  # Hangul Syllable Dde
+    (0x0b5d8, 0x0b5d8,),  # Hangul Syllable Ddyeo
+    (0x0b5f4, 0x0b5f4,),  # Hangul Syllable Ddye
+    (0x0b610, 0x0b610,),  # Hangul Syllable Ddo
+    (0x0b62c, 0x0b62c,),  # Hangul Syllable Ddwa
+    (0x0b648, 0x0b648,),  # Hangul Syllable Ddwae
+    (0x0b664, 0x0b664,),  # Hangul Syllable Ddoe
+    (0x0b680, 0x0b680,),  # Hangul Syllable Ddyo
+    (0x0b69c, 0x0b69c,),  # Hangul Syllable Ddu
+    (0x0b6b8, 0x0b6b8,),  # Hangul Syllable Ddweo
+    (0x0b6d4, 0x0b6d4,),  # Hangul Syllable Ddwe
+    (0x0b6f0, 0x0b6f0,),  # Hangul Syllable Ddwi
+    (0x0b70c, 0x0b70c,),  # Hangul Syllable Ddyu
+    (0x0b728, 0x0b728,),  # Hangul Syllable Ddeu
+    (0x0b744, 0x0b744,),  # Hangul Syllable Ddyi
+    (0x0b760, 0x0b760,),  # Hangul Syllable Ddi
+    (0x0b77c, 0x0b77c,),  # Hangul Syllable Ra
+    (0x0b798, 0x0b798,),  # Hangul Syllable Rae
+    (0x0b7b4, 0x0b7b4,),  # Hangul Syllable Rya
+    (0x0b7d0, 0x0b7d0,),  # Hangul Syllable Ryae
+    (0x0b7ec, 0x0b7ec,),  # Hangul Syllable Reo
+    (0x0b808, 0x0b808,),  # Hangul Syllable Re
+    (0x0b824, 0x0b824,),  # Hangul Syllable Ryeo
+    (0x0b840, 0x0b840,),  # Hangul Syllable Rye
+    (0x0b85c, 0x0b85c,),  # Hangul Syllable Ro
+    (0x0b878, 0x0b878,),  # Hangul Syllable Rwa
+    (0x0b894, 0x0b894,),  # Hangul Syllable Rwae
+    (0x0b8b0, 0x0b8b0,),  # Hangul Syllable Roe
+    (0x0b8cc, 0x0b8cc,),  # Hangul Syllable Ryo
+    (0x0b8e8, 0x0b8e8,),  # Hangul Syllable Ru
+    (0x0b904, 0x0b904,),  # Hangul Syllable Rweo
+    (0x0b920, 0x0b920,),  # Hangul Syllable Rwe
+    (0x0b93c, 0x0b93c,),  # Hangul Syllable Rwi
+    (0x0b958, 0x0b958,),  # Hangul Syllable Ryu
+    (0x0b974, 0x0b974,),  # Hangul Syllable Reu
+    (0x0b990, 0x0b990,),  # Hangul Syllable Ryi
+    (0x0b9ac, 0x0b9ac,),  # Hangul Syllable Ri
+    (0x0b9c8, 0x0b9c8,),  # Hangul Syllable Ma
+    (0x0b9e4, 0x0b9e4,),  # Hangul Syllable Mae
+    (0x0ba00, 0x0ba00,),  # Hangul Syllable Mya
+    (0x0ba1c, 0x0ba1c,),  # Hangul Syllable Myae
+    (0x0ba38, 0x0ba38,),  # Hangul Syllable Meo
+    (0x0ba54, 0x0ba54,),  # Hangul Syllable Me
+    (0x0ba70, 0x0ba70,),  # Hangul Syllable Myeo
+    (0x0ba8c, 0x0ba8c,),  # Hangul Syllable Mye
+    (0x0baa8, 0x0baa8,),  # Hangul Syllable Mo
+    (0x0bac4, 0x0bac4,),  # Hangul Syllable Mwa
+    (0x0bae0, 0x0bae0,),  # Hangul Syllable Mwae
+    (0x0bafc, 0x0bafc,),  # Hangul Syllable Moe
+    (0x0bb18, 0x0bb18,),  # Hangul Syllable Myo
+    (0x0bb34, 0x0bb34,),  # Hangul Syllable Mu
+    (0x0bb50, 0x0bb50,),  # Hangul Syllable Mweo
+    (0x0bb6c, 0x0bb6c,),  # Hangul Syllable Mwe
+    (0x0bb88, 0x0bb88,),  # Hangul Syllable Mwi
+    (0x0bba4, 0x0bba4,),  # Hangul Syllable Myu
+    (0x0bbc0, 0x0bbc0,),  # Hangul Syllable Meu
+    (0x0bbdc, 0x0bbdc,),  # Hangul Syllable Myi
+    (0x0bbf8, 0x0bbf8,),  # Hangul Syllable Mi
+    (0x0bc14, 0x0bc14,),  # Hangul Syllable Ba
+    (0x0bc30, 0x0bc30,),  # Hangul Syllable Bae
+    (0x0bc4c, 0x0bc4c,),  # Hangul Syllable Bya
+    (0x0bc68, 0x0bc68,),  # Hangul Syllable Byae
+    (0x0bc84, 0x0bc84,),  # Hangul Syllable Beo
+    (0x0bca0, 0x0bca0,),  # Hangul Syllable Be
+    (0x0bcbc, 0x0bcbc,),  # Hangul Syllable Byeo
+    (0x0bcd8, 0x0bcd8,),  # Hangul Syllable Bye
+    (0x0bcf4, 0x0bcf4,),  # Hangul Syllable Bo
+    (0x0bd10, 0x0bd10,),  # Hangul Syllable Bwa
+    (0x0bd2c, 0x0bd2c,),  # Hangul Syllable Bwae
+    (0x0bd48, 0x0bd48,),  # Hangul Syllable Boe
+    (0x0bd64, 0x0bd64,),  # Hangul Syllable Byo
+    (0x0bd80, 0x0bd80,),  # Hangul Syllable Bu
+    (0x0bd9c, 0x0bd9c,),  # Hangul Syllable Bweo
+    (0x0bdb8, 0x0bdb8,),  # Hangul Syllable Bwe
+    (0x0bdd4, 0x0bdd4,),  # Hangul Syllable Bwi
+    (0x0bdf0, 0x0bdf0,),  # Hangul Syllable Byu
+    (0x0be0c, 0x0be0c,),  # Hangul Syllable Beu
+    (0x0be28, 0x0be28,),  # Hangul Syllable Byi
+    (0x0be44, 0x0be44,),  # Hangul Syllable Bi
+    (0x0be60, 0x0be60,),  # Hangul Syllable Bba
+    (0x0be7c, 0x0be7c,),  # Hangul Syllable Bbae
+    (0x0be98, 0x0be98,),  # Hangul Syllable Bbya
+    (0x0beb4, 0x0beb4,),  # Hangul Syllable Bbyae
+    (0x0bed0, 0x0bed0,),  # Hangul Syllable Bbeo
+    (0x0beec, 0x0beec,),  # Hangul Syllable Bbe
+    (0x0bf08, 0x0bf08,),  # Hangul Syllable Bbyeo
+    (0x0bf24, 0x0bf24,),  # Hangul Syllable Bbye
+    (0x0bf40, 0x0bf40,),  # Hangul Syllable Bbo
+    (0x0bf5c, 0x0bf5c,),  # Hangul Syllable Bbwa
+    (0x0bf78, 0x0bf78,),  # Hangul Syllable Bbwae
+    (0x0bf94, 0x0bf94,),  # Hangul Syllable Bboe
+    (0x0bfb0, 0x0bfb0,),  # Hangul Syllable Bbyo
+    (0x0bfcc, 0x0bfcc,),  # Hangul Syllable Bbu
+    (0x0bfe8, 0x0bfe8,),  # Hangul Syllable Bbweo
+    (0x0c004, 0x0c004,),  # Hangul Syllable Bbwe
+    (0x0c020, 0x0c020,),  # Hangul Syllable Bbwi
+    (0x0c03c, 0x0c03c,),  # Hangul Syllable Bbyu
+    (0x0c058, 0x0c058,),  # Hangul Syllable Bbeu
+    (0x0c074, 0x0c074,),  # Hangul Syllable Bbyi
+    (0x0c090, 0x0c090,),  # Hangul Syllable Bbi
+    (0x0c0ac, 0x0c0ac,),  # Hangul Syllable Sa
+    (0x0c0c8, 0x0c0c8,),  # Hangul Syllable Sae
+    (0x0c0e4, 0x0c0e4,),  # Hangul Syllable Sya
+    (0x0c100, 0x0c100,),  # Hangul Syllable Syae
+    (0x0c11c, 0x0c11c,),  # Hangul Syllable Seo
+    (0x0c138, 0x0c138,),  # Hangul Syllable Se
+    (0x0c154, 0x0c154,),  # Hangul Syllable Syeo
+    (0x0c170, 0x0c170,),  # Hangul Syllable Sye
+    (0x0c18c, 0x0c18c,),  # Hangul Syllable So
+    (0x0c1a8, 0x0c1a8,),  # Hangul Syllable Swa
+    (0x0c1c4, 0x0c1c4,),  # Hangul Syllable Swae
+    (0x0c1e0, 0x0c1e0,),  # Hangul Syllable Soe
+    (0x0c1fc, 0x0c1fc,),  # Hangul Syllable Syo
+    (0x0c218, 0x0c218,),  # Hangul Syllable Su
+    (0x0c234, 0x0c234,),  # Hangul Syllable Sweo
+    (0x0c250, 0x0c250,),  # Hangul Syllable Swe
+    (0x0c26c, 0x0c26c,),  # Hangul Syllable Swi
+    (0x0c288, 0x0c288,),  # Hangul Syllable Syu
+    (0x0c2a4, 0x0c2a4,),  # Hangul Syllable Seu
+    (0x0c2c0, 0x0c2c0,),  # Hangul Syllable Syi
+    (0x0c2dc, 0x0c2dc,),  # Hangul Syllable Si
+    (0x0c2f8, 0x0c2f8,),  # Hangul Syllable Ssa
+    (0x0c314, 0x0c314,),  # Hangul Syllable Ssae
+    (0x0c330, 0x0c330,),  # Hangul Syllable Ssya
+    (0x0c34c, 0x0c34c,),  # Hangul Syllable Ssyae
+    (0x0c368, 0x0c368,),  # Hangul Syllable Sseo
+    (0x0c384, 0x0c384,),  # Hangul Syllable Sse
+    (0x0c3a0, 0x0c3a0,),  # Hangul Syllable Ssyeo
+    (0x0c3bc, 0x0c3bc,),  # Hangul Syllable Ssye
+    (0x0c3d8, 0x0c3d8,),  # Hangul Syllable Sso
+    (0x0c3f4, 0x0c3f4,),  # Hangul Syllable Sswa
+    (0x0c410, 0x0c410,),  # Hangul Syllable Sswae
+    (0x0c42c, 0x0c42c,),  # Hangul Syllable Ssoe
+    (0x0c448, 0x0c448,),  # Hangul Syllable Ssyo
+    (0x0c464, 0x0c464,),  # Hangul Syllable Ssu
+    (0x0c480, 0x0c480,),  # Hangul Syllable Ssweo
+    (0x0c49c, 0x0c49c,),  # Hangul Syllable Sswe
+    (0x0c4b8, 0x0c4b8,),  # Hangul Syllable Sswi
+    (0x0c4d4, 0x0c4d4,),  # Hangul Syllable Ssyu
+    (0x0c4f0, 0x0c4f0,),  # Hangul Syllable Sseu
+    (0x0c50c, 0x0c50c,),  # Hangul Syllable Ssyi
+    (0x0c528, 0x0c528,),  # Hangul Syllable Ssi
+    (0x0c544, 0x0c544,),  # Hangul Syllable A
+    (0x0c560, 0x0c560,),  # Hangul Syllable Ae
+    (0x0c57c, 0x0c57c,),  # Hangul Syllable Ya
+    (0x0c598, 0x0c598,),  # Hangul Syllable Yae
+    (0x0c5b4, 0x0c5b4,),  # Hangul Syllable Eo
+    (0x0c5d0, 0x0c5d0,),  # Hangul Syllable E
+    (0x0c5ec, 0x0c5ec,),  # Hangul Syllable Yeo
+    (0x0c608, 0x0c608,),  # Hangul Syllable Ye
+    (0x0c624, 0x0c624,),  # Hangul Syllable O
+    (0x0c640, 0x0c640,),  # Hangul Syllable Wa
+    (0x0c65c, 0x0c65c,),  # Hangul Syllable Wae
+    (0x0c678, 0x0c678,),  # Hangul Syllable Oe
+    (0x0c694, 0x0c694,),  # Hangul Syllable Yo
+    (0x0c6b0, 0x0c6b0,),  # Hangul Syllable U
+    (0x0c6cc, 0x0c6cc,),  # Hangul Syllable Weo
+    (0x0c6e8, 0x0c6e8,),  # Hangul Syllable We
+    (0x0c704, 0x0c704,),  # Hangul Syllable Wi
+    (0x0c720, 0x0c720,),  # Hangul Syllable Yu
+    (0x0c73c, 0x0c73c,),  # Hangul Syllable Eu
+    (0x0c758, 0x0c758,),  # Hangul Syllable Yi
+    (0x0c774, 0x0c774,),  # Hangul Syllable I
+    (0x0c790, 0x0c790,),  # Hangul Syllable Ja
+    (0x0c7ac, 0x0c7ac,),  # Hangul Syllable Jae
+    (0x0c7c8, 0x0c7c8,),  # Hangul Syllable Jya
+    (0x0c7e4, 0x0c7e4,),  # Hangul Syllable Jyae
+    (0x0c800, 0x0c800,),  # Hangul Syllable Jeo
+    (0x0c81c, 0x0c81c,),  # Hangul Syllable Je
+    (0x0c838, 0x0c838,),  # Hangul Syllable Jyeo
+    (0x0c854, 0x0c854,),  # Hangul Syllable Jye
+    (0x0c870, 0x0c870,),  # Hangul Syllable Jo
+    (0x0c88c, 0x0c88c,),  # Hangul Syllable Jwa
+    (0x0c8a8, 0x0c8a8,),  # Hangul Syllable Jwae
+    (0x0c8c4, 0x0c8c4,),  # Hangul Syllable Joe
+    (0x0c8e0, 0x0c8e0,),  # Hangul Syllable Jyo
+    (0x0c8fc, 0x0c8fc,),  # Hangul Syllable Ju
+    (0x0c918, 0x0c918,),  # Hangul Syllable Jweo
+    (0x0c934, 0x0c934,),  # Hangul Syllable Jwe
+    (0x0c950, 0x0c950,),  # Hangul Syllable Jwi
+    (0x0c96c, 0x0c96c,),  # Hangul Syllable Jyu
+    (0x0c988, 0x0c988,),  # Hangul Syllable Jeu
+    (0x0c9a4, 0x0c9a4,),  # Hangul Syllable Jyi
+    (0x0c9c0, 0x0c9c0,),  # Hangul Syllable Ji
+    (0x0c9dc, 0x0c9dc,),  # Hangul Syllable Jja
+    (0x0c9f8, 0x0c9f8,),  # Hangul Syllable Jjae
+    (0x0ca14, 0x0ca14,),  # Hangul Syllable Jjya
+    (0x0ca30, 0x0ca30,),  # Hangul Syllable Jjyae
+    (0x0ca4c, 0x0ca4c,),  # Hangul Syllable Jjeo
+    (0x0ca68, 0x0ca68,),  # Hangul Syllable Jje
+    (0x0ca84, 0x0ca84,),  # Hangul Syllable Jjyeo
+    (0x0caa0, 0x0caa0,),  # Hangul Syllable Jjye
+    (0x0cabc, 0x0cabc,),  # Hangul Syllable Jjo
+    (0x0cad8, 0x0cad8,),  # Hangul Syllable Jjwa
+    (0x0caf4, 0x0caf4,),  # Hangul Syllable Jjwae
+    (0x0cb10, 0x0cb10,),  # Hangul Syllable Jjoe
+    (0x0cb2c, 0x0cb2c,),  # Hangul Syllable Jjyo
+    (0x0cb48, 0x0cb48,),  # Hangul Syllable Jju
+    (0x0cb64, 0x0cb64,),  # Hangul Syllable Jjweo
+    (0x0cb80, 0x0cb80,),  # Hangul Syllable Jjwe
+    (0x0cb9c, 0x0cb9c,),  # Hangul Syllable Jjwi
+    (0x0cbb8, 0x0cbb8,),  # Hangul Syllable Jjyu
+    (0x0cbd4, 0x0cbd4,),  # Hangul Syllable Jjeu
+    (0x0cbf0, 0x0cbf0,),  # Hangul Syllable Jjyi
+    (0x0cc0c, 0x0cc0c,),  # Hangul Syllable Jji
+    (0x0cc28, 0x0cc28,),  # Hangul Syllable Ca
+    (0x0cc44, 0x0cc44,),  # Hangul Syllable Cae
+    (0x0cc60, 0x0cc60,),  # Hangul Syllable Cya
+    (0x0cc7c, 0x0cc7c,),  # Hangul Syllable Cyae
+    (0x0cc98, 0x0cc98,),  # Hangul Syllable Ceo
+    (0x0ccb4, 0x0ccb4,),  # Hangul Syllable Ce
+    (0x0ccd0, 0x0ccd0,),  # Hangul Syllable Cyeo
+    (0x0ccec, 0x0ccec,),  # Hangul Syllable Cye
+    (0x0cd08, 0x0cd08,),  # Hangul Syllable Co
+    (0x0cd24, 0x0cd24,),  # Hangul Syllable Cwa
+    (0x0cd40, 0x0cd40,),  # Hangul Syllable Cwae
+    (0x0cd5c, 0x0cd5c,),  # Hangul Syllable Coe
+    (0x0cd78, 0x0cd78,),  # Hangul Syllable Cyo
+    (0x0cd94, 0x0cd94,),  # Hangul Syllable Cu
+    (0x0cdb0, 0x0cdb0,),  # Hangul Syllable Cweo
+    (0x0cdcc, 0x0cdcc,),  # Hangul Syllable Cwe
+    (0x0cde8, 0x0cde8,),  # Hangul Syllable Cwi
+    (0x0ce04, 0x0ce04,),  # Hangul Syllable Cyu
+    (0x0ce20, 0x0ce20,),  # Hangul Syllable Ceu
+    (0x0ce3c, 0x0ce3c,),  # Hangul Syllable Cyi
+    (0x0ce58, 0x0ce58,),  # Hangul Syllable Ci
+    (0x0ce74, 0x0ce74,),  # Hangul Syllable Ka
+    (0x0ce90, 0x0ce90,),  # Hangul Syllable Kae
+    (0x0ceac, 0x0ceac,),  # Hangul Syllable Kya
+    (0x0cec8, 0x0cec8,),  # Hangul Syllable Kyae
+    (0x0cee4, 0x0cee4,),  # Hangul Syllable Keo
+    (0x0cf00, 0x0cf00,),  # Hangul Syllable Ke
+    (0x0cf1c, 0x0cf1c,),  # Hangul Syllable Kyeo
+    (0x0cf38, 0x0cf38,),  # Hangul Syllable Kye
+    (0x0cf54, 0x0cf54,),  # Hangul Syllable Ko
+    (0x0cf70, 0x0cf70,),  # Hangul Syllable Kwa
+    (0x0cf8c, 0x0cf8c,),  # Hangul Syllable Kwae
+    (0x0cfa8, 0x0cfa8,),  # Hangul Syllable Koe
+    (0x0cfc4, 0x0cfc4,),  # Hangul Syllable Kyo
+    (0x0cfe0, 0x0cfe0,),  # Hangul Syllable Ku
+    (0x0cffc, 0x0cffc,),  # Hangul Syllable Kweo
+    (0x0d018, 0x0d018,),  # Hangul Syllable Kwe
+    (0x0d034, 0x0d034,),  # Hangul Syllable Kwi
+    (0x0d050, 0x0d050,),  # Hangul Syllable Kyu
+    (0x0d06c, 0x0d06c,),  # Hangul Syllable Keu
+    (0x0d088, 0x0d088,),  # Hangul Syllable Kyi
+    (0x0d0a4, 0x0d0a4,),  # Hangul Syllable Ki
+    (0x0d0c0, 0x0d0c0,),  # Hangul Syllable Ta
+    (0x0d0dc, 0x0d0dc,),  # Hangul Syllable Tae
+    (0x0d0f8, 0x0d0f8,),  # Hangul Syllable Tya
+    (0x0d114, 0x0d114,),  # Hangul Syllable Tyae
+    (0x0d130, 0x0d130,),  # Hangul Syllable Teo
+    (0x0d14c, 0x0d14c,),  # Hangul Syllable Te
+    (0x0d168, 0x0d168,),  # Hangul Syllable Tyeo
+    (0x0d184, 0x0d184,),  # Hangul Syllable Tye
+    (0x0d1a0, 0x0d1a0,),  # Hangul Syllable To
+    (0x0d1bc, 0x0d1bc,),  # Hangul Syllable Twa
+    (0x0d1d8, 0x0d1d8,),  # Hangul Syllable Twae
+    (0x0d1f4, 0x0d1f4,),  # Hangul Syllable Toe
+    (0x0d210, 0x0d210,),  # Hangul Syllable Tyo
+    (0x0d22c, 0x0d22c,),  # Hangul Syllable Tu
+    (0x0d248, 0x0d248,),  # Hangul Syllable Tweo
+    (0x0d264, 0x0d264,),  # Hangul Syllable Twe
+    (0x0d280, 0x0d280,),  # Hangul Syllable Twi
+    (0x0d29c, 0x0d29c,),  # Hangul Syllable Tyu
+    (0x0d2b8, 0x0d2b8,),  # Hangul Syllable Teu
+    (0x0d2d4, 0x0d2d4,),  # Hangul Syllable Tyi
+    (0x0d2f0, 0x0d2f0,),  # Hangul Syllable Ti
+    (0x0d30c, 0x0d30c,),  # Hangul Syllable Pa
+    (0x0d328, 0x0d328,),  # Hangul Syllable Pae
+    (0x0d344, 0x0d344,),  # Hangul Syllable Pya
+    (0x0d360, 0x0d360,),  # Hangul Syllable Pyae
+    (0x0d37c, 0x0d37c,),  # Hangul Syllable Peo
+    (0x0d398, 0x0d398,),  # Hangul Syllable Pe
+    (0x0d3b4, 0x0d3b4,),  # Hangul Syllable Pyeo
+    (0x0d3d0, 0x0d3d0,),  # Hangul Syllable Pye
+    (0x0d3ec, 0x0d3ec,),  # Hangul Syllable Po
+    (0x0d408, 0x0d408,),  # Hangul Syllable Pwa
+    (0x0d424, 0x0d424,),  # Hangul Syllable Pwae
+    (0x0d440, 0x0d440,),  # Hangul Syllable Poe
+    (0x0d45c, 0x0d45c,),  # Hangul Syllable Pyo
+    (0x0d478, 0x0d478,),  # Hangul Syllable Pu
+    (0x0d494, 0x0d494,),  # Hangul Syllable Pweo
+    (0x0d4b0, 0x0d4b0,),  # Hangul Syllable Pwe
+    (0x0d4cc, 0x0d4cc,),  # Hangul Syllable Pwi
+    (0x0d4e8, 0x0d4e8,),  # Hangul Syllable Pyu
+    (0x0d504, 0x0d504,),  # Hangul Syllable Peu
+    (0x0d520, 0x0d520,),  # Hangul Syllable Pyi
+    (0x0d53c, 0x0d53c,),  # Hangul Syllable Pi
+    (0x0d558, 0x0d558,),  # Hangul Syllable Ha
+    (0x0d574, 0x0d574,),  # Hangul Syllable Hae
+    (0x0d590, 0x0d590,),  # Hangul Syllable Hya
+    (0x0d5ac, 0x0d5ac,),  # Hangul Syllable Hyae
+    (0x0d5c8, 0x0d5c8,),  # Hangul Syllable Heo
+    (0x0d5e4, 0x0d5e4,),  # Hangul Syllable He
+    (0x0d600, 0x0d600,),  # Hangul Syllable Hyeo
+    (0x0d61c, 0x0d61c,),  # Hangul Syllable Hye
+    (0x0d638, 0x0d638,),  # Hangul Syllable Ho
+    (0x0d654, 0x0d654,),  # Hangul Syllable Hwa
+    (0x0d670, 0x0d670,),  # Hangul Syllable Hwae
+    (0x0d68c, 0x0d68c,),  # Hangul Syllable Hoe
+    (0x0d6a8, 0x0d6a8,),  # Hangul Syllable Hyo
+    (0x0d6c4, 0x0d6c4,),  # Hangul Syllable Hu
+    (0x0d6e0, 0x0d6e0,),  # Hangul Syllable Hweo
+    (0x0d6fc, 0x0d6fc,),  # Hangul Syllable Hwe
+    (0x0d718, 0x0d718,),  # Hangul Syllable Hwi
+    (0x0d734, 0x0d734,),  # Hangul Syllable Hyu
+    (0x0d750, 0x0d750,),  # Hangul Syllable Heu
+    (0x0d76c, 0x0d76c,),  # Hangul Syllable Hyi
+    (0x0d788, 0x0d788,),  # Hangul Syllable Hi
+)
+
+GRAPHEME_LVT = (
+    # Source: GraphemeBreakProperty-17.0.0.txt
+    # Date: 2025-06-30, 06:20:23 GMT
+    #
+    (0x0ac01, 0x0ac1b,),  # Hangul Syllable Gag     ..Hangul Syllable Gah
+    (0x0ac1d, 0x0ac37,),  # Hangul Syllable Gaeg    ..Hangul Syllable Gaeh
+    (0x0ac39, 0x0ac53,),  # Hangul Syllable Gyag    ..Hangul Syllable Gyah
+    (0x0ac55, 0x0ac6f,),  # Hangul Syllable Gyaeg   ..Hangul Syllable Gyaeh
+    (0x0ac71, 0x0ac8b,),  # Hangul Syllable Geog    ..Hangul Syllable Geoh
+    (0x0ac8d, 0x0aca7,),  # Hangul Syllable Geg     ..Hangul Syllable Geh
+    (0x0aca9, 0x0acc3,),  # Hangul Syllable Gyeog   ..Hangul Syllable Gyeoh
+    (0x0acc5, 0x0acdf,),  # Hangul Syllable Gyeg    ..Hangul Syllable Gyeh
+    (0x0ace1, 0x0acfb,),  # Hangul Syllable Gog     ..Hangul Syllable Goh
+    (0x0acfd, 0x0ad17,),  # Hangul Syllable Gwag    ..Hangul Syllable Gwah
+    (0x0ad19, 0x0ad33,),  # Hangul Syllable Gwaeg   ..Hangul Syllable Gwaeh
+    (0x0ad35, 0x0ad4f,),  # Hangul Syllable Goeg    ..Hangul Syllable Goeh
+    (0x0ad51, 0x0ad6b,),  # Hangul Syllable Gyog    ..Hangul Syllable Gyoh
+    (0x0ad6d, 0x0ad87,),  # Hangul Syllable Gug     ..Hangul Syllable Guh
+    (0x0ad89, 0x0ada3,),  # Hangul Syllable Gweog   ..Hangul Syllable Gweoh
+    (0x0ada5, 0x0adbf,),  # Hangul Syllable Gweg    ..Hangul Syllable Gweh
+    (0x0adc1, 0x0addb,),  # Hangul Syllable Gwig    ..Hangul Syllable Gwih
+    (0x0addd, 0x0adf7,),  # Hangul Syllable Gyug    ..Hangul Syllable Gyuh
+    (0x0adf9, 0x0ae13,),  # Hangul Syllable Geug    ..Hangul Syllable Geuh
+    (0x0ae15, 0x0ae2f,),  # Hangul Syllable Gyig    ..Hangul Syllable Gyih
+    (0x0ae31, 0x0ae4b,),  # Hangul Syllable Gig     ..Hangul Syllable Gih
+    (0x0ae4d, 0x0ae67,),  # Hangul Syllable Ggag    ..Hangul Syllable Ggah
+    (0x0ae69, 0x0ae83,),  # Hangul Syllable Ggaeg   ..Hangul Syllable Ggaeh
+    (0x0ae85, 0x0ae9f,),  # Hangul Syllable Ggyag   ..Hangul Syllable Ggyah
+    (0x0aea1, 0x0aebb,),  # Hangul Syllable Ggyaeg  ..Hangul Syllable Ggyaeh
+    (0x0aebd, 0x0aed7,),  # Hangul Syllable Ggeog   ..Hangul Syllable Ggeoh
+    (0x0aed9, 0x0aef3,),  # Hangul Syllable Ggeg    ..Hangul Syllable Ggeh
+    (0x0aef5, 0x0af0f,),  # Hangul Syllable Ggyeog  ..Hangul Syllable Ggyeoh
+    (0x0af11, 0x0af2b,),  # Hangul Syllable Ggyeg   ..Hangul Syllable Ggyeh
+    (0x0af2d, 0x0af47,),  # Hangul Syllable Ggog    ..Hangul Syllable Ggoh
+    (0x0af49, 0x0af63,),  # Hangul Syllable Ggwag   ..Hangul Syllable Ggwah
+    (0x0af65, 0x0af7f,),  # Hangul Syllable Ggwaeg  ..Hangul Syllable Ggwaeh
+    (0x0af81, 0x0af9b,),  # Hangul Syllable Ggoeg   ..Hangul Syllable Ggoeh
+    (0x0af9d, 0x0afb7,),  # Hangul Syllable Ggyog   ..Hangul Syllable Ggyoh
+    (0x0afb9, 0x0afd3,),  # Hangul Syllable Ggug    ..Hangul Syllable Gguh
+    (0x0afd5, 0x0afef,),  # Hangul Syllable Ggweog  ..Hangul Syllable Ggweoh
+    (0x0aff1, 0x0b00b,),  # Hangul Syllable Ggweg   ..Hangul Syllable Ggweh
+    (0x0b00d, 0x0b027,),  # Hangul Syllable Ggwig   ..Hangul Syllable Ggwih
+    (0x0b029, 0x0b043,),  # Hangul Syllable Ggyug   ..Hangul Syllable Ggyuh
+    (0x0b045, 0x0b05f,),  # Hangul Syllable Ggeug   ..Hangul Syllable Ggeuh
+    (0x0b061, 0x0b07b,),  # Hangul Syllable Ggyig   ..Hangul Syllable Ggyih
+    (0x0b07d, 0x0b097,),  # Hangul Syllable Ggig    ..Hangul Syllable Ggih
+    (0x0b099, 0x0b0b3,),  # Hangul Syllable Nag     ..Hangul Syllable Nah
+    (0x0b0b5, 0x0b0cf,),  # Hangul Syllable Naeg    ..Hangul Syllable Naeh
+    (0x0b0d1, 0x0b0eb,),  # Hangul Syllable Nyag    ..Hangul Syllable Nyah
+    (0x0b0ed, 0x0b107,),  # Hangul Syllable Nyaeg   ..Hangul Syllable Nyaeh
+    (0x0b109, 0x0b123,),  # Hangul Syllable Neog    ..Hangul Syllable Neoh
+    (0x0b125, 0x0b13f,),  # Hangul Syllable Neg     ..Hangul Syllable Neh
+    (0x0b141, 0x0b15b,),  # Hangul Syllable Nyeog   ..Hangul Syllable Nyeoh
+    (0x0b15d, 0x0b177,),  # Hangul Syllable Nyeg    ..Hangul Syllable Nyeh
+    (0x0b179, 0x0b193,),  # Hangul Syllable Nog     ..Hangul Syllable Noh
+    (0x0b195, 0x0b1af,),  # Hangul Syllable Nwag    ..Hangul Syllable Nwah
+    (0x0b1b1, 0x0b1cb,),  # Hangul Syllable Nwaeg   ..Hangul Syllable Nwaeh
+    (0x0b1cd, 0x0b1e7,),  # Hangul Syllable Noeg    ..Hangul Syllable Noeh
+    (0x0b1e9, 0x0b203,),  # Hangul Syllable Nyog    ..Hangul Syllable Nyoh
+    (0x0b205, 0x0b21f,),  # Hangul Syllable Nug     ..Hangul Syllable Nuh
+    (0x0b221, 0x0b23b,),  # Hangul Syllable Nweog   ..Hangul Syllable Nweoh
+    (0x0b23d, 0x0b257,),  # Hangul Syllable Nweg    ..Hangul Syllable Nweh
+    (0x0b259, 0x0b273,),  # Hangul Syllable Nwig    ..Hangul Syllable Nwih
+    (0x0b275, 0x0b28f,),  # Hangul Syllable Nyug    ..Hangul Syllable Nyuh
+    (0x0b291, 0x0b2ab,),  # Hangul Syllable Neug    ..Hangul Syllable Neuh
+    (0x0b2ad, 0x0b2c7,),  # Hangul Syllable Nyig    ..Hangul Syllable Nyih
+    (0x0b2c9, 0x0b2e3,),  # Hangul Syllable Nig     ..Hangul Syllable Nih
+    (0x0b2e5, 0x0b2ff,),  # Hangul Syllable Dag     ..Hangul Syllable Dah
+    (0x0b301, 0x0b31b,),  # Hangul Syllable Daeg    ..Hangul Syllable Daeh
+    (0x0b31d, 0x0b337,),  # Hangul Syllable Dyag    ..Hangul Syllable Dyah
+    (0x0b339, 0x0b353,),  # Hangul Syllable Dyaeg   ..Hangul Syllable Dyaeh
+    (0x0b355, 0x0b36f,),  # Hangul Syllable Deog    ..Hangul Syllable Deoh
+    (0x0b371, 0x0b38b,),  # Hangul Syllable Deg     ..Hangul Syllable Deh
+    (0x0b38d, 0x0b3a7,),  # Hangul Syllable Dyeog   ..Hangul Syllable Dyeoh
+    (0x0b3a9, 0x0b3c3,),  # Hangul Syllable Dyeg    ..Hangul Syllable Dyeh
+    (0x0b3c5, 0x0b3df,),  # Hangul Syllable Dog     ..Hangul Syllable Doh
+    (0x0b3e1, 0x0b3fb,),  # Hangul Syllable Dwag    ..Hangul Syllable Dwah
+    (0x0b3fd, 0x0b417,),  # Hangul Syllable Dwaeg   ..Hangul Syllable Dwaeh
+    (0x0b419, 0x0b433,),  # Hangul Syllable Doeg    ..Hangul Syllable Doeh
+    (0x0b435, 0x0b44f,),  # Hangul Syllable Dyog    ..Hangul Syllable Dyoh
+    (0x0b451, 0x0b46b,),  # Hangul Syllable Dug     ..Hangul Syllable Duh
+    (0x0b46d, 0x0b487,),  # Hangul Syllable Dweog   ..Hangul Syllable Dweoh
+    (0x0b489, 0x0b4a3,),  # Hangul Syllable Dweg    ..Hangul Syllable Dweh
+    (0x0b4a5, 0x0b4bf,),  # Hangul Syllable Dwig    ..Hangul Syllable Dwih
+    (0x0b4c1, 0x0b4db,),  # Hangul Syllable Dyug    ..Hangul Syllable Dyuh
+    (0x0b4dd, 0x0b4f7,),  # Hangul Syllable Deug    ..Hangul Syllable Deuh
+    (0x0b4f9, 0x0b513,),  # Hangul Syllable Dyig    ..Hangul Syllable Dyih
+    (0x0b515, 0x0b52f,),  # Hangul Syllable Dig     ..Hangul Syllable Dih
+    (0x0b531, 0x0b54b,),  # Hangul Syllable Ddag    ..Hangul Syllable Ddah
+    (0x0b54d, 0x0b567,),  # Hangul Syllable Ddaeg   ..Hangul Syllable Ddaeh
+    (0x0b569, 0x0b583,),  # Hangul Syllable Ddyag   ..Hangul Syllable Ddyah
+    (0x0b585, 0x0b59f,),  # Hangul Syllable Ddyaeg  ..Hangul Syllable Ddyaeh
+    (0x0b5a1, 0x0b5bb,),  # Hangul Syllable Ddeog   ..Hangul Syllable Ddeoh
+    (0x0b5bd, 0x0b5d7,),  # Hangul Syllable Ddeg    ..Hangul Syllable Ddeh
+    (0x0b5d9, 0x0b5f3,),  # Hangul Syllable Ddyeog  ..Hangul Syllable Ddyeoh
+    (0x0b5f5, 0x0b60f,),  # Hangul Syllable Ddyeg   ..Hangul Syllable Ddyeh
+    (0x0b611, 0x0b62b,),  # Hangul Syllable Ddog    ..Hangul Syllable Ddoh
+    (0x0b62d, 0x0b647,),  # Hangul Syllable Ddwag   ..Hangul Syllable Ddwah
+    (0x0b649, 0x0b663,),  # Hangul Syllable Ddwaeg  ..Hangul Syllable Ddwaeh
+    (0x0b665, 0x0b67f,),  # Hangul Syllable Ddoeg   ..Hangul Syllable Ddoeh
+    (0x0b681, 0x0b69b,),  # Hangul Syllable Ddyog   ..Hangul Syllable Ddyoh
+    (0x0b69d, 0x0b6b7,),  # Hangul Syllable Ddug    ..Hangul Syllable Dduh
+    (0x0b6b9, 0x0b6d3,),  # Hangul Syllable Ddweog  ..Hangul Syllable Ddweoh
+    (0x0b6d5, 0x0b6ef,),  # Hangul Syllable Ddweg   ..Hangul Syllable Ddweh
+    (0x0b6f1, 0x0b70b,),  # Hangul Syllable Ddwig   ..Hangul Syllable Ddwih
+    (0x0b70d, 0x0b727,),  # Hangul Syllable Ddyug   ..Hangul Syllable Ddyuh
+    (0x0b729, 0x0b743,),  # Hangul Syllable Ddeug   ..Hangul Syllable Ddeuh
+    (0x0b745, 0x0b75f,),  # Hangul Syllable Ddyig   ..Hangul Syllable Ddyih
+    (0x0b761, 0x0b77b,),  # Hangul Syllable Ddig    ..Hangul Syllable Ddih
+    (0x0b77d, 0x0b797,),  # Hangul Syllable Rag     ..Hangul Syllable Rah
+    (0x0b799, 0x0b7b3,),  # Hangul Syllable Raeg    ..Hangul Syllable Raeh
+    (0x0b7b5, 0x0b7cf,),  # Hangul Syllable Ryag    ..Hangul Syllable Ryah
+    (0x0b7d1, 0x0b7eb,),  # Hangul Syllable Ryaeg   ..Hangul Syllable Ryaeh
+    (0x0b7ed, 0x0b807,),  # Hangul Syllable Reog    ..Hangul Syllable Reoh
+    (0x0b809, 0x0b823,),  # Hangul Syllable Reg     ..Hangul Syllable Reh
+    (0x0b825, 0x0b83f,),  # Hangul Syllable Ryeog   ..Hangul Syllable Ryeoh
+    (0x0b841, 0x0b85b,),  # Hangul Syllable Ryeg    ..Hangul Syllable Ryeh
+    (0x0b85d, 0x0b877,),  # Hangul Syllable Rog     ..Hangul Syllable Roh
+    (0x0b879, 0x0b893,),  # Hangul Syllable Rwag    ..Hangul Syllable Rwah
+    (0x0b895, 0x0b8af,),  # Hangul Syllable Rwaeg   ..Hangul Syllable Rwaeh
+    (0x0b8b1, 0x0b8cb,),  # Hangul Syllable Roeg    ..Hangul Syllable Roeh
+    (0x0b8cd, 0x0b8e7,),  # Hangul Syllable Ryog    ..Hangul Syllable Ryoh
+    (0x0b8e9, 0x0b903,),  # Hangul Syllable Rug     ..Hangul Syllable Ruh
+    (0x0b905, 0x0b91f,),  # Hangul Syllable Rweog   ..Hangul Syllable Rweoh
+    (0x0b921, 0x0b93b,),  # Hangul Syllable Rweg    ..Hangul Syllable Rweh
+    (0x0b93d, 0x0b957,),  # Hangul Syllable Rwig    ..Hangul Syllable Rwih
+    (0x0b959, 0x0b973,),  # Hangul Syllable Ryug    ..Hangul Syllable Ryuh
+    (0x0b975, 0x0b98f,),  # Hangul Syllable Reug    ..Hangul Syllable Reuh
+    (0x0b991, 0x0b9ab,),  # Hangul Syllable Ryig    ..Hangul Syllable Ryih
+    (0x0b9ad, 0x0b9c7,),  # Hangul Syllable Rig     ..Hangul Syllable Rih
+    (0x0b9c9, 0x0b9e3,),  # Hangul Syllable Mag     ..Hangul Syllable Mah
+    (0x0b9e5, 0x0b9ff,),  # Hangul Syllable Maeg    ..Hangul Syllable Maeh
+    (0x0ba01, 0x0ba1b,),  # Hangul Syllable Myag    ..Hangul Syllable Myah
+    (0x0ba1d, 0x0ba37,),  # Hangul Syllable Myaeg   ..Hangul Syllable Myaeh
+    (0x0ba39, 0x0ba53,),  # Hangul Syllable Meog    ..Hangul Syllable Meoh
+    (0x0ba55, 0x0ba6f,),  # Hangul Syllable Meg     ..Hangul Syllable Meh
+    (0x0ba71, 0x0ba8b,),  # Hangul Syllable Myeog   ..Hangul Syllable Myeoh
+    (0x0ba8d, 0x0baa7,),  # Hangul Syllable Myeg    ..Hangul Syllable Myeh
+    (0x0baa9, 0x0bac3,),  # Hangul Syllable Mog     ..Hangul Syllable Moh
+    (0x0bac5, 0x0badf,),  # Hangul Syllable Mwag    ..Hangul Syllable Mwah
+    (0x0bae1, 0x0bafb,),  # Hangul Syllable Mwaeg   ..Hangul Syllable Mwaeh
+    (0x0bafd, 0x0bb17,),  # Hangul Syllable Moeg    ..Hangul Syllable Moeh
+    (0x0bb19, 0x0bb33,),  # Hangul Syllable Myog    ..Hangul Syllable Myoh
+    (0x0bb35, 0x0bb4f,),  # Hangul Syllable Mug     ..Hangul Syllable Muh
+    (0x0bb51, 0x0bb6b,),  # Hangul Syllable Mweog   ..Hangul Syllable Mweoh
+    (0x0bb6d, 0x0bb87,),  # Hangul Syllable Mweg    ..Hangul Syllable Mweh
+    (0x0bb89, 0x0bba3,),  # Hangul Syllable Mwig    ..Hangul Syllable Mwih
+    (0x0bba5, 0x0bbbf,),  # Hangul Syllable Myug    ..Hangul Syllable Myuh
+    (0x0bbc1, 0x0bbdb,),  # Hangul Syllable Meug    ..Hangul Syllable Meuh
+    (0x0bbdd, 0x0bbf7,),  # Hangul Syllable Myig    ..Hangul Syllable Myih
+    (0x0bbf9, 0x0bc13,),  # Hangul Syllable Mig     ..Hangul Syllable Mih
+    (0x0bc15, 0x0bc2f,),  # Hangul Syllable Bag     ..Hangul Syllable Bah
+    (0x0bc31, 0x0bc4b,),  # Hangul Syllable Baeg    ..Hangul Syllable Baeh
+    (0x0bc4d, 0x0bc67,),  # Hangul Syllable Byag    ..Hangul Syllable Byah
+    (0x0bc69, 0x0bc83,),  # Hangul Syllable Byaeg   ..Hangul Syllable Byaeh
+    (0x0bc85, 0x0bc9f,),  # Hangul Syllable Beog    ..Hangul Syllable Beoh
+    (0x0bca1, 0x0bcbb,),  # Hangul Syllable Beg     ..Hangul Syllable Beh
+    (0x0bcbd, 0x0bcd7,),  # Hangul Syllable Byeog   ..Hangul Syllable Byeoh
+    (0x0bcd9, 0x0bcf3,),  # Hangul Syllable Byeg    ..Hangul Syllable Byeh
+    (0x0bcf5, 0x0bd0f,),  # Hangul Syllable Bog     ..Hangul Syllable Boh
+    (0x0bd11, 0x0bd2b,),  # Hangul Syllable Bwag    ..Hangul Syllable Bwah
+    (0x0bd2d, 0x0bd47,),  # Hangul Syllable Bwaeg   ..Hangul Syllable Bwaeh
+    (0x0bd49, 0x0bd63,),  # Hangul Syllable Boeg    ..Hangul Syllable Boeh
+    (0x0bd65, 0x0bd7f,),  # Hangul Syllable Byog    ..Hangul Syllable Byoh
+    (0x0bd81, 0x0bd9b,),  # Hangul Syllable Bug     ..Hangul Syllable Buh
+    (0x0bd9d, 0x0bdb7,),  # Hangul Syllable Bweog   ..Hangul Syllable Bweoh
+    (0x0bdb9, 0x0bdd3,),  # Hangul Syllable Bweg    ..Hangul Syllable Bweh
+    (0x0bdd5, 0x0bdef,),  # Hangul Syllable Bwig    ..Hangul Syllable Bwih
+    (0x0bdf1, 0x0be0b,),  # Hangul Syllable Byug    ..Hangul Syllable Byuh
+    (0x0be0d, 0x0be27,),  # Hangul Syllable Beug    ..Hangul Syllable Beuh
+    (0x0be29, 0x0be43,),  # Hangul Syllable Byig    ..Hangul Syllable Byih
+    (0x0be45, 0x0be5f,),  # Hangul Syllable Big     ..Hangul Syllable Bih
+    (0x0be61, 0x0be7b,),  # Hangul Syllable Bbag    ..Hangul Syllable Bbah
+    (0x0be7d, 0x0be97,),  # Hangul Syllable Bbaeg   ..Hangul Syllable Bbaeh
+    (0x0be99, 0x0beb3,),  # Hangul Syllable Bbyag   ..Hangul Syllable Bbyah
+    (0x0beb5, 0x0becf,),  # Hangul Syllable Bbyaeg  ..Hangul Syllable Bbyaeh
+    (0x0bed1, 0x0beeb,),  # Hangul Syllable Bbeog   ..Hangul Syllable Bbeoh
+    (0x0beed, 0x0bf07,),  # Hangul Syllable Bbeg    ..Hangul Syllable Bbeh
+    (0x0bf09, 0x0bf23,),  # Hangul Syllable Bbyeog  ..Hangul Syllable Bbyeoh
+    (0x0bf25, 0x0bf3f,),  # Hangul Syllable Bbyeg   ..Hangul Syllable Bbyeh
+    (0x0bf41, 0x0bf5b,),  # Hangul Syllable Bbog    ..Hangul Syllable Bboh
+    (0x0bf5d, 0x0bf77,),  # Hangul Syllable Bbwag   ..Hangul Syllable Bbwah
+    (0x0bf79, 0x0bf93,),  # Hangul Syllable Bbwaeg  ..Hangul Syllable Bbwaeh
+    (0x0bf95, 0x0bfaf,),  # Hangul Syllable Bboeg   ..Hangul Syllable Bboeh
+    (0x0bfb1, 0x0bfcb,),  # Hangul Syllable Bbyog   ..Hangul Syllable Bbyoh
+    (0x0bfcd, 0x0bfe7,),  # Hangul Syllable Bbug    ..Hangul Syllable Bbuh
+    (0x0bfe9, 0x0c003,),  # Hangul Syllable Bbweog  ..Hangul Syllable Bbweoh
+    (0x0c005, 0x0c01f,),  # Hangul Syllable Bbweg   ..Hangul Syllable Bbweh
+    (0x0c021, 0x0c03b,),  # Hangul Syllable Bbwig   ..Hangul Syllable Bbwih
+    (0x0c03d, 0x0c057,),  # Hangul Syllable Bbyug   ..Hangul Syllable Bbyuh
+    (0x0c059, 0x0c073,),  # Hangul Syllable Bbeug   ..Hangul Syllable Bbeuh
+    (0x0c075, 0x0c08f,),  # Hangul Syllable Bbyig   ..Hangul Syllable Bbyih
+    (0x0c091, 0x0c0ab,),  # Hangul Syllable Bbig    ..Hangul Syllable Bbih
+    (0x0c0ad, 0x0c0c7,),  # Hangul Syllable Sag     ..Hangul Syllable Sah
+    (0x0c0c9, 0x0c0e3,),  # Hangul Syllable Saeg    ..Hangul Syllable Saeh
+    (0x0c0e5, 0x0c0ff,),  # Hangul Syllable Syag    ..Hangul Syllable Syah
+    (0x0c101, 0x0c11b,),  # Hangul Syllable Syaeg   ..Hangul Syllable Syaeh
+    (0x0c11d, 0x0c137,),  # Hangul Syllable Seog    ..Hangul Syllable Seoh
+    (0x0c139, 0x0c153,),  # Hangul Syllable Seg     ..Hangul Syllable Seh
+    (0x0c155, 0x0c16f,),  # Hangul Syllable Syeog   ..Hangul Syllable Syeoh
+    (0x0c171, 0x0c18b,),  # Hangul Syllable Syeg    ..Hangul Syllable Syeh
+    (0x0c18d, 0x0c1a7,),  # Hangul Syllable Sog     ..Hangul Syllable Soh
+    (0x0c1a9, 0x0c1c3,),  # Hangul Syllable Swag    ..Hangul Syllable Swah
+    (0x0c1c5, 0x0c1df,),  # Hangul Syllable Swaeg   ..Hangul Syllable Swaeh
+    (0x0c1e1, 0x0c1fb,),  # Hangul Syllable Soeg    ..Hangul Syllable Soeh
+    (0x0c1fd, 0x0c217,),  # Hangul Syllable Syog    ..Hangul Syllable Syoh
+    (0x0c219, 0x0c233,),  # Hangul Syllable Sug     ..Hangul Syllable Suh
+    (0x0c235, 0x0c24f,),  # Hangul Syllable Sweog   ..Hangul Syllable Sweoh
+    (0x0c251, 0x0c26b,),  # Hangul Syllable Sweg    ..Hangul Syllable Sweh
+    (0x0c26d, 0x0c287,),  # Hangul Syllable Swig    ..Hangul Syllable Swih
+    (0x0c289, 0x0c2a3,),  # Hangul Syllable Syug    ..Hangul Syllable Syuh
+    (0x0c2a5, 0x0c2bf,),  # Hangul Syllable Seug    ..Hangul Syllable Seuh
+    (0x0c2c1, 0x0c2db,),  # Hangul Syllable Syig    ..Hangul Syllable Syih
+    (0x0c2dd, 0x0c2f7,),  # Hangul Syllable Sig     ..Hangul Syllable Sih
+    (0x0c2f9, 0x0c313,),  # Hangul Syllable Ssag    ..Hangul Syllable Ssah
+    (0x0c315, 0x0c32f,),  # Hangul Syllable Ssaeg   ..Hangul Syllable Ssaeh
+    (0x0c331, 0x0c34b,),  # Hangul Syllable Ssyag   ..Hangul Syllable Ssyah
+    (0x0c34d, 0x0c367,),  # Hangul Syllable Ssyaeg  ..Hangul Syllable Ssyaeh
+    (0x0c369, 0x0c383,),  # Hangul Syllable Sseog   ..Hangul Syllable Sseoh
+    (0x0c385, 0x0c39f,),  # Hangul Syllable Sseg    ..Hangul Syllable Sseh
+    (0x0c3a1, 0x0c3bb,),  # Hangul Syllable Ssyeog  ..Hangul Syllable Ssyeoh
+    (0x0c3bd, 0x0c3d7,),  # Hangul Syllable Ssyeg   ..Hangul Syllable Ssyeh
+    (0x0c3d9, 0x0c3f3,),  # Hangul Syllable Ssog    ..Hangul Syllable Ssoh
+    (0x0c3f5, 0x0c40f,),  # Hangul Syllable Sswag   ..Hangul Syllable Sswah
+    (0x0c411, 0x0c42b,),  # Hangul Syllable Sswaeg  ..Hangul Syllable Sswaeh
+    (0x0c42d, 0x0c447,),  # Hangul Syllable Ssoeg   ..Hangul Syllable Ssoeh
+    (0x0c449, 0x0c463,),  # Hangul Syllable Ssyog   ..Hangul Syllable Ssyoh
+    (0x0c465, 0x0c47f,),  # Hangul Syllable Ssug    ..Hangul Syllable Ssuh
+    (0x0c481, 0x0c49b,),  # Hangul Syllable Ssweog  ..Hangul Syllable Ssweoh
+    (0x0c49d, 0x0c4b7,),  # Hangul Syllable Ssweg   ..Hangul Syllable Ssweh
+    (0x0c4b9, 0x0c4d3,),  # Hangul Syllable Sswig   ..Hangul Syllable Sswih
+    (0x0c4d5, 0x0c4ef,),  # Hangul Syllable Ssyug   ..Hangul Syllable Ssyuh
+    (0x0c4f1, 0x0c50b,),  # Hangul Syllable Sseug   ..Hangul Syllable Sseuh
+    (0x0c50d, 0x0c527,),  # Hangul Syllable Ssyig   ..Hangul Syllable Ssyih
+    (0x0c529, 0x0c543,),  # Hangul Syllable Ssig    ..Hangul Syllable Ssih
+    (0x0c545, 0x0c55f,),  # Hangul Syllable Ag      ..Hangul Syllable Ah
+    (0x0c561, 0x0c57b,),  # Hangul Syllable Aeg     ..Hangul Syllable Aeh
+    (0x0c57d, 0x0c597,),  # Hangul Syllable Yag     ..Hangul Syllable Yah
+    (0x0c599, 0x0c5b3,),  # Hangul Syllable Yaeg    ..Hangul Syllable Yaeh
+    (0x0c5b5, 0x0c5cf,),  # Hangul Syllable Eog     ..Hangul Syllable Eoh
+    (0x0c5d1, 0x0c5eb,),  # Hangul Syllable Eg      ..Hangul Syllable Eh
+    (0x0c5ed, 0x0c607,),  # Hangul Syllable Yeog    ..Hangul Syllable Yeoh
+    (0x0c609, 0x0c623,),  # Hangul Syllable Yeg     ..Hangul Syllable Yeh
+    (0x0c625, 0x0c63f,),  # Hangul Syllable Og      ..Hangul Syllable Oh
+    (0x0c641, 0x0c65b,),  # Hangul Syllable Wag     ..Hangul Syllable Wah
+    (0x0c65d, 0x0c677,),  # Hangul Syllable Waeg    ..Hangul Syllable Waeh
+    (0x0c679, 0x0c693,),  # Hangul Syllable Oeg     ..Hangul Syllable Oeh
+    (0x0c695, 0x0c6af,),  # Hangul Syllable Yog     ..Hangul Syllable Yoh
+    (0x0c6b1, 0x0c6cb,),  # Hangul Syllable Ug      ..Hangul Syllable Uh
+    (0x0c6cd, 0x0c6e7,),  # Hangul Syllable Weog    ..Hangul Syllable Weoh
+    (0x0c6e9, 0x0c703,),  # Hangul Syllable Weg     ..Hangul Syllable Weh
+    (0x0c705, 0x0c71f,),  # Hangul Syllable Wig     ..Hangul Syllable Wih
+    (0x0c721, 0x0c73b,),  # Hangul Syllable Yug     ..Hangul Syllable Yuh
+    (0x0c73d, 0x0c757,),  # Hangul Syllable Eug     ..Hangul Syllable Euh
+    (0x0c759, 0x0c773,),  # Hangul Syllable Yig     ..Hangul Syllable Yih
+    (0x0c775, 0x0c78f,),  # Hangul Syllable Ig      ..Hangul Syllable Ih
+    (0x0c791, 0x0c7ab,),  # Hangul Syllable Jag     ..Hangul Syllable Jah
+    (0x0c7ad, 0x0c7c7,),  # Hangul Syllable Jaeg    ..Hangul Syllable Jaeh
+    (0x0c7c9, 0x0c7e3,),  # Hangul Syllable Jyag    ..Hangul Syllable Jyah
+    (0x0c7e5, 0x0c7ff,),  # Hangul Syllable Jyaeg   ..Hangul Syllable Jyaeh
+    (0x0c801, 0x0c81b,),  # Hangul Syllable Jeog    ..Hangul Syllable Jeoh
+    (0x0c81d, 0x0c837,),  # Hangul Syllable Jeg     ..Hangul Syllable Jeh
+    (0x0c839, 0x0c853,),  # Hangul Syllable Jyeog   ..Hangul Syllable Jyeoh
+    (0x0c855, 0x0c86f,),  # Hangul Syllable Jyeg    ..Hangul Syllable Jyeh
+    (0x0c871, 0x0c88b,),  # Hangul Syllable Jog     ..Hangul Syllable Joh
+    (0x0c88d, 0x0c8a7,),  # Hangul Syllable Jwag    ..Hangul Syllable Jwah
+    (0x0c8a9, 0x0c8c3,),  # Hangul Syllable Jwaeg   ..Hangul Syllable Jwaeh
+    (0x0c8c5, 0x0c8df,),  # Hangul Syllable Joeg    ..Hangul Syllable Joeh
+    (0x0c8e1, 0x0c8fb,),  # Hangul Syllable Jyog    ..Hangul Syllable Jyoh
+    (0x0c8fd, 0x0c917,),  # Hangul Syllable Jug     ..Hangul Syllable Juh
+    (0x0c919, 0x0c933,),  # Hangul Syllable Jweog   ..Hangul Syllable Jweoh
+    (0x0c935, 0x0c94f,),  # Hangul Syllable Jweg    ..Hangul Syllable Jweh
+    (0x0c951, 0x0c96b,),  # Hangul Syllable Jwig    ..Hangul Syllable Jwih
+    (0x0c96d, 0x0c987,),  # Hangul Syllable Jyug    ..Hangul Syllable Jyuh
+    (0x0c989, 0x0c9a3,),  # Hangul Syllable Jeug    ..Hangul Syllable Jeuh
+    (0x0c9a5, 0x0c9bf,),  # Hangul Syllable Jyig    ..Hangul Syllable Jyih
+    (0x0c9c1, 0x0c9db,),  # Hangul Syllable Jig     ..Hangul Syllable Jih
+    (0x0c9dd, 0x0c9f7,),  # Hangul Syllable Jjag    ..Hangul Syllable Jjah
+    (0x0c9f9, 0x0ca13,),  # Hangul Syllable Jjaeg   ..Hangul Syllable Jjaeh
+    (0x0ca15, 0x0ca2f,),  # Hangul Syllable Jjyag   ..Hangul Syllable Jjyah
+    (0x0ca31, 0x0ca4b,),  # Hangul Syllable Jjyaeg  ..Hangul Syllable Jjyaeh
+    (0x0ca4d, 0x0ca67,),  # Hangul Syllable Jjeog   ..Hangul Syllable Jjeoh
+    (0x0ca69, 0x0ca83,),  # Hangul Syllable Jjeg    ..Hangul Syllable Jjeh
+    (0x0ca85, 0x0ca9f,),  # Hangul Syllable Jjyeog  ..Hangul Syllable Jjyeoh
+    (0x0caa1, 0x0cabb,),  # Hangul Syllable Jjyeg   ..Hangul Syllable Jjyeh
+    (0x0cabd, 0x0cad7,),  # Hangul Syllable Jjog    ..Hangul Syllable Jjoh
+    (0x0cad9, 0x0caf3,),  # Hangul Syllable Jjwag   ..Hangul Syllable Jjwah
+    (0x0caf5, 0x0cb0f,),  # Hangul Syllable Jjwaeg  ..Hangul Syllable Jjwaeh
+    (0x0cb11, 0x0cb2b,),  # Hangul Syllable Jjoeg   ..Hangul Syllable Jjoeh
+    (0x0cb2d, 0x0cb47,),  # Hangul Syllable Jjyog   ..Hangul Syllable Jjyoh
+    (0x0cb49, 0x0cb63,),  # Hangul Syllable Jjug    ..Hangul Syllable Jjuh
+    (0x0cb65, 0x0cb7f,),  # Hangul Syllable Jjweog  ..Hangul Syllable Jjweoh
+    (0x0cb81, 0x0cb9b,),  # Hangul Syllable Jjweg   ..Hangul Syllable Jjweh
+    (0x0cb9d, 0x0cbb7,),  # Hangul Syllable Jjwig   ..Hangul Syllable Jjwih
+    (0x0cbb9, 0x0cbd3,),  # Hangul Syllable Jjyug   ..Hangul Syllable Jjyuh
+    (0x0cbd5, 0x0cbef,),  # Hangul Syllable Jjeug   ..Hangul Syllable Jjeuh
+    (0x0cbf1, 0x0cc0b,),  # Hangul Syllable Jjyig   ..Hangul Syllable Jjyih
+    (0x0cc0d, 0x0cc27,),  # Hangul Syllable Jjig    ..Hangul Syllable Jjih
+    (0x0cc29, 0x0cc43,),  # Hangul Syllable Cag     ..Hangul Syllable Cah
+    (0x0cc45, 0x0cc5f,),  # Hangul Syllable Caeg    ..Hangul Syllable Caeh
+    (0x0cc61, 0x0cc7b,),  # Hangul Syllable Cyag    ..Hangul Syllable Cyah
+    (0x0cc7d, 0x0cc97,),  # Hangul Syllable Cyaeg   ..Hangul Syllable Cyaeh
+    (0x0cc99, 0x0ccb3,),  # Hangul Syllable Ceog    ..Hangul Syllable Ceoh
+    (0x0ccb5, 0x0cccf,),  # Hangul Syllable Ceg     ..Hangul Syllable Ceh
+    (0x0ccd1, 0x0cceb,),  # Hangul Syllable Cyeog   ..Hangul Syllable Cyeoh
+    (0x0cced, 0x0cd07,),  # Hangul Syllable Cyeg    ..Hangul Syllable Cyeh
+    (0x0cd09, 0x0cd23,),  # Hangul Syllable Cog     ..Hangul Syllable Coh
+    (0x0cd25, 0x0cd3f,),  # Hangul Syllable Cwag    ..Hangul Syllable Cwah
+    (0x0cd41, 0x0cd5b,),  # Hangul Syllable Cwaeg   ..Hangul Syllable Cwaeh
+    (0x0cd5d, 0x0cd77,),  # Hangul Syllable Coeg    ..Hangul Syllable Coeh
+    (0x0cd79, 0x0cd93,),  # Hangul Syllable Cyog    ..Hangul Syllable Cyoh
+    (0x0cd95, 0x0cdaf,),  # Hangul Syllable Cug     ..Hangul Syllable Cuh
+    (0x0cdb1, 0x0cdcb,),  # Hangul Syllable Cweog   ..Hangul Syllable Cweoh
+    (0x0cdcd, 0x0cde7,),  # Hangul Syllable Cweg    ..Hangul Syllable Cweh
+    (0x0cde9, 0x0ce03,),  # Hangul Syllable Cwig    ..Hangul Syllable Cwih
+    (0x0ce05, 0x0ce1f,),  # Hangul Syllable Cyug    ..Hangul Syllable Cyuh
+    (0x0ce21, 0x0ce3b,),  # Hangul Syllable Ceug    ..Hangul Syllable Ceuh
+    (0x0ce3d, 0x0ce57,),  # Hangul Syllable Cyig    ..Hangul Syllable Cyih
+    (0x0ce59, 0x0ce73,),  # Hangul Syllable Cig     ..Hangul Syllable Cih
+    (0x0ce75, 0x0ce8f,),  # Hangul Syllable Kag     ..Hangul Syllable Kah
+    (0x0ce91, 0x0ceab,),  # Hangul Syllable Kaeg    ..Hangul Syllable Kaeh
+    (0x0cead, 0x0cec7,),  # Hangul Syllable Kyag    ..Hangul Syllable Kyah
+    (0x0cec9, 0x0cee3,),  # Hangul Syllable Kyaeg   ..Hangul Syllable Kyaeh
+    (0x0cee5, 0x0ceff,),  # Hangul Syllable Keog    ..Hangul Syllable Keoh
+    (0x0cf01, 0x0cf1b,),  # Hangul Syllable Keg     ..Hangul Syllable Keh
+    (0x0cf1d, 0x0cf37,),  # Hangul Syllable Kyeog   ..Hangul Syllable Kyeoh
+    (0x0cf39, 0x0cf53,),  # Hangul Syllable Kyeg    ..Hangul Syllable Kyeh
+    (0x0cf55, 0x0cf6f,),  # Hangul Syllable Kog     ..Hangul Syllable Koh
+    (0x0cf71, 0x0cf8b,),  # Hangul Syllable Kwag    ..Hangul Syllable Kwah
+    (0x0cf8d, 0x0cfa7,),  # Hangul Syllable Kwaeg   ..Hangul Syllable Kwaeh
+    (0x0cfa9, 0x0cfc3,),  # Hangul Syllable Koeg    ..Hangul Syllable Koeh
+    (0x0cfc5, 0x0cfdf,),  # Hangul Syllable Kyog    ..Hangul Syllable Kyoh
+    (0x0cfe1, 0x0cffb,),  # Hangul Syllable Kug     ..Hangul Syllable Kuh
+    (0x0cffd, 0x0d017,),  # Hangul Syllable Kweog   ..Hangul Syllable Kweoh
+    (0x0d019, 0x0d033,),  # Hangul Syllable Kweg    ..Hangul Syllable Kweh
+    (0x0d035, 0x0d04f,),  # Hangul Syllable Kwig    ..Hangul Syllable Kwih
+    (0x0d051, 0x0d06b,),  # Hangul Syllable Kyug    ..Hangul Syllable Kyuh
+    (0x0d06d, 0x0d087,),  # Hangul Syllable Keug    ..Hangul Syllable Keuh
+    (0x0d089, 0x0d0a3,),  # Hangul Syllable Kyig    ..Hangul Syllable Kyih
+    (0x0d0a5, 0x0d0bf,),  # Hangul Syllable Kig     ..Hangul Syllable Kih
+    (0x0d0c1, 0x0d0db,),  # Hangul Syllable Tag     ..Hangul Syllable Tah
+    (0x0d0dd, 0x0d0f7,),  # Hangul Syllable Taeg    ..Hangul Syllable Taeh
+    (0x0d0f9, 0x0d113,),  # Hangul Syllable Tyag    ..Hangul Syllable Tyah
+    (0x0d115, 0x0d12f,),  # Hangul Syllable Tyaeg   ..Hangul Syllable Tyaeh
+    (0x0d131, 0x0d14b,),  # Hangul Syllable Teog    ..Hangul Syllable Teoh
+    (0x0d14d, 0x0d167,),  # Hangul Syllable Teg     ..Hangul Syllable Teh
+    (0x0d169, 0x0d183,),  # Hangul Syllable Tyeog   ..Hangul Syllable Tyeoh
+    (0x0d185, 0x0d19f,),  # Hangul Syllable Tyeg    ..Hangul Syllable Tyeh
+    (0x0d1a1, 0x0d1bb,),  # Hangul Syllable Tog     ..Hangul Syllable Toh
+    (0x0d1bd, 0x0d1d7,),  # Hangul Syllable Twag    ..Hangul Syllable Twah
+    (0x0d1d9, 0x0d1f3,),  # Hangul Syllable Twaeg   ..Hangul Syllable Twaeh
+    (0x0d1f5, 0x0d20f,),  # Hangul Syllable Toeg    ..Hangul Syllable Toeh
+    (0x0d211, 0x0d22b,),  # Hangul Syllable Tyog    ..Hangul Syllable Tyoh
+    (0x0d22d, 0x0d247,),  # Hangul Syllable Tug     ..Hangul Syllable Tuh
+    (0x0d249, 0x0d263,),  # Hangul Syllable Tweog   ..Hangul Syllable Tweoh
+    (0x0d265, 0x0d27f,),  # Hangul Syllable Tweg    ..Hangul Syllable Tweh
+    (0x0d281, 0x0d29b,),  # Hangul Syllable Twig    ..Hangul Syllable Twih
+    (0x0d29d, 0x0d2b7,),  # Hangul Syllable Tyug    ..Hangul Syllable Tyuh
+    (0x0d2b9, 0x0d2d3,),  # Hangul Syllable Teug    ..Hangul Syllable Teuh
+    (0x0d2d5, 0x0d2ef,),  # Hangul Syllable Tyig    ..Hangul Syllable Tyih
+    (0x0d2f1, 0x0d30b,),  # Hangul Syllable Tig     ..Hangul Syllable Tih
+    (0x0d30d, 0x0d327,),  # Hangul Syllable Pag     ..Hangul Syllable Pah
+    (0x0d329, 0x0d343,),  # Hangul Syllable Paeg    ..Hangul Syllable Paeh
+    (0x0d345, 0x0d35f,),  # Hangul Syllable Pyag    ..Hangul Syllable Pyah
+    (0x0d361, 0x0d37b,),  # Hangul Syllable Pyaeg   ..Hangul Syllable Pyaeh
+    (0x0d37d, 0x0d397,),  # Hangul Syllable Peog    ..Hangul Syllable Peoh
+    (0x0d399, 0x0d3b3,),  # Hangul Syllable Peg     ..Hangul Syllable Peh
+    (0x0d3b5, 0x0d3cf,),  # Hangul Syllable Pyeog   ..Hangul Syllable Pyeoh
+    (0x0d3d1, 0x0d3eb,),  # Hangul Syllable Pyeg    ..Hangul Syllable Pyeh
+    (0x0d3ed, 0x0d407,),  # Hangul Syllable Pog     ..Hangul Syllable Poh
+    (0x0d409, 0x0d423,),  # Hangul Syllable Pwag    ..Hangul Syllable Pwah
+    (0x0d425, 0x0d43f,),  # Hangul Syllable Pwaeg   ..Hangul Syllable Pwaeh
+    (0x0d441, 0x0d45b,),  # Hangul Syllable Poeg    ..Hangul Syllable Poeh
+    (0x0d45d, 0x0d477,),  # Hangul Syllable Pyog    ..Hangul Syllable Pyoh
+    (0x0d479, 0x0d493,),  # Hangul Syllable Pug     ..Hangul Syllable Puh
+    (0x0d495, 0x0d4af,),  # Hangul Syllable Pweog   ..Hangul Syllable Pweoh
+    (0x0d4b1, 0x0d4cb,),  # Hangul Syllable Pweg    ..Hangul Syllable Pweh
+    (0x0d4cd, 0x0d4e7,),  # Hangul Syllable Pwig    ..Hangul Syllable Pwih
+    (0x0d4e9, 0x0d503,),  # Hangul Syllable Pyug    ..Hangul Syllable Pyuh
+    (0x0d505, 0x0d51f,),  # Hangul Syllable Peug    ..Hangul Syllable Peuh
+    (0x0d521, 0x0d53b,),  # Hangul Syllable Pyig    ..Hangul Syllable Pyih
+    (0x0d53d, 0x0d557,),  # Hangul Syllable Pig     ..Hangul Syllable Pih
+    (0x0d559, 0x0d573,),  # Hangul Syllable Hag     ..Hangul Syllable Hah
+    (0x0d575, 0x0d58f,),  # Hangul Syllable Haeg    ..Hangul Syllable Haeh
+    (0x0d591, 0x0d5ab,),  # Hangul Syllable Hyag    ..Hangul Syllable Hyah
+    (0x0d5ad, 0x0d5c7,),  # Hangul Syllable Hyaeg   ..Hangul Syllable Hyaeh
+    (0x0d5c9, 0x0d5e3,),  # Hangul Syllable Heog    ..Hangul Syllable Heoh
+    (0x0d5e5, 0x0d5ff,),  # Hangul Syllable Heg     ..Hangul Syllable Heh
+    (0x0d601, 0x0d61b,),  # Hangul Syllable Hyeog   ..Hangul Syllable Hyeoh
+    (0x0d61d, 0x0d637,),  # Hangul Syllable Hyeg    ..Hangul Syllable Hyeh
+    (0x0d639, 0x0d653,),  # Hangul Syllable Hog     ..Hangul Syllable Hoh
+    (0x0d655, 0x0d66f,),  # Hangul Syllable Hwag    ..Hangul Syllable Hwah
+    (0x0d671, 0x0d68b,),  # Hangul Syllable Hwaeg   ..Hangul Syllable Hwaeh
+    (0x0d68d, 0x0d6a7,),  # Hangul Syllable Hoeg    ..Hangul Syllable Hoeh
+    (0x0d6a9, 0x0d6c3,),  # Hangul Syllable Hyog    ..Hangul Syllable Hyoh
+    (0x0d6c5, 0x0d6df,),  # Hangul Syllable Hug     ..Hangul Syllable Huh
+    (0x0d6e1, 0x0d6fb,),  # Hangul Syllable Hweog   ..Hangul Syllable Hweoh
+    (0x0d6fd, 0x0d717,),  # Hangul Syllable Hweg    ..Hangul Syllable Hweh
+    (0x0d719, 0x0d733,),  # Hangul Syllable Hwig    ..Hangul Syllable Hwih
+    (0x0d735, 0x0d74f,),  # Hangul Syllable Hyug    ..Hangul Syllable Hyuh
+    (0x0d751, 0x0d76b,),  # Hangul Syllable Heug    ..Hangul Syllable Heuh
+    (0x0d76d, 0x0d787,),  # Hangul Syllable Hyig    ..Hangul Syllable Hyih
+    (0x0d789, 0x0d7a3,),  # Hangul Syllable Hig     ..Hangul Syllable Hih
+)
+
+EXTENDED_PICTOGRAPHIC = (
+    # Source: emoji-data.txt
+    # Date: 2025-07-25, 17:54:31 GMT
+    #
+    (0x000a9, 0x000a9,),  # Copyright Sign
+    (0x000ae, 0x000ae,),  # Registered Sign
+    (0x0203c, 0x0203c,),  # Double Exclamation Mark
+    (0x02049, 0x02049,),  # Exclamation Question Mark
+    (0x02122, 0x02122,),  # Trade Mark Sign
+    (0x02139, 0x02139,),  # Information Source
+    (0x02194, 0x02199,),  # Left Right Arrow        ..South West Arrow
+    (0x021a9, 0x021aa,),  # Leftwards Arrow With Hoo..Rightwards Arrow With Ho
+    (0x0231a, 0x0231b,),  # Watch                   ..Hourglass
+    (0x02328, 0x02328,),  # Keyboard
+    (0x023cf, 0x023cf,),  # Eject Symbol
+    (0x023e9, 0x023f3,),  # Black Right-pointing Dou..Hourglass With Flowing S
+    (0x023f8, 0x023fa,),  # Double Vertical Bar     ..Black Circle For Record
+    (0x024c2, 0x024c2,),  # Circled Latin Capital Letter M
+    (0x025aa, 0x025ab,),  # Black Small Square      ..White Small Square
+    (0x025b6, 0x025b6,),  # Black Right-pointing Triangle
+    (0x025c0, 0x025c0,),  # Black Left-pointing Triangle
+    (0x025fb, 0x025fe,),  # White Medium Square     ..Black Medium Small Squar
+    (0x02600, 0x02604,),  # Black Sun With Rays     ..Comet
+    (0x0260e, 0x0260e,),  # Black Telephone
+    (0x02611, 0x02611,),  # Ballot Box With Check
+    (0x02614, 0x02615,),  # Umbrella With Rain Drops..Hot Beverage
+    (0x02618, 0x02618,),  # Shamrock
+    (0x0261d, 0x0261d,),  # White Up Pointing Index
+    (0x02620, 0x02620,),  # Skull And Crossbones
+    (0x02622, 0x02623,),  # Radioactive Sign        ..Biohazard Sign
+    (0x02626, 0x02626,),  # Orthodox Cross
+    (0x0262a, 0x0262a,),  # Star And Crescent
+    (0x0262e, 0x0262f,),  # Peace Symbol            ..Yin Yang
+    (0x02638, 0x0263a,),  # Wheel Of Dharma         ..White Smiling Face
+    (0x02640, 0x02640,),  # Female Sign
+    (0x02642, 0x02642,),  # Male Sign
+    (0x02648, 0x02653,),  # Aries                   ..Pisces
+    (0x0265f, 0x02660,),  # Black Chess Pawn        ..Black Spade Suit
+    (0x02663, 0x02663,),  # Black Club Suit
+    (0x02665, 0x02666,),  # Black Heart Suit        ..Black Diamond Suit
+    (0x02668, 0x02668,),  # Hot Springs
+    (0x0267b, 0x0267b,),  # Black Universal Recycling Symbol
+    (0x0267e, 0x0267f,),  # Permanent Paper Sign    ..Wheelchair Symbol
+    (0x02692, 0x02697,),  # Hammer And Pick         ..Alembic
+    (0x02699, 0x02699,),  # Gear
+    (0x0269b, 0x0269c,),  # Atom Symbol             ..Fleur-de-lis
+    (0x026a0, 0x026a1,),  # Warning Sign            ..High Voltage Sign
+    (0x026a7, 0x026a7,),  # Male With Stroke And Male And Female Sign
+    (0x026aa, 0x026ab,),  # Medium White Circle     ..Medium Black Circle
+    (0x026b0, 0x026b1,),  # Coffin                  ..Funeral Urn
+    (0x026bd, 0x026be,),  # Soccer Ball             ..Baseball
+    (0x026c4, 0x026c5,),  # Snowman Without Snow    ..Sun Behind Cloud
+    (0x026c8, 0x026c8,),  # Thunder Cloud And Rain
+    (0x026ce, 0x026cf,),  # Ophiuchus               ..Pick
+    (0x026d1, 0x026d1,),  # Helmet With White Cross
+    (0x026d3, 0x026d4,),  # Chains                  ..No Entry
+    (0x026e9, 0x026ea,),  # Shinto Shrine           ..Church
+    (0x026f0, 0x026f5,),  # Mountain                ..Sailboat
+    (0x026f7, 0x026fa,),  # Skier                   ..Tent
+    (0x026fd, 0x026fd,),  # Fuel Pump
+    (0x02702, 0x02702,),  # Black Scissors
+    (0x02705, 0x02705,),  # White Heavy Check Mark
+    (0x02708, 0x0270d,),  # Airplane                ..Writing Hand
+    (0x0270f, 0x0270f,),  # Pencil
+    (0x02712, 0x02712,),  # Black Nib
+    (0x02714, 0x02714,),  # Heavy Check Mark
+    (0x02716, 0x02716,),  # Heavy Multiplication X
+    (0x0271d, 0x0271d,),  # Latin Cross
+    (0x02721, 0x02721,),  # Star Of David
+    (0x02728, 0x02728,),  # Sparkles
+    (0x02733, 0x02734,),  # Eight Spoked Asterisk   ..Eight Pointed Black Star
+    (0x02744, 0x02744,),  # Snowflake
+    (0x02747, 0x02747,),  # Sparkle
+    (0x0274c, 0x0274c,),  # Cross Mark
+    (0x0274e, 0x0274e,),  # Negative Squared Cross Mark
+    (0x02753, 0x02755,),  # Black Question Mark Orna..White Exclamation Mark O
+    (0x02757, 0x02757,),  # Heavy Exclamation Mark Symbol
+    (0x02763, 0x02764,),  # Heavy Heart Exclamation ..Heavy Black Heart
+    (0x02795, 0x02797,),  # Heavy Plus Sign         ..Heavy Division Sign
+    (0x027a1, 0x027a1,),  # Black Rightwards Arrow
+    (0x027b0, 0x027b0,),  # Curly Loop
+    (0x027bf, 0x027bf,),  # Double Curly Loop
+    (0x02934, 0x02935,),  # Arrow Pointing Rightward..Arrow Pointing Rightward
+    (0x02b05, 0x02b07,),  # Leftwards Black Arrow   ..Downwards Black Arrow
+    (0x02b1b, 0x02b1c,),  # Black Large Square      ..White Large Square
+    (0x02b50, 0x02b50,),  # White Medium Star
+    (0x02b55, 0x02b55,),  # Heavy Large Circle
+    (0x03030, 0x03030,),  # Wavy Dash
+    (0x0303d, 0x0303d,),  # Part Alternation Mark
+    (0x03297, 0x03297,),  # Circled Ideograph Congratulation
+    (0x03299, 0x03299,),  # Circled Ideograph Secret
+    (0x1f004, 0x1f004,),  # Mahjong Tile Red Dragon
+    (0x1f02c, 0x1f02f,),  # (nil)
+    (0x1f094, 0x1f09f,),  # (nil)
+    (0x1f0af, 0x1f0b0,),  # (nil)
+    (0x1f0c0, 0x1f0c0,),  # (nil)
+    (0x1f0cf, 0x1f0d0,),  # Playing Card Black Joker..(nil)
+    (0x1f0f6, 0x1f0ff,),  # (nil)
+    (0x1f170, 0x1f171,),  # Negative Squared Latin C..Negative Squared Latin C
+    (0x1f17e, 0x1f17f,),  # Negative Squared Latin C..Negative Squared Latin C
+    (0x1f18e, 0x1f18e,),  # Negative Squared Ab
+    (0x1f191, 0x1f19a,),  # Squared Cl              ..Squared Vs
+    (0x1f1ae, 0x1f1e5,),  # (nil)
+    (0x1f201, 0x1f20f,),  # Squared Katakana Koko   ..(nil)
+    (0x1f21a, 0x1f21a,),  # Squared Cjk Unified Ideograph-7121
+    (0x1f22f, 0x1f22f,),  # Squared Cjk Unified Ideograph-6307
+    (0x1f232, 0x1f23a,),  # Squared Cjk Unified Ideo..Squared Cjk Unified Ideo
+    (0x1f23c, 0x1f23f,),  # (nil)
+    (0x1f249, 0x1f25f,),  # (nil)
+    (0x1f266, 0x1f321,),  # (nil)                   ..Thermometer
+    (0x1f324, 0x1f393,),  # White Sun With Small Clo..Graduation Cap
+    (0x1f396, 0x1f397,),  # Military Medal          ..Reminder Ribbon
+    (0x1f399, 0x1f39b,),  # Studio Microphone       ..Control Knobs
+    (0x1f39e, 0x1f3f0,),  # Film Frames             ..European Castle
+    (0x1f3f3, 0x1f3f5,),  # Waving White Flag       ..Rosette
+    (0x1f3f7, 0x1f3fa,),  # Label                   ..Amphora
+    (0x1f400, 0x1f4fd,),  # Rat                     ..Film Projector
+    (0x1f4ff, 0x1f53d,),  # Prayer Beads            ..Down-pointing Small Red
+    (0x1f549, 0x1f54e,),  # Om Symbol               ..Menorah With Nine Branch
+    (0x1f550, 0x1f567,),  # Clock Face One Oclock   ..Clock Face Twelve-thirty
+    (0x1f56f, 0x1f570,),  # Candle                  ..Mantelpiece Clock
+    (0x1f573, 0x1f57a,),  # Hole                    ..Man Dancing
+    (0x1f587, 0x1f587,),  # Linked Paperclips
+    (0x1f58a, 0x1f58d,),  # Lower Left Ballpoint Pen..Lower Left Crayon
+    (0x1f590, 0x1f590,),  # Raised Hand With Fingers Splayed
+    (0x1f595, 0x1f596,),  # Reversed Hand With Middl..Raised Hand With Part Be
+    (0x1f5a4, 0x1f5a5,),  # Black Heart             ..Desktop Computer
+    (0x1f5a8, 0x1f5a8,),  # Printer
+    (0x1f5b1, 0x1f5b2,),  # Three Button Mouse      ..Trackball
+    (0x1f5bc, 0x1f5bc,),  # Frame With Picture
+    (0x1f5c2, 0x1f5c4,),  # Card Index Dividers     ..File Cabinet
+    (0x1f5d1, 0x1f5d3,),  # Wastebasket             ..Spiral Calendar Pad
+    (0x1f5dc, 0x1f5de,),  # Compression             ..Rolled-up Newspaper
+    (0x1f5e1, 0x1f5e1,),  # Dagger Knife
+    (0x1f5e3, 0x1f5e3,),  # Speaking Head In Silhouette
+    (0x1f5e8, 0x1f5e8,),  # Left Speech Bubble
+    (0x1f5ef, 0x1f5ef,),  # Right Anger Bubble
+    (0x1f5f3, 0x1f5f3,),  # Ballot Box With Ballot
+    (0x1f5fa, 0x1f64f,),  # World Map               ..Person With Folded Hands
+    (0x1f680, 0x1f6c5,),  # Rocket                  ..Left Luggage
+    (0x1f6cb, 0x1f6d2,),  # Couch And Lamp          ..Shopping Trolley
+    (0x1f6d5, 0x1f6e5,),  # Hindu Temple            ..Motor Boat
+    (0x1f6e9, 0x1f6e9,),  # Small Airplane
+    (0x1f6eb, 0x1f6f0,),  # Airplane Departure      ..Satellite
+    (0x1f6f3, 0x1f6ff,),  # Passenger Ship          ..(nil)
+    (0x1f7da, 0x1f7ff,),  # (nil)
+    (0x1f80c, 0x1f80f,),  # (nil)
+    (0x1f848, 0x1f84f,),  # (nil)
+    (0x1f85a, 0x1f85f,),  # (nil)
+    (0x1f888, 0x1f88f,),  # (nil)
+    (0x1f8ae, 0x1f8af,),  # (nil)
+    (0x1f8bc, 0x1f8bf,),  # (nil)
+    (0x1f8c2, 0x1f8cf,),  # (nil)
+    (0x1f8d9, 0x1f8ff,),  # (nil)
+    (0x1f90c, 0x1f93a,),  # Pinched Fingers         ..Fencer
+    (0x1f93c, 0x1f945,),  # Wrestlers               ..Goal Net
+    (0x1f947, 0x1f9ff,),  # First Place Medal       ..Nazar Amulet
+    (0x1fa58, 0x1fa5f,),  # (nil)
+    (0x1fa6e, 0x1faff,),  # (nil)
+    (0x1fc00, 0x1fffd,),  # (nil)
+)
+
+INCB_LINKER = (
+    # Source: DerivedCoreProperties
+    # Date: see file
+    #
+    (0x0094d, 0x0094d,),  # Devanagari Sign Virama
+    (0x009cd, 0x009cd,),  # Bengali Sign Virama
+    (0x00acd, 0x00acd,),  # Gujarati Sign Virama
+    (0x00b4d, 0x00b4d,),  # Oriya Sign Virama
+    (0x00c4d, 0x00c4d,),  # Telugu Sign Virama
+    (0x00d4d, 0x00d4d,),  # Malayalam Sign Virama
+    (0x01039, 0x01039,),  # Myanmar Sign Virama
+    (0x017d2, 0x017d2,),  # Khmer Sign Coeng
+    (0x01a60, 0x01a60,),  # Tai Tham Sign Sakot
+    (0x01b44, 0x01b44,),  # Balinese Adeg Adeg
+    (0x01bab, 0x01bab,),  # Sundanese Sign Virama
+    (0x0a9c0, 0x0a9c0,),  # Javanese Pangkon
+    (0x0aaf6, 0x0aaf6,),  # Meetei Mayek Virama
+    (0x10a3f, 0x10a3f,),  # Kharoshthi Virama
+    (0x11133, 0x11133,),  # Chakma Virama
+    (0x113d0, 0x113d0,),  # Tulu-tigalari Conjoiner
+    (0x1193e, 0x1193e,),  # Dives Akuru Virama
+    (0x11a47, 0x11a47,),  # Zanabazar Square Subjoiner
+    (0x11a99, 0x11a99,),  # Soyombo Subjoiner
+    (0x11f42, 0x11f42,),  # Kawi Conjoiner
+)
+
+INCB_CONSONANT = (
+    # Source: DerivedCoreProperties
+    # Date: see file
+    #
+    (0x00915, 0x00939,),  # Devanagari Letter Ka    ..Devanagari Letter Ha
+    (0x00958, 0x0095f,),  # Devanagari Letter Qa    ..Devanagari Letter Yya
+    (0x00978, 0x0097f,),  # Devanagari Letter Marwar..Devanagari Letter Bba
+    (0x00995, 0x009a8,),  # Bengali Letter Ka       ..Bengali Letter Na
+    (0x009aa, 0x009b0,),  # Bengali Letter Pa       ..Bengali Letter Ra
+    (0x009b2, 0x009b2,),  # Bengali Letter La
+    (0x009b6, 0x009b9,),  # Bengali Letter Sha      ..Bengali Letter Ha
+    (0x009dc, 0x009dd,),  # Bengali Letter Rra      ..Bengali Letter Rha
+    (0x009df, 0x009df,),  # Bengali Letter Yya
+    (0x009f0, 0x009f1,),  # Bengali Letter Ra With M..Bengali Letter Ra With L
+    (0x00a95, 0x00aa8,),  # Gujarati Letter Ka      ..Gujarati Letter Na
+    (0x00aaa, 0x00ab0,),  # Gujarati Letter Pa      ..Gujarati Letter Ra
+    (0x00ab2, 0x00ab3,),  # Gujarati Letter La      ..Gujarati Letter Lla
+    (0x00ab5, 0x00ab9,),  # Gujarati Letter Va      ..Gujarati Letter Ha
+    (0x00af9, 0x00af9,),  # Gujarati Letter Zha
+    (0x00b15, 0x00b28,),  # Oriya Letter Ka         ..Oriya Letter Na
+    (0x00b2a, 0x00b30,),  # Oriya Letter Pa         ..Oriya Letter Ra
+    (0x00b32, 0x00b33,),  # Oriya Letter La         ..Oriya Letter Lla
+    (0x00b35, 0x00b39,),  # Oriya Letter Va         ..Oriya Letter Ha
+    (0x00b5c, 0x00b5d,),  # Oriya Letter Rra        ..Oriya Letter Rha
+    (0x00b5f, 0x00b5f,),  # Oriya Letter Yya
+    (0x00b71, 0x00b71,),  # Oriya Letter Wa
+    (0x00c15, 0x00c28,),  # Telugu Letter Ka        ..Telugu Letter Na
+    (0x00c2a, 0x00c39,),  # Telugu Letter Pa        ..Telugu Letter Ha
+    (0x00c58, 0x00c5a,),  # Telugu Letter Tsa       ..Telugu Letter Rrra
+    (0x00d15, 0x00d3a,),  # Malayalam Letter Ka     ..Malayalam Letter Ttta
+    (0x01000, 0x0102a,),  # Myanmar Letter Ka       ..Myanmar Letter Au
+    (0x0103f, 0x0103f,),  # Myanmar Letter Great Sa
+    (0x01050, 0x01055,),  # Myanmar Letter Sha      ..Myanmar Letter Vocalic L
+    (0x0105a, 0x0105d,),  # Myanmar Letter Mon Nga  ..Myanmar Letter Mon Bbe
+    (0x01061, 0x01061,),  # Myanmar Letter Sgaw Karen Sha
+    (0x01065, 0x01066,),  # Myanmar Letter Western P..Myanmar Letter Western P
+    (0x0106e, 0x01070,),  # Myanmar Letter Eastern P..Myanmar Letter Eastern P
+    (0x01075, 0x01081,),  # Myanmar Letter Shan Ka  ..Myanmar Letter Shan Ha
+    (0x0108e, 0x0108e,),  # Myanmar Letter Rumai Palaung Fa
+    (0x01780, 0x017b3,),  # Khmer Letter Ka         ..Khmer Independent Vowel
+    (0x01a20, 0x01a54,),  # Tai Tham Letter High Ka ..Tai Tham Letter Great Sa
+    (0x01b0b, 0x01b0c,),  # Balinese Letter Ra Repa ..Balinese Letter Ra Repa
+    (0x01b13, 0x01b33,),  # Balinese Letter Ka      ..Balinese Letter Ha
+    (0x01b45, 0x01b4c,),  # Balinese Letter Kaf Sasa..Balinese Letter Archaic
+    (0x01b83, 0x01ba0,),  # Sundanese Letter A      ..Sundanese Letter Ha
+    (0x01bae, 0x01baf,),  # Sundanese Letter Kha    ..Sundanese Letter Sya
+    (0x01bbb, 0x01bbd,),  # Sundanese Letter Reu    ..Sundanese Letter Bha
+    (0x0a989, 0x0a98b,),  # Javanese Letter Pa Cerek..Javanese Letter Nga Lele
+    (0x0a98f, 0x0a9b2,),  # Javanese Letter Ka      ..Javanese Letter Ha
+    (0x0a9e0, 0x0a9e4,),  # Myanmar Letter Shan Gha ..Myanmar Letter Shan Bha
+    (0x0a9e7, 0x0a9ef,),  # Myanmar Letter Tai Laing..Myanmar Letter Tai Laing
+    (0x0a9fa, 0x0a9fe,),  # Myanmar Letter Tai Laing..Myanmar Letter Tai Laing
+    (0x0aa60, 0x0aa6f,),  # Myanmar Letter Khamti Ga..Myanmar Letter Khamti Fa
+    (0x0aa71, 0x0aa73,),  # Myanmar Letter Khamti Xa..Myanmar Letter Khamti Ra
+    (0x0aa7a, 0x0aa7a,),  # Myanmar Letter Aiton Ra
+    (0x0aa7e, 0x0aa7f,),  # Myanmar Letter Shwe Pala..Myanmar Letter Shwe Pala
+    (0x0aae0, 0x0aaea,),  # Meetei Mayek Letter E   ..Meetei Mayek Letter Ssa
+    (0x0abc0, 0x0abda,),  # Meetei Mayek Letter Kok ..Meetei Mayek Letter Bham
+    (0x10a00, 0x10a00,),  # Kharoshthi Letter A
+    (0x10a10, 0x10a13,),  # Kharoshthi Letter Ka    ..Kharoshthi Letter Gha
+    (0x10a15, 0x10a17,),  # Kharoshthi Letter Ca    ..Kharoshthi Letter Ja
+    (0x10a19, 0x10a35,),  # Kharoshthi Letter Nya   ..Kharoshthi Letter Vha
+    (0x11103, 0x11126,),  # Chakma Letter Aa        ..Chakma Letter Haa
+    (0x11144, 0x11144,),  # Chakma Letter Lhaa
+    (0x11147, 0x11147,),  # Chakma Letter Vaa
+    (0x11380, 0x11389,),  # Tulu-tigalari Letter A  ..Tulu-tigalari Letter Voc
+    (0x1138b, 0x1138b,),  # Tulu-tigalari Letter Ee
+    (0x1138e, 0x1138e,),  # Tulu-tigalari Letter Ai
+    (0x11390, 0x113b5,),  # Tulu-tigalari Letter Oo ..Tulu-tigalari Letter Lll
+    (0x11900, 0x11906,),  # Dives Akuru Letter A    ..Dives Akuru Letter E
+    (0x11909, 0x11909,),  # Dives Akuru Letter O
+    (0x1190c, 0x11913,),  # Dives Akuru Letter Ka   ..Dives Akuru Letter Ja
+    (0x11915, 0x11916,),  # Dives Akuru Letter Nya  ..Dives Akuru Letter Tta
+    (0x11918, 0x1192f,),  # Dives Akuru Letter Dda  ..Dives Akuru Letter Za
+    (0x11a00, 0x11a00,),  # Zanabazar Square Letter A
+    (0x11a0b, 0x11a32,),  # Zanabazar Square Letter ..Zanabazar Square Letter
+    (0x11a50, 0x11a50,),  # Soyombo Letter A
+    (0x11a5c, 0x11a83,),  # Soyombo Letter Ka       ..Soyombo Letter Kssa
+    (0x11f04, 0x11f10,),  # Kawi Letter A           ..Kawi Letter O
+    (0x11f12, 0x11f33,),  # Kawi Letter Ka          ..Kawi Letter Jnya
+)
+
+INCB_EXTEND = (
+    # Source: DerivedCoreProperties
+    # Date: see file
+    #
+    (0x00300, 0x0036f,),  # Combining Grave Accent  ..Combining Latin Small Le
+    (0x00483, 0x00489,),  # Combining Cyrillic Titlo..Combining Cyrillic Milli
+    (0x00591, 0x005bd,),  # Hebrew Accent Etnahta   ..Hebrew Point Meteg
+    (0x005bf, 0x005bf,),  # Hebrew Point Rafe
+    (0x005c1, 0x005c2,),  # Hebrew Point Shin Dot   ..Hebrew Point Sin Dot
+    (0x005c4, 0x005c5,),  # Hebrew Mark Upper Dot   ..Hebrew Mark Lower Dot
+    (0x005c7, 0x005c7,),  # Hebrew Point Qamats Qatan
+    (0x00610, 0x0061a,),  # Arabic Sign Sallallahou ..Arabic Small Kasra
+    (0x0064b, 0x0065f,),  # Arabic Fathatan         ..Arabic Wavy Hamza Below
+    (0x00670, 0x00670,),  # Arabic Letter Superscript Alef
+    (0x006d6, 0x006dc,),  # Arabic Small High Ligatu..Arabic Small High Seen
+    (0x006df, 0x006e4,),  # Arabic Small High Rounde..Arabic Small High Madda
+    (0x006e7, 0x006e8,),  # Arabic Small High Yeh   ..Arabic Small High Noon
+    (0x006ea, 0x006ed,),  # Arabic Empty Centre Low ..Arabic Small Low Meem
+    (0x00711, 0x00711,),  # Syriac Letter Superscript Alaph
+    (0x00730, 0x0074a,),  # Syriac Pthaha Above     ..Syriac Barrekh
+    (0x007a6, 0x007b0,),  # Thaana Abafili          ..Thaana Sukun
+    (0x007eb, 0x007f3,),  # Nko Combining Short High..Nko Combining Double Dot
+    (0x007fd, 0x007fd,),  # Nko Dantayalan
+    (0x00816, 0x00819,),  # Samaritan Mark In       ..Samaritan Mark Dagesh
+    (0x0081b, 0x00823,),  # Samaritan Mark Epentheti..Samaritan Vowel Sign A
+    (0x00825, 0x00827,),  # Samaritan Vowel Sign Sho..Samaritan Vowel Sign U
+    (0x00829, 0x0082d,),  # Samaritan Vowel Sign Lon..Samaritan Mark Nequdaa
+    (0x00859, 0x0085b,),  # Mandaic Affrication Mark..Mandaic Gemination Mark
+    (0x00897, 0x0089f,),  # Arabic Pepet            ..Arabic Half Madda Over M
+    (0x008ca, 0x008e1,),  # Arabic Small High Farsi ..Arabic Small High Sign S
+    (0x008e3, 0x00902,),  # Arabic Turned Damma Belo..Devanagari Sign Anusvara
+    (0x0093a, 0x0093a,),  # Devanagari Vowel Sign Oe
+    (0x0093c, 0x0093c,),  # Devanagari Sign Nukta
+    (0x00941, 0x00948,),  # Devanagari Vowel Sign U ..Devanagari Vowel Sign Ai
+    (0x00951, 0x00957,),  # Devanagari Stress Sign U..Devanagari Vowel Sign Uu
+    (0x00962, 0x00963,),  # Devanagari Vowel Sign Vo..Devanagari Vowel Sign Vo
+    (0x00981, 0x00981,),  # Bengali Sign Candrabindu
+    (0x009bc, 0x009bc,),  # Bengali Sign Nukta
+    (0x009be, 0x009be,),  # Bengali Vowel Sign Aa
+    (0x009c1, 0x009c4,),  # Bengali Vowel Sign U    ..Bengali Vowel Sign Vocal
+    (0x009d7, 0x009d7,),  # Bengali Au Length Mark
+    (0x009e2, 0x009e3,),  # Bengali Vowel Sign Vocal..Bengali Vowel Sign Vocal
+    (0x009fe, 0x009fe,),  # Bengali Sandhi Mark
+    (0x00a01, 0x00a02,),  # Gurmukhi Sign Adak Bindi..Gurmukhi Sign Bindi
+    (0x00a3c, 0x00a3c,),  # Gurmukhi Sign Nukta
+    (0x00a41, 0x00a42,),  # Gurmukhi Vowel Sign U   ..Gurmukhi Vowel Sign Uu
+    (0x00a47, 0x00a48,),  # Gurmukhi Vowel Sign Ee  ..Gurmukhi Vowel Sign Ai
+    (0x00a4b, 0x00a4d,),  # Gurmukhi Vowel Sign Oo  ..Gurmukhi Sign Virama
+    (0x00a51, 0x00a51,),  # Gurmukhi Sign Udaat
+    (0x00a70, 0x00a71,),  # Gurmukhi Tippi          ..Gurmukhi Addak
+    (0x00a75, 0x00a75,),  # Gurmukhi Sign Yakash
+    (0x00a81, 0x00a82,),  # Gujarati Sign Candrabind..Gujarati Sign Anusvara
+    (0x00abc, 0x00abc,),  # Gujarati Sign Nukta
+    (0x00ac1, 0x00ac5,),  # Gujarati Vowel Sign U   ..Gujarati Vowel Sign Cand
+    (0x00ac7, 0x00ac8,),  # Gujarati Vowel Sign E   ..Gujarati Vowel Sign Ai
+    (0x00ae2, 0x00ae3,),  # Gujarati Vowel Sign Voca..Gujarati Vowel Sign Voca
+    (0x00afa, 0x00aff,),  # Gujarati Sign Sukun     ..Gujarati Sign Two-circle
+    (0x00b01, 0x00b01,),  # Oriya Sign Candrabindu
+    (0x00b3c, 0x00b3c,),  # Oriya Sign Nukta
+    (0x00b3e, 0x00b3f,),  # Oriya Vowel Sign Aa     ..Oriya Vowel Sign I
+    (0x00b41, 0x00b44,),  # Oriya Vowel Sign U      ..Oriya Vowel Sign Vocalic
+    (0x00b55, 0x00b57,),  # Oriya Sign Overline     ..Oriya Au Length Mark
+    (0x00b62, 0x00b63,),  # Oriya Vowel Sign Vocalic..Oriya Vowel Sign Vocalic
+    (0x00b82, 0x00b82,),  # Tamil Sign Anusvara
+    (0x00bbe, 0x00bbe,),  # Tamil Vowel Sign Aa
+    (0x00bc0, 0x00bc0,),  # Tamil Vowel Sign Ii
+    (0x00bcd, 0x00bcd,),  # Tamil Sign Virama
+    (0x00bd7, 0x00bd7,),  # Tamil Au Length Mark
+    (0x00c00, 0x00c00,),  # Telugu Sign Combining Candrabindu Above
+    (0x00c04, 0x00c04,),  # Telugu Sign Combining Anusvara Above
+    (0x00c3c, 0x00c3c,),  # Telugu Sign Nukta
+    (0x00c3e, 0x00c40,),  # Telugu Vowel Sign Aa    ..Telugu Vowel Sign Ii
+    (0x00c46, 0x00c48,),  # Telugu Vowel Sign E     ..Telugu Vowel Sign Ai
+    (0x00c4a, 0x00c4c,),  # Telugu Vowel Sign O     ..Telugu Vowel Sign Au
+    (0x00c55, 0x00c56,),  # Telugu Length Mark      ..Telugu Ai Length Mark
+    (0x00c62, 0x00c63,),  # Telugu Vowel Sign Vocali..Telugu Vowel Sign Vocali
+    (0x00c81, 0x00c81,),  # Kannada Sign Candrabindu
+    (0x00cbc, 0x00cbc,),  # Kannada Sign Nukta
+    (0x00cbf, 0x00cc0,),  # Kannada Vowel Sign I    ..Kannada Vowel Sign Ii
+    (0x00cc2, 0x00cc2,),  # Kannada Vowel Sign Uu
+    (0x00cc6, 0x00cc8,),  # Kannada Vowel Sign E    ..Kannada Vowel Sign Ai
+    (0x00cca, 0x00ccd,),  # Kannada Vowel Sign O    ..Kannada Sign Virama
+    (0x00cd5, 0x00cd6,),  # Kannada Length Mark     ..Kannada Ai Length Mark
+    (0x00ce2, 0x00ce3,),  # Kannada Vowel Sign Vocal..Kannada Vowel Sign Vocal
+    (0x00d00, 0x00d01,),  # Malayalam Sign Combining..Malayalam Sign Candrabin
+    (0x00d3b, 0x00d3c,),  # Malayalam Sign Vertical ..Malayalam Sign Circular
+    (0x00d3e, 0x00d3e,),  # Malayalam Vowel Sign Aa
+    (0x00d41, 0x00d44,),  # Malayalam Vowel Sign U  ..Malayalam Vowel Sign Voc
+    (0x00d57, 0x00d57,),  # Malayalam Au Length Mark
+    (0x00d62, 0x00d63,),  # Malayalam Vowel Sign Voc..Malayalam Vowel Sign Voc
+    (0x00d81, 0x00d81,),  # Sinhala Sign Candrabindu
+    (0x00dca, 0x00dca,),  # Sinhala Sign Al-lakuna
+    (0x00dcf, 0x00dcf,),  # Sinhala Vowel Sign Aela-pilla
+    (0x00dd2, 0x00dd4,),  # Sinhala Vowel Sign Ketti..Sinhala Vowel Sign Ketti
+    (0x00dd6, 0x00dd6,),  # Sinhala Vowel Sign Diga Paa-pilla
+    (0x00ddf, 0x00ddf,),  # Sinhala Vowel Sign Gayanukitta
+    (0x00e31, 0x00e31,),  # Thai Character Mai Han-akat
+    (0x00e34, 0x00e3a,),  # Thai Character Sara I   ..Thai Character Phinthu
+    (0x00e47, 0x00e4e,),  # Thai Character Maitaikhu..Thai Character Yamakkan
+    (0x00eb1, 0x00eb1,),  # Lao Vowel Sign Mai Kan
+    (0x00eb4, 0x00ebc,),  # Lao Vowel Sign I        ..Lao Semivowel Sign Lo
+    (0x00ec8, 0x00ece,),  # Lao Tone Mai Ek         ..Lao Yamakkan
+    (0x00f18, 0x00f19,),  # Tibetan Astrological Sig..Tibetan Astrological Sig
+    (0x00f35, 0x00f35,),  # Tibetan Mark Ngas Bzung Nyi Zla
+    (0x00f37, 0x00f37,),  # Tibetan Mark Ngas Bzung Sgor Rtags
+    (0x00f39, 0x00f39,),  # Tibetan Mark Tsa -phru
+    (0x00f71, 0x00f7e,),  # Tibetan Vowel Sign Aa   ..Tibetan Sign Rjes Su Nga
+    (0x00f80, 0x00f84,),  # Tibetan Vowel Sign Rever..Tibetan Mark Halanta
+    (0x00f86, 0x00f87,),  # Tibetan Sign Lci Rtags  ..Tibetan Sign Yang Rtags
+    (0x00f8d, 0x00f97,),  # Tibetan Subjoined Sign L..Tibetan Subjoined Letter
+    (0x00f99, 0x00fbc,),  # Tibetan Subjoined Letter..Tibetan Subjoined Letter
+    (0x00fc6, 0x00fc6,),  # Tibetan Symbol Padma Gdan
+    (0x0102d, 0x01030,),  # Myanmar Vowel Sign I    ..Myanmar Vowel Sign Uu
+    (0x01032, 0x01037,),  # Myanmar Vowel Sign Ai   ..Myanmar Sign Dot Below
+    (0x0103a, 0x0103a,),  # Myanmar Sign Asat
+    (0x0103d, 0x0103e,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+    (0x01058, 0x01059,),  # Myanmar Vowel Sign Vocal..Myanmar Vowel Sign Vocal
+    (0x0105e, 0x01060,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+    (0x01071, 0x01074,),  # Myanmar Vowel Sign Geba ..Myanmar Vowel Sign Kayah
+    (0x01082, 0x01082,),  # Myanmar Consonant Sign Shan Medial Wa
+    (0x01085, 0x01086,),  # Myanmar Vowel Sign Shan ..Myanmar Vowel Sign Shan
+    (0x0108d, 0x0108d,),  # Myanmar Sign Shan Council Emphatic Tone
+    (0x0109d, 0x0109d,),  # Myanmar Vowel Sign Aiton Ai
+    (0x0135d, 0x0135f,),  # Ethiopic Combining Gemin..Ethiopic Combining Gemin
+    (0x01712, 0x01715,),  # Tagalog Vowel Sign I    ..Tagalog Sign Pamudpod
+    (0x01732, 0x01734,),  # Hanunoo Vowel Sign I    ..Hanunoo Sign Pamudpod
+    (0x01752, 0x01753,),  # Buhid Vowel Sign I      ..Buhid Vowel Sign U
+    (0x01772, 0x01773,),  # Tagbanwa Vowel Sign I   ..Tagbanwa Vowel Sign U
+    (0x017b4, 0x017b5,),  # Khmer Vowel Inherent Aq ..Khmer Vowel Inherent Aa
+    (0x017b7, 0x017bd,),  # Khmer Vowel Sign I      ..Khmer Vowel Sign Ua
+    (0x017c6, 0x017c6,),  # Khmer Sign Nikahit
+    (0x017c9, 0x017d1,),  # Khmer Sign Muusikatoan  ..Khmer Sign Viriam
+    (0x017d3, 0x017d3,),  # Khmer Sign Bathamasat
+    (0x017dd, 0x017dd,),  # Khmer Sign Atthacan
+    (0x0180b, 0x0180d,),  # Mongolian Free Variation..Mongolian Free Variation
+    (0x0180f, 0x0180f,),  # Mongolian Free Variation Selector Four
+    (0x01885, 0x01886,),  # Mongolian Letter Ali Gal..Mongolian Letter Ali Gal
+    (0x018a9, 0x018a9,),  # Mongolian Letter Ali Gali Dagalga
+    (0x01920, 0x01922,),  # Limbu Vowel Sign A      ..Limbu Vowel Sign U
+    (0x01927, 0x01928,),  # Limbu Vowel Sign E      ..Limbu Vowel Sign O
+    (0x01932, 0x01932,),  # Limbu Small Letter Anusvara
+    (0x01939, 0x0193b,),  # Limbu Sign Mukphreng    ..Limbu Sign Sa-i
+    (0x01a17, 0x01a18,),  # Buginese Vowel Sign I   ..Buginese Vowel Sign U
+    (0x01a1b, 0x01a1b,),  # Buginese Vowel Sign Ae
+    (0x01a56, 0x01a56,),  # Tai Tham Consonant Sign Medial La
+    (0x01a58, 0x01a5e,),  # Tai Tham Sign Mai Kang L..Tai Tham Consonant Sign
+    (0x01a62, 0x01a62,),  # Tai Tham Vowel Sign Mai Sat
+    (0x01a65, 0x01a6c,),  # Tai Tham Vowel Sign I   ..Tai Tham Vowel Sign Oa B
+    (0x01a73, 0x01a7c,),  # Tai Tham Vowel Sign Oa A..Tai Tham Sign Khuen-lue
+    (0x01a7f, 0x01a7f,),  # Tai Tham Combining Cryptogrammic Dot
+    (0x01ab0, 0x01add,),  # Combining Doubled Circum..Combining Dot-and-ring B
+    (0x01ae0, 0x01aeb,),  # Combining Left Tack Abov..Combining Double Rightwa
+    (0x01b00, 0x01b03,),  # Balinese Sign Ulu Ricem ..Balinese Sign Surang
+    (0x01b34, 0x01b3d,),  # Balinese Sign Rerekan   ..Balinese Vowel Sign La L
+    (0x01b42, 0x01b43,),  # Balinese Vowel Sign Pepe..Balinese Vowel Sign Pepe
+    (0x01b6b, 0x01b73,),  # Balinese Musical Symbol ..Balinese Musical Symbol
+    (0x01b80, 0x01b81,),  # Sundanese Sign Panyecek ..Sundanese Sign Panglayar
+    (0x01ba2, 0x01ba5,),  # Sundanese Consonant Sign..Sundanese Vowel Sign Pan
+    (0x01ba8, 0x01baa,),  # Sundanese Vowel Sign Pam..Sundanese Sign Pamaaeh
+    (0x01bac, 0x01bad,),  # Sundanese Consonant Sign..Sundanese Consonant Sign
+    (0x01be6, 0x01be6,),  # Batak Sign Tompi
+    (0x01be8, 0x01be9,),  # Batak Vowel Sign Pakpak ..Batak Vowel Sign Ee
+    (0x01bed, 0x01bed,),  # Batak Vowel Sign Karo O
+    (0x01bef, 0x01bf3,),  # Batak Vowel Sign U For S..Batak Panongonan
+    (0x01c2c, 0x01c33,),  # Lepcha Vowel Sign E     ..Lepcha Consonant Sign T
+    (0x01c36, 0x01c37,),  # Lepcha Sign Ran         ..Lepcha Sign Nukta
+    (0x01cd0, 0x01cd2,),  # Vedic Tone Karshana     ..Vedic Tone Prenkha
+    (0x01cd4, 0x01ce0,),  # Vedic Sign Yajurvedic Mi..Vedic Tone Rigvedic Kash
+    (0x01ce2, 0x01ce8,),  # Vedic Sign Visarga Svari..Vedic Sign Visarga Anuda
+    (0x01ced, 0x01ced,),  # Vedic Sign Tiryak
+    (0x01cf4, 0x01cf4,),  # Vedic Tone Candra Above
+    (0x01cf8, 0x01cf9,),  # Vedic Tone Ring Above   ..Vedic Tone Double Ring A
+    (0x01dc0, 0x01dff,),  # Combining Dotted Grave A..Combining Right Arrowhea
+    (0x0200d, 0x0200d,),  # Zero Width Joiner
+    (0x020d0, 0x020f0,),  # Combining Left Harpoon A..Combining Asterisk Above
+    (0x02cef, 0x02cf1,),  # Coptic Combining Ni Abov..Coptic Combining Spiritu
+    (0x02d7f, 0x02d7f,),  # Tifinagh Consonant Joiner
+    (0x02de0, 0x02dff,),  # Combining Cyrillic Lette..Combining Cyrillic Lette
+    (0x0302a, 0x0302f,),  # Ideographic Level Tone M..Hangul Double Dot Tone M
+    (0x03099, 0x0309a,),  # Combining Katakana-hirag..Combining Katakana-hirag
+    (0x0a66f, 0x0a672,),  # Combining Cyrillic Vzmet..Combining Cyrillic Thous
+    (0x0a674, 0x0a67d,),  # Combining Cyrillic Lette..Combining Cyrillic Payer
+    (0x0a69e, 0x0a69f,),  # Combining Cyrillic Lette..Combining Cyrillic Lette
+    (0x0a6f0, 0x0a6f1,),  # Bamum Combining Mark Koq..Bamum Combining Mark Tuk
+    (0x0a802, 0x0a802,),  # Syloti Nagri Sign Dvisvara
+    (0x0a806, 0x0a806,),  # Syloti Nagri Sign Hasanta
+    (0x0a80b, 0x0a80b,),  # Syloti Nagri Sign Anusvara
+    (0x0a825, 0x0a826,),  # Syloti Nagri Vowel Sign ..Syloti Nagri Vowel Sign
+    (0x0a82c, 0x0a82c,),  # Syloti Nagri Sign Alternate Hasanta
+    (0x0a8c4, 0x0a8c5,),  # Saurashtra Sign Virama  ..Saurashtra Sign Candrabi
+    (0x0a8e0, 0x0a8f1,),  # Combining Devanagari Dig..Combining Devanagari Sig
+    (0x0a8ff, 0x0a8ff,),  # Devanagari Vowel Sign Ay
+    (0x0a926, 0x0a92d,),  # Kayah Li Vowel Ue       ..Kayah Li Tone Calya Plop
+    (0x0a947, 0x0a951,),  # Rejang Vowel Sign I     ..Rejang Consonant Sign R
+    (0x0a953, 0x0a953,),  # Rejang Virama
+    (0x0a980, 0x0a982,),  # Javanese Sign Panyangga ..Javanese Sign Layar
+    (0x0a9b3, 0x0a9b3,),  # Javanese Sign Cecak Telu
+    (0x0a9b6, 0x0a9b9,),  # Javanese Vowel Sign Wulu..Javanese Vowel Sign Suku
+    (0x0a9bc, 0x0a9bd,),  # Javanese Vowel Sign Pepe..Javanese Consonant Sign
+    (0x0a9e5, 0x0a9e5,),  # Myanmar Sign Shan Saw
+    (0x0aa29, 0x0aa2e,),  # Cham Vowel Sign Aa      ..Cham Vowel Sign Oe
+    (0x0aa31, 0x0aa32,),  # Cham Vowel Sign Au      ..Cham Vowel Sign Ue
+    (0x0aa35, 0x0aa36,),  # Cham Consonant Sign La  ..Cham Consonant Sign Wa
+    (0x0aa43, 0x0aa43,),  # Cham Consonant Sign Final Ng
+    (0x0aa4c, 0x0aa4c,),  # Cham Consonant Sign Final M
+    (0x0aa7c, 0x0aa7c,),  # Myanmar Sign Tai Laing Tone-2
+    (0x0aab0, 0x0aab0,),  # Tai Viet Mai Kang
+    (0x0aab2, 0x0aab4,),  # Tai Viet Vowel I        ..Tai Viet Vowel U
+    (0x0aab7, 0x0aab8,),  # Tai Viet Mai Khit       ..Tai Viet Vowel Ia
+    (0x0aabe, 0x0aabf,),  # Tai Viet Vowel Am       ..Tai Viet Tone Mai Ek
+    (0x0aac1, 0x0aac1,),  # Tai Viet Tone Mai Tho
+    (0x0aaec, 0x0aaed,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+    (0x0abe5, 0x0abe5,),  # Meetei Mayek Vowel Sign Anap
+    (0x0abe8, 0x0abe8,),  # Meetei Mayek Vowel Sign Unap
+    (0x0abed, 0x0abed,),  # Meetei Mayek Apun Iyek
+    (0x0fb1e, 0x0fb1e,),  # Hebrew Point Judeo-spanish Varika
+    (0x0fe00, 0x0fe0f,),  # Variation Selector-1    ..Variation Selector-16
+    (0x0fe20, 0x0fe2f,),  # Combining Ligature Left ..Combining Cyrillic Titlo
+    (0x0ff9e, 0x0ff9f,),  # Halfwidth Katakana Voice..Halfwidth Katakana Semi-
+    (0x101fd, 0x101fd,),  # Phaistos Disc Sign Combining Oblique Stroke
+    (0x102e0, 0x102e0,),  # Coptic Epact Thousands Mark
+    (0x10376, 0x1037a,),  # Combining Old Permic Let..Combining Old Permic Let
+    (0x10a01, 0x10a03,),  # Kharoshthi Vowel Sign I ..Kharoshthi Vowel Sign Vo
+    (0x10a05, 0x10a06,),  # Kharoshthi Vowel Sign E ..Kharoshthi Vowel Sign O
+    (0x10a0c, 0x10a0f,),  # Kharoshthi Vowel Length ..Kharoshthi Sign Visarga
+    (0x10a38, 0x10a3a,),  # Kharoshthi Sign Bar Abov..Kharoshthi Sign Dot Belo
+    (0x10ae5, 0x10ae6,),  # Manichaean Abbreviation ..Manichaean Abbreviation
+    (0x10d24, 0x10d27,),  # Hanifi Rohingya Sign Har..Hanifi Rohingya Sign Tas
+    (0x10d69, 0x10d6d,),  # Garay Vowel Sign E      ..Garay Consonant Nasaliza
+    (0x10eab, 0x10eac,),  # Yezidi Combining Hamza M..Yezidi Combining Madda M
+    (0x10efa, 0x10eff,),  # Arabic Double Vertical B..Arabic Small Low Word Ma
+    (0x10f46, 0x10f50,),  # Sogdian Combining Dot Be..Sogdian Combining Stroke
+    (0x10f82, 0x10f85,),  # Old Uyghur Combining Dot..Old Uyghur Combining Two
+    (0x11001, 0x11001,),  # Brahmi Sign Anusvara
+    (0x11038, 0x11046,),  # Brahmi Vowel Sign Aa    ..Brahmi Virama
+    (0x11070, 0x11070,),  # Brahmi Sign Old Tamil Virama
+    (0x11073, 0x11074,),  # Brahmi Vowel Sign Old Ta..Brahmi Vowel Sign Old Ta
+    (0x1107f, 0x11081,),  # Brahmi Number Joiner    ..Kaithi Sign Anusvara
+    (0x110b3, 0x110b6,),  # Kaithi Vowel Sign U     ..Kaithi Vowel Sign Ai
+    (0x110b9, 0x110ba,),  # Kaithi Sign Virama      ..Kaithi Sign Nukta
+    (0x110c2, 0x110c2,),  # Kaithi Vowel Sign Vocalic R
+    (0x11100, 0x11102,),  # Chakma Sign Candrabindu ..Chakma Sign Visarga
+    (0x11127, 0x1112b,),  # Chakma Vowel Sign A     ..Chakma Vowel Sign Uu
+    (0x1112d, 0x11132,),  # Chakma Vowel Sign Ai    ..Chakma Au Mark
+    (0x11134, 0x11134,),  # Chakma Maayyaa
+    (0x11173, 0x11173,),  # Mahajani Sign Nukta
+    (0x11180, 0x11181,),  # Sharada Sign Candrabindu..Sharada Sign Anusvara
+    (0x111b6, 0x111be,),  # Sharada Vowel Sign U    ..Sharada Vowel Sign O
+    (0x111c0, 0x111c0,),  # Sharada Sign Virama
+    (0x111c9, 0x111cc,),  # Sharada Sandhi Mark     ..Sharada Extra Short Vowe
+    (0x111cf, 0x111cf,),  # Sharada Sign Inverted Candrabindu
+    (0x1122f, 0x11231,),  # Khojki Vowel Sign U     ..Khojki Vowel Sign Ai
+    (0x11234, 0x11237,),  # Khojki Sign Anusvara    ..Khojki Sign Shadda
+    (0x1123e, 0x1123e,),  # Khojki Sign Sukun
+    (0x11241, 0x11241,),  # Khojki Vowel Sign Vocalic R
+    (0x112df, 0x112df,),  # Khudawadi Sign Anusvara
+    (0x112e3, 0x112ea,),  # Khudawadi Vowel Sign U  ..Khudawadi Sign Virama
+    (0x11300, 0x11301,),  # Grantha Sign Combining A..Grantha Sign Candrabindu
+    (0x1133b, 0x1133c,),  # Combining Bindu Below   ..Grantha Sign Nukta
+    (0x1133e, 0x1133e,),  # Grantha Vowel Sign Aa
+    (0x11340, 0x11340,),  # Grantha Vowel Sign Ii
+    (0x1134d, 0x1134d,),  # Grantha Sign Virama
+    (0x11357, 0x11357,),  # Grantha Au Length Mark
+    (0x11366, 0x1136c,),  # Combining Grantha Digit ..Combining Grantha Digit
+    (0x11370, 0x11374,),  # Combining Grantha Letter..Combining Grantha Letter
+    (0x113b8, 0x113b8,),  # Tulu-tigalari Vowel Sign Aa
+    (0x113bb, 0x113c0,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Vowel Sign
+    (0x113c2, 0x113c2,),  # Tulu-tigalari Vowel Sign Ee
+    (0x113c5, 0x113c5,),  # Tulu-tigalari Vowel Sign Ai
+    (0x113c7, 0x113c9,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Au Length
+    (0x113ce, 0x113cf,),  # Tulu-tigalari Sign Viram..Tulu-tigalari Sign Loope
+    (0x113d2, 0x113d2,),  # Tulu-tigalari Gemination Mark
+    (0x113e1, 0x113e2,),  # Tulu-tigalari Vedic Tone..Tulu-tigalari Vedic Tone
+    (0x11438, 0x1143f,),  # Newa Vowel Sign U       ..Newa Vowel Sign Ai
+    (0x11442, 0x11444,),  # Newa Sign Virama        ..Newa Sign Anusvara
+    (0x11446, 0x11446,),  # Newa Sign Nukta
+    (0x1145e, 0x1145e,),  # Newa Sandhi Mark
+    (0x114b0, 0x114b0,),  # Tirhuta Vowel Sign Aa
+    (0x114b3, 0x114b8,),  # Tirhuta Vowel Sign U    ..Tirhuta Vowel Sign Vocal
+    (0x114ba, 0x114ba,),  # Tirhuta Vowel Sign Short E
+    (0x114bd, 0x114bd,),  # Tirhuta Vowel Sign Short O
+    (0x114bf, 0x114c0,),  # Tirhuta Sign Candrabindu..Tirhuta Sign Anusvara
+    (0x114c2, 0x114c3,),  # Tirhuta Sign Virama     ..Tirhuta Sign Nukta
+    (0x115af, 0x115af,),  # Siddham Vowel Sign Aa
+    (0x115b2, 0x115b5,),  # Siddham Vowel Sign U    ..Siddham Vowel Sign Vocal
+    (0x115bc, 0x115bd,),  # Siddham Sign Candrabindu..Siddham Sign Anusvara
+    (0x115bf, 0x115c0,),  # Siddham Sign Virama     ..Siddham Sign Nukta
+    (0x115dc, 0x115dd,),  # Siddham Vowel Sign Alter..Siddham Vowel Sign Alter
+    (0x11633, 0x1163a,),  # Modi Vowel Sign U       ..Modi Vowel Sign Ai
+    (0x1163d, 0x1163d,),  # Modi Sign Anusvara
+    (0x1163f, 0x11640,),  # Modi Sign Virama        ..Modi Sign Ardhacandra
+    (0x116ab, 0x116ab,),  # Takri Sign Anusvara
+    (0x116ad, 0x116ad,),  # Takri Vowel Sign Aa
+    (0x116b0, 0x116b7,),  # Takri Vowel Sign U      ..Takri Sign Nukta
+    (0x1171d, 0x1171d,),  # Ahom Consonant Sign Medial La
+    (0x1171f, 0x1171f,),  # Ahom Consonant Sign Medial Ligating Ra
+    (0x11722, 0x11725,),  # Ahom Vowel Sign I       ..Ahom Vowel Sign Uu
+    (0x11727, 0x1172b,),  # Ahom Vowel Sign Aw      ..Ahom Sign Killer
+    (0x1182f, 0x11837,),  # Dogra Vowel Sign U      ..Dogra Sign Anusvara
+    (0x11839, 0x1183a,),  # Dogra Sign Virama       ..Dogra Sign Nukta
+    (0x11930, 0x11930,),  # Dives Akuru Vowel Sign Aa
+    (0x1193b, 0x1193d,),  # Dives Akuru Sign Anusvar..Dives Akuru Sign Halanta
+    (0x11943, 0x11943,),  # Dives Akuru Sign Nukta
+    (0x119d4, 0x119d7,),  # Nandinagari Vowel Sign U..Nandinagari Vowel Sign V
+    (0x119da, 0x119db,),  # Nandinagari Vowel Sign E..Nandinagari Vowel Sign A
+    (0x119e0, 0x119e0,),  # Nandinagari Sign Virama
+    (0x11a01, 0x11a0a,),  # Zanabazar Square Vowel S..Zanabazar Square Vowel L
+    (0x11a33, 0x11a38,),  # Zanabazar Square Final C..Zanabazar Square Sign An
+    (0x11a3b, 0x11a3e,),  # Zanabazar Square Cluster..Zanabazar Square Cluster
+    (0x11a51, 0x11a56,),  # Soyombo Vowel Sign I    ..Soyombo Vowel Sign Oe
+    (0x11a59, 0x11a5b,),  # Soyombo Vowel Sign Vocal..Soyombo Vowel Length Mar
+    (0x11a8a, 0x11a96,),  # Soyombo Final Consonant ..Soyombo Sign Anusvara
+    (0x11a98, 0x11a98,),  # Soyombo Gemination Mark
+    (0x11b60, 0x11b60,),  # Sharada Vowel Sign Oe
+    (0x11b62, 0x11b64,),  # Sharada Vowel Sign Ue   ..Sharada Vowel Sign Short
+    (0x11b66, 0x11b66,),  # Sharada Vowel Sign Candra E
+    (0x11c30, 0x11c36,),  # Bhaiksuki Vowel Sign I  ..Bhaiksuki Vowel Sign Voc
+    (0x11c38, 0x11c3d,),  # Bhaiksuki Vowel Sign E  ..Bhaiksuki Sign Anusvara
+    (0x11c3f, 0x11c3f,),  # Bhaiksuki Sign Virama
+    (0x11c92, 0x11ca7,),  # Marchen Subjoined Letter..Marchen Subjoined Letter
+    (0x11caa, 0x11cb0,),  # Marchen Subjoined Letter..Marchen Vowel Sign Aa
+    (0x11cb2, 0x11cb3,),  # Marchen Vowel Sign U    ..Marchen Vowel Sign E
+    (0x11cb5, 0x11cb6,),  # Marchen Sign Anusvara   ..Marchen Sign Candrabindu
+    (0x11d31, 0x11d36,),  # Masaram Gondi Vowel Sign..Masaram Gondi Vowel Sign
+    (0x11d3a, 0x11d3a,),  # Masaram Gondi Vowel Sign E
+    (0x11d3c, 0x11d3d,),  # Masaram Gondi Vowel Sign..Masaram Gondi Vowel Sign
+    (0x11d3f, 0x11d45,),  # Masaram Gondi Vowel Sign..Masaram Gondi Virama
+    (0x11d47, 0x11d47,),  # Masaram Gondi Ra-kara
+    (0x11d90, 0x11d91,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+    (0x11d95, 0x11d95,),  # Gunjala Gondi Sign Anusvara
+    (0x11d97, 0x11d97,),  # Gunjala Gondi Virama
+    (0x11ef3, 0x11ef4,),  # Makasar Vowel Sign I    ..Makasar Vowel Sign U
+    (0x11f00, 0x11f01,),  # Kawi Sign Candrabindu   ..Kawi Sign Anusvara
+    (0x11f36, 0x11f3a,),  # Kawi Vowel Sign I       ..Kawi Vowel Sign Vocalic
+    (0x11f40, 0x11f41,),  # Kawi Vowel Sign Eu      ..Kawi Sign Killer
+    (0x11f5a, 0x11f5a,),  # Kawi Sign Nukta
+    (0x13440, 0x13440,),  # Egyptian Hieroglyph Mirror Horizontally
+    (0x13447, 0x13455,),  # Egyptian Hieroglyph Modi..Egyptian Hieroglyph Modi
+    (0x1611e, 0x16129,),  # Gurung Khema Vowel Sign ..Gurung Khema Vowel Lengt
+    (0x1612d, 0x1612f,),  # Gurung Khema Sign Anusva..Gurung Khema Sign Tholho
+    (0x16af0, 0x16af4,),  # Bassa Vah Combining High..Bassa Vah Combining High
+    (0x16b30, 0x16b36,),  # Pahawh Hmong Mark Cim Tu..Pahawh Hmong Mark Cim Ta
+    (0x16f4f, 0x16f4f,),  # Miao Sign Consonant Modifier Bar
+    (0x16f8f, 0x16f92,),  # Miao Tone Right         ..Miao Tone Below
+    (0x16fe4, 0x16fe4,),  # Khitan Small Script Filler
+    (0x16ff0, 0x16ff1,),  # Vietnamese Alternate Rea..Vietnamese Alternate Rea
+    (0x1bc9d, 0x1bc9e,),  # Duployan Thick Letter Se..Duployan Double Mark
+    (0x1cf00, 0x1cf2d,),  # Znamenny Combining Mark ..Znamenny Combining Mark
+    (0x1cf30, 0x1cf46,),  # Znamenny Combining Tonal..Znamenny Priznak Modifie
+    (0x1d165, 0x1d169,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d16d, 0x1d172,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d17b, 0x1d182,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d185, 0x1d18b,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d1aa, 0x1d1ad,),  # Musical Symbol Combining..Musical Symbol Combining
+    (0x1d242, 0x1d244,),  # Combining Greek Musical ..Combining Greek Musical
+    (0x1da00, 0x1da36,),  # Signwriting Head Rim    ..Signwriting Air Sucking
+    (0x1da3b, 0x1da6c,),  # Signwriting Mouth Closed..Signwriting Excitement
+    (0x1da75, 0x1da75,),  # Signwriting Upper Body Tilting From Hip Joints
+    (0x1da84, 0x1da84,),  # Signwriting Location Head Neck
+    (0x1da9b, 0x1da9f,),  # Signwriting Fill Modifie..Signwriting Fill Modifie
+    (0x1daa1, 0x1daaf,),  # Signwriting Rotation Mod..Signwriting Rotation Mod
+    (0x1e000, 0x1e006,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e008, 0x1e018,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e01b, 0x1e021,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e023, 0x1e024,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e026, 0x1e02a,),  # Combining Glagolitic Let..Combining Glagolitic Let
+    (0x1e08f, 0x1e08f,),  # Combining Cyrillic Small Letter Byelorussian-ukr
+    (0x1e130, 0x1e136,),  # Nyiakeng Puachue Hmong T..Nyiakeng Puachue Hmong T
+    (0x1e2ae, 0x1e2ae,),  # Toto Sign Rising Tone
+    (0x1e2ec, 0x1e2ef,),  # Wancho Tone Tup         ..Wancho Tone Koini
+    (0x1e4ec, 0x1e4ef,),  # Nag Mundari Sign Muhor  ..Nag Mundari Sign Sutuh
+    (0x1e5ee, 0x1e5ef,),  # Ol Onal Sign Mu         ..Ol Onal Sign Ikir
+    (0x1e6e3, 0x1e6e3,),  # Tai Yo Sign Ue
+    (0x1e6e6, 0x1e6e6,),  # Tai Yo Sign Au
+    (0x1e6ee, 0x1e6ef,),  # Tai Yo Sign Ay          ..Tai Yo Sign Ang
+    (0x1e6f5, 0x1e6f5,),  # Tai Yo Sign Om
+    (0x1e8d0, 0x1e8d6,),  # Mende Kikakui Combining ..Mende Kikakui Combining
+    (0x1e944, 0x1e94a,),  # Adlam Alif Lengthener   ..Adlam Nukta
+    (0x1f3fb, 0x1f3ff,),  # Emoji Modifier Fitzpatri..Emoji Modifier Fitzpatri
+    (0xe0020, 0xe007f,),  # Tag Space               ..Cancel Tag
+    (0xe0100, 0xe01ef,),  # Variation Selector-17   ..Variation Selector-256
+)
+
+ISC_CONSONANT = (
+    # Source: IndicSyllabicCategory
+    # Date: see file
+    #
+    (0x00915, 0x00939,),  # Devanagari Letter Ka    ..Devanagari Letter Ha
+    (0x00958, 0x0095f,),  # Devanagari Letter Qa    ..Devanagari Letter Yya
+    (0x00978, 0x0097f,),  # Devanagari Letter Marwar..Devanagari Letter Bba
+    (0x00995, 0x009a8,),  # Bengali Letter Ka       ..Bengali Letter Na
+    (0x009aa, 0x009b0,),  # Bengali Letter Pa       ..Bengali Letter Ra
+    (0x009b2, 0x009b2,),  # Bengali Letter La
+    (0x009b6, 0x009b9,),  # Bengali Letter Sha      ..Bengali Letter Ha
+    (0x009dc, 0x009dd,),  # Bengali Letter Rra      ..Bengali Letter Rha
+    (0x009df, 0x009df,),  # Bengali Letter Yya
+    (0x009f0, 0x009f1,),  # Bengali Letter Ra With M..Bengali Letter Ra With L
+    (0x00a15, 0x00a28,),  # Gurmukhi Letter Ka      ..Gurmukhi Letter Na
+    (0x00a2a, 0x00a30,),  # Gurmukhi Letter Pa      ..Gurmukhi Letter Ra
+    (0x00a32, 0x00a33,),  # Gurmukhi Letter La      ..Gurmukhi Letter Lla
+    (0x00a35, 0x00a36,),  # Gurmukhi Letter Va      ..Gurmukhi Letter Sha
+    (0x00a38, 0x00a39,),  # Gurmukhi Letter Sa      ..Gurmukhi Letter Ha
+    (0x00a59, 0x00a5c,),  # Gurmukhi Letter Khha    ..Gurmukhi Letter Rra
+    (0x00a5e, 0x00a5e,),  # Gurmukhi Letter Fa
+    (0x00a95, 0x00aa8,),  # Gujarati Letter Ka      ..Gujarati Letter Na
+    (0x00aaa, 0x00ab0,),  # Gujarati Letter Pa      ..Gujarati Letter Ra
+    (0x00ab2, 0x00ab3,),  # Gujarati Letter La      ..Gujarati Letter Lla
+    (0x00ab5, 0x00ab9,),  # Gujarati Letter Va      ..Gujarati Letter Ha
+    (0x00af9, 0x00af9,),  # Gujarati Letter Zha
+    (0x00b15, 0x00b28,),  # Oriya Letter Ka         ..Oriya Letter Na
+    (0x00b2a, 0x00b30,),  # Oriya Letter Pa         ..Oriya Letter Ra
+    (0x00b32, 0x00b33,),  # Oriya Letter La         ..Oriya Letter Lla
+    (0x00b35, 0x00b39,),  # Oriya Letter Va         ..Oriya Letter Ha
+    (0x00b5c, 0x00b5d,),  # Oriya Letter Rra        ..Oriya Letter Rha
+    (0x00b5f, 0x00b5f,),  # Oriya Letter Yya
+    (0x00b71, 0x00b71,),  # Oriya Letter Wa
+    (0x00b95, 0x00b95,),  # Tamil Letter Ka
+    (0x00b99, 0x00b9a,),  # Tamil Letter Nga        ..Tamil Letter Ca
+    (0x00b9c, 0x00b9c,),  # Tamil Letter Ja
+    (0x00b9e, 0x00b9f,),  # Tamil Letter Nya        ..Tamil Letter Tta
+    (0x00ba3, 0x00ba4,),  # Tamil Letter Nna        ..Tamil Letter Ta
+    (0x00ba8, 0x00baa,),  # Tamil Letter Na         ..Tamil Letter Pa
+    (0x00bae, 0x00bb9,),  # Tamil Letter Ma         ..Tamil Letter Ha
+    (0x00c15, 0x00c28,),  # Telugu Letter Ka        ..Telugu Letter Na
+    (0x00c2a, 0x00c39,),  # Telugu Letter Pa        ..Telugu Letter Ha
+    (0x00c58, 0x00c5a,),  # Telugu Letter Tsa       ..Telugu Letter Rrra
+    (0x00c95, 0x00ca8,),  # Kannada Letter Ka       ..Kannada Letter Na
+    (0x00caa, 0x00cb3,),  # Kannada Letter Pa       ..Kannada Letter Lla
+    (0x00cb5, 0x00cb9,),  # Kannada Letter Va       ..Kannada Letter Ha
+    (0x00cde, 0x00cde,),  # Kannada Letter Fa
+    (0x00d15, 0x00d3a,),  # Malayalam Letter Ka     ..Malayalam Letter Ttta
+    (0x00d9a, 0x00db1,),  # Sinhala Letter Alpapraan..Sinhala Letter Dantaja N
+    (0x00db3, 0x00dbb,),  # Sinhala Letter Sanyaka D..Sinhala Letter Rayanna
+    (0x00dbd, 0x00dbd,),  # Sinhala Letter Dantaja Layanna
+    (0x00dc0, 0x00dc6,),  # Sinhala Letter Vayanna  ..Sinhala Letter Fayanna
+    (0x00e01, 0x00e2e,),  # Thai Character Ko Kai   ..Thai Character Ho Nokhuk
+    (0x00e81, 0x00e82,),  # Lao Letter Ko           ..Lao Letter Kho Sung
+    (0x00e84, 0x00e84,),  # Lao Letter Kho Tam
+    (0x00e86, 0x00e8a,),  # Lao Letter Pali Gha     ..Lao Letter So Tam
+    (0x00e8c, 0x00ea3,),  # Lao Letter Pali Jha     ..Lao Letter Lo Ling
+    (0x00ea5, 0x00ea5,),  # Lao Letter Lo Loot
+    (0x00ea7, 0x00eae,),  # Lao Letter Wo           ..Lao Letter Ho Tam
+    (0x00edc, 0x00edf,),  # Lao Ho No               ..Lao Letter Khmu Nyo
+    (0x00f40, 0x00f47,),  # Tibetan Letter Ka       ..Tibetan Letter Ja
+    (0x00f49, 0x00f6c,),  # Tibetan Letter Nya      ..Tibetan Letter Rra
+    (0x01000, 0x01020,),  # Myanmar Letter Ka       ..Myanmar Letter Lla
+    (0x0103f, 0x0103f,),  # Myanmar Letter Great Sa
+    (0x01050, 0x01051,),  # Myanmar Letter Sha      ..Myanmar Letter Ssa
+    (0x0105a, 0x0105d,),  # Myanmar Letter Mon Nga  ..Myanmar Letter Mon Bbe
+    (0x01061, 0x01061,),  # Myanmar Letter Sgaw Karen Sha
+    (0x01065, 0x01066,),  # Myanmar Letter Western P..Myanmar Letter Western P
+    (0x0106e, 0x01070,),  # Myanmar Letter Eastern P..Myanmar Letter Eastern P
+    (0x01075, 0x01081,),  # Myanmar Letter Shan Ka  ..Myanmar Letter Shan Ha
+    (0x0108e, 0x0108e,),  # Myanmar Letter Rumai Palaung Fa
+    (0x01703, 0x01711,),  # Tagalog Letter Ka       ..Tagalog Letter Ha
+    (0x0171f, 0x0171f,),  # Tagalog Letter Archaic Ra
+    (0x01723, 0x01731,),  # Hanunoo Letter Ka       ..Hanunoo Letter Ha
+    (0x01743, 0x01751,),  # Buhid Letter Ka         ..Buhid Letter Ha
+    (0x01763, 0x0176c,),  # Tagbanwa Letter Ka      ..Tagbanwa Letter Ya
+    (0x0176e, 0x01770,),  # Tagbanwa Letter La      ..Tagbanwa Letter Sa
+    (0x01780, 0x017a2,),  # Khmer Letter Ka         ..Khmer Letter Qa
+    (0x01900, 0x0191e,),  # Limbu Vowel-carrier Lett..Limbu Letter Tra
+    (0x01950, 0x01962,),  # Tai Le Letter Ka        ..Tai Le Letter Na
+    (0x01980, 0x019ab,),  # New Tai Lue Letter High ..New Tai Lue Letter Low S
+    (0x01a00, 0x01a16,),  # Buginese Letter Ka      ..Buginese Letter Ha
+    (0x01a20, 0x01a4c,),  # Tai Tham Letter High Ka ..Tai Tham Letter Low Ha
+    (0x01a53, 0x01a54,),  # Tai Tham Letter Lae     ..Tai Tham Letter Great Sa
+    (0x01b13, 0x01b33,),  # Balinese Letter Ka      ..Balinese Letter Ha
+    (0x01b45, 0x01b4c,),  # Balinese Letter Kaf Sasa..Balinese Letter Archaic
+    (0x01b8a, 0x01ba0,),  # Sundanese Letter Ka     ..Sundanese Letter Ha
+    (0x01bae, 0x01baf,),  # Sundanese Letter Kha    ..Sundanese Letter Sya
+    (0x01bbb, 0x01bbd,),  # Sundanese Letter Reu    ..Sundanese Letter Bha
+    (0x01bc0, 0x01be3,),  # Batak Letter A          ..Batak Letter Mba
+    (0x01c00, 0x01c23,),  # Lepcha Letter Ka        ..Lepcha Letter A
+    (0x01c4d, 0x01c4f,),  # Lepcha Letter Tta       ..Lepcha Letter Dda
+    (0x0a807, 0x0a80a,),  # Syloti Nagri Letter Ko  ..Syloti Nagri Letter Gho
+    (0x0a80c, 0x0a822,),  # Syloti Nagri Letter Co  ..Syloti Nagri Letter Ho
+    (0x0a840, 0x0a85d,),  # Phags-pa Letter Ka      ..Phags-pa Letter A
+    (0x0a862, 0x0a865,),  # Phags-pa Letter Qa      ..Phags-pa Letter Gga
+    (0x0a869, 0x0a870,),  # Phags-pa Letter Tta     ..Phags-pa Letter Aspirate
+    (0x0a872, 0x0a872,),  # Phags-pa Superfixed Letter Ra
+    (0x0a892, 0x0a8b3,),  # Saurashtra Letter Ka    ..Saurashtra Letter Lla
+    (0x0a90a, 0x0a921,),  # Kayah Li Letter Ka      ..Kayah Li Letter Ca
+    (0x0a930, 0x0a946,),  # Rejang Letter Ka        ..Rejang Letter A
+    (0x0a989, 0x0a98b,),  # Javanese Letter Pa Cerek..Javanese Letter Nga Lele
+    (0x0a98f, 0x0a9b2,),  # Javanese Letter Ka      ..Javanese Letter Ha
+    (0x0a9e0, 0x0a9e4,),  # Myanmar Letter Shan Gha ..Myanmar Letter Shan Bha
+    (0x0a9e7, 0x0a9ef,),  # Myanmar Letter Tai Laing..Myanmar Letter Tai Laing
+    (0x0a9fa, 0x0a9fe,),  # Myanmar Letter Tai Laing..Myanmar Letter Tai Laing
+    (0x0aa06, 0x0aa28,),  # Cham Letter Ka          ..Cham Letter Ha
+    (0x0aa60, 0x0aa6f,),  # Myanmar Letter Khamti Ga..Myanmar Letter Khamti Fa
+    (0x0aa71, 0x0aa73,),  # Myanmar Letter Khamti Xa..Myanmar Letter Khamti Ra
+    (0x0aa7a, 0x0aa7a,),  # Myanmar Letter Aiton Ra
+    (0x0aa7e, 0x0aaaf,),  # Myanmar Letter Shwe Pala..Tai Viet Letter High O
+    (0x0aae2, 0x0aaea,),  # Meetei Mayek Letter Cha ..Meetei Mayek Letter Ssa
+    (0x0abc0, 0x0abcd,),  # Meetei Mayek Letter Kok ..Meetei Mayek Letter Huk
+    (0x0abd0, 0x0abd0,),  # Meetei Mayek Letter Pham
+    (0x0abd2, 0x0abda,),  # Meetei Mayek Letter Gok ..Meetei Mayek Letter Bham
+    (0x10a00, 0x10a00,),  # Kharoshthi Letter A
+    (0x10a10, 0x10a13,),  # Kharoshthi Letter Ka    ..Kharoshthi Letter Gha
+    (0x10a15, 0x10a17,),  # Kharoshthi Letter Ca    ..Kharoshthi Letter Ja
+    (0x10a19, 0x10a35,),  # Kharoshthi Letter Nya   ..Kharoshthi Letter Vha
+    (0x11013, 0x11037,),  # Brahmi Letter Ka        ..Brahmi Letter Old Tamil
+    (0x11075, 0x11075,),  # Brahmi Letter Old Tamil Lla
+    (0x1108d, 0x110af,),  # Kaithi Letter Ka        ..Kaithi Letter Ha
+    (0x11107, 0x11126,),  # Chakma Letter Kaa       ..Chakma Letter Haa
+    (0x11144, 0x11144,),  # Chakma Letter Lhaa
+    (0x11147, 0x11147,),  # Chakma Letter Vaa
+    (0x11155, 0x11172,),  # Mahajani Letter Ka      ..Mahajani Letter Rra
+    (0x11191, 0x111b2,),  # Sharada Letter Ka       ..Sharada Letter Ha
+    (0x11208, 0x11211,),  # Khojki Letter Ka        ..Khojki Letter Jja
+    (0x11213, 0x1122b,),  # Khojki Letter Nya       ..Khojki Letter Lla
+    (0x1123f, 0x1123f,),  # Khojki Letter Qa
+    (0x11284, 0x11286,),  # Multani Letter Ka       ..Multani Letter Ga
+    (0x11288, 0x11288,),  # Multani Letter Gha
+    (0x1128a, 0x1128d,),  # Multani Letter Ca       ..Multani Letter Jja
+    (0x1128f, 0x1129d,),  # Multani Letter Nya      ..Multani Letter Ba
+    (0x1129f, 0x112a8,),  # Multani Letter Bha      ..Multani Letter Rha
+    (0x112ba, 0x112de,),  # Khudawadi Letter Ka     ..Khudawadi Letter Ha
+    (0x11315, 0x11328,),  # Grantha Letter Ka       ..Grantha Letter Na
+    (0x1132a, 0x11330,),  # Grantha Letter Pa       ..Grantha Letter Ra
+    (0x11332, 0x11333,),  # Grantha Letter La       ..Grantha Letter Lla
+    (0x11335, 0x11339,),  # Grantha Letter Va       ..Grantha Letter Ha
+    (0x11392, 0x113b5,),  # Tulu-tigalari Letter Ka ..Tulu-tigalari Letter Lll
+    (0x1140e, 0x11434,),  # Newa Letter Ka          ..Newa Letter Ha
+    (0x1148f, 0x114af,),  # Tirhuta Letter Ka       ..Tirhuta Letter Ha
+    (0x1158e, 0x115ae,),  # Siddham Letter Ka       ..Siddham Letter Ha
+    (0x1160e, 0x1162f,),  # Modi Letter Ka          ..Modi Letter Lla
+    (0x1168a, 0x116aa,),  # Takri Letter Ka         ..Takri Letter Rra
+    (0x116b8, 0x116b8,),  # Takri Letter Archaic Kha
+    (0x11700, 0x1171a,),  # Ahom Letter Ka          ..Ahom Letter Alternate Ba
+    (0x11740, 0x11746,),  # Ahom Letter Ca          ..Ahom Letter Lla
+    (0x1180a, 0x1182b,),  # Dogra Letter Ka         ..Dogra Letter Rra
+    (0x1190c, 0x11913,),  # Dives Akuru Letter Ka   ..Dives Akuru Letter Ja
+    (0x11915, 0x11916,),  # Dives Akuru Letter Nya  ..Dives Akuru Letter Tta
+    (0x11918, 0x1192f,),  # Dives Akuru Letter Dda  ..Dives Akuru Letter Za
+    (0x119ae, 0x119d0,),  # Nandinagari Letter Ka   ..Nandinagari Letter Rra
+    (0x11a00, 0x11a00,),  # Zanabazar Square Letter A
+    (0x11a0b, 0x11a32,),  # Zanabazar Square Letter ..Zanabazar Square Letter
+    (0x11a50, 0x11a50,),  # Soyombo Letter A
+    (0x11a5c, 0x11a83,),  # Soyombo Letter Ka       ..Soyombo Letter Kssa
+    (0x11c0e, 0x11c2e,),  # Bhaiksuki Letter Ka     ..Bhaiksuki Letter Ha
+    (0x11c72, 0x11c8f,),  # Marchen Letter Ka       ..Marchen Letter A
+    (0x11d0c, 0x11d30,),  # Masaram Gondi Letter Ka ..Masaram Gondi Letter Tra
+    (0x11d6c, 0x11d89,),  # Gunjala Gondi Letter Ya ..Gunjala Gondi Letter Sa
+    (0x11ee0, 0x11ef1,),  # Makasar Letter Ka       ..Makasar Letter A
+    (0x11f12, 0x11f33,),  # Kawi Letter Ka          ..Kawi Letter Jnya
+    (0x16101, 0x1611d,),  # Gurung Khema Letter Ka  ..Gurung Khema Letter Sa
+    (0x16d43, 0x16d62,),  # Kirat Rai Letter A      ..Kirat Rai Letter Ha
+)
diff --git a/lib/wcwidth/table_mc.py b/lib/wcwidth/table_mc.py
new file mode 100644
index 0000000..7c2e691
--- /dev/null
+++ b/lib/wcwidth/table_mc.py
@@ -0,0 +1,206 @@
+"""
+Exports CATEGORY_MC table keyed by supporting unicode version level.
+
+This code generated by wcwidth/bin/update-tables.py on 2026-01-29 00:47:54 UTC.
+"""
+# pylint: disable=duplicate-code
+CATEGORY_MC = {
+    '17.0.0': (
+        # Source: DerivedGeneralCategory-17.0.0.txt
+        # Date: 2025-07-24, 00:12:50 GMT
+        #
+        (0x00903, 0x00903,),  # Devanagari Sign Visarga
+        (0x0093b, 0x0093b,),  # Devanagari Vowel Sign Ooe
+        (0x0093e, 0x00940,),  # Devanagari Vowel Sign Aa..Devanagari Vowel Sign Ii
+        (0x00949, 0x0094c,),  # Devanagari Vowel Sign Ca..Devanagari Vowel Sign Au
+        (0x0094e, 0x0094f,),  # Devanagari Vowel Sign Pr..Devanagari Vowel Sign Aw
+        (0x00982, 0x00983,),  # Bengali Sign Anusvara   ..Bengali Sign Visarga
+        (0x009be, 0x009c0,),  # Bengali Vowel Sign Aa   ..Bengali Vowel Sign Ii
+        (0x009c7, 0x009c8,),  # Bengali Vowel Sign E    ..Bengali Vowel Sign Ai
+        (0x009cb, 0x009cc,),  # Bengali Vowel Sign O    ..Bengali Vowel Sign Au
+        (0x009d7, 0x009d7,),  # Bengali Au Length Mark
+        (0x00a03, 0x00a03,),  # Gurmukhi Sign Visarga
+        (0x00a3e, 0x00a40,),  # Gurmukhi Vowel Sign Aa  ..Gurmukhi Vowel Sign Ii
+        (0x00a83, 0x00a83,),  # Gujarati Sign Visarga
+        (0x00abe, 0x00ac0,),  # Gujarati Vowel Sign Aa  ..Gujarati Vowel Sign Ii
+        (0x00ac9, 0x00ac9,),  # Gujarati Vowel Sign Candra O
+        (0x00acb, 0x00acc,),  # Gujarati Vowel Sign O   ..Gujarati Vowel Sign Au
+        (0x00b02, 0x00b03,),  # Oriya Sign Anusvara     ..Oriya Sign Visarga
+        (0x00b3e, 0x00b3e,),  # Oriya Vowel Sign Aa
+        (0x00b40, 0x00b40,),  # Oriya Vowel Sign Ii
+        (0x00b47, 0x00b48,),  # Oriya Vowel Sign E      ..Oriya Vowel Sign Ai
+        (0x00b4b, 0x00b4c,),  # Oriya Vowel Sign O      ..Oriya Vowel Sign Au
+        (0x00b57, 0x00b57,),  # Oriya Au Length Mark
+        (0x00bbe, 0x00bbf,),  # Tamil Vowel Sign Aa     ..Tamil Vowel Sign I
+        (0x00bc1, 0x00bc2,),  # Tamil Vowel Sign U      ..Tamil Vowel Sign Uu
+        (0x00bc6, 0x00bc8,),  # Tamil Vowel Sign E      ..Tamil Vowel Sign Ai
+        (0x00bca, 0x00bcc,),  # Tamil Vowel Sign O      ..Tamil Vowel Sign Au
+        (0x00bd7, 0x00bd7,),  # Tamil Au Length Mark
+        (0x00c01, 0x00c03,),  # Telugu Sign Candrabindu ..Telugu Sign Visarga
+        (0x00c41, 0x00c44,),  # Telugu Vowel Sign U     ..Telugu Vowel Sign Vocali
+        (0x00c82, 0x00c83,),  # Kannada Sign Anusvara   ..Kannada Sign Visarga
+        (0x00cbe, 0x00cbe,),  # Kannada Vowel Sign Aa
+        (0x00cc0, 0x00cc4,),  # Kannada Vowel Sign Ii   ..Kannada Vowel Sign Vocal
+        (0x00cc7, 0x00cc8,),  # Kannada Vowel Sign Ee   ..Kannada Vowel Sign Ai
+        (0x00cca, 0x00ccb,),  # Kannada Vowel Sign O    ..Kannada Vowel Sign Oo
+        (0x00cd5, 0x00cd6,),  # Kannada Length Mark     ..Kannada Ai Length Mark
+        (0x00cf3, 0x00cf3,),  # Kannada Sign Combining Anusvara Above Right
+        (0x00d02, 0x00d03,),  # Malayalam Sign Anusvara ..Malayalam Sign Visarga
+        (0x00d3e, 0x00d40,),  # Malayalam Vowel Sign Aa ..Malayalam Vowel Sign Ii
+        (0x00d46, 0x00d48,),  # Malayalam Vowel Sign E  ..Malayalam Vowel Sign Ai
+        (0x00d4a, 0x00d4c,),  # Malayalam Vowel Sign O  ..Malayalam Vowel Sign Au
+        (0x00d57, 0x00d57,),  # Malayalam Au Length Mark
+        (0x00d82, 0x00d83,),  # Sinhala Sign Anusvaraya ..Sinhala Sign Visargaya
+        (0x00dcf, 0x00dd1,),  # Sinhala Vowel Sign Aela-..Sinhala Vowel Sign Diga
+        (0x00dd8, 0x00ddf,),  # Sinhala Vowel Sign Gaett..Sinhala Vowel Sign Gayan
+        (0x00df2, 0x00df3,),  # Sinhala Vowel Sign Diga ..Sinhala Vowel Sign Diga
+        (0x00f3e, 0x00f3f,),  # Tibetan Sign Yar Tshes  ..Tibetan Sign Mar Tshes
+        (0x00f7f, 0x00f7f,),  # Tibetan Sign Rnam Bcad
+        (0x0102b, 0x0102c,),  # Myanmar Vowel Sign Tall ..Myanmar Vowel Sign Aa
+        (0x01031, 0x01031,),  # Myanmar Vowel Sign E
+        (0x01038, 0x01038,),  # Myanmar Sign Visarga
+        (0x0103b, 0x0103c,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+        (0x01056, 0x01057,),  # Myanmar Vowel Sign Vocal..Myanmar Vowel Sign Vocal
+        (0x01062, 0x01064,),  # Myanmar Vowel Sign Sgaw ..Myanmar Tone Mark Sgaw K
+        (0x01067, 0x0106d,),  # Myanmar Vowel Sign Weste..Myanmar Sign Western Pwo
+        (0x01083, 0x01084,),  # Myanmar Vowel Sign Shan ..Myanmar Vowel Sign Shan
+        (0x01087, 0x0108c,),  # Myanmar Sign Shan Tone-2..Myanmar Sign Shan Counci
+        (0x0108f, 0x0108f,),  # Myanmar Sign Rumai Palaung Tone-5
+        (0x0109a, 0x0109c,),  # Myanmar Sign Khamti Tone..Myanmar Vowel Sign Aiton
+        (0x01715, 0x01715,),  # Tagalog Sign Pamudpod
+        (0x01734, 0x01734,),  # Hanunoo Sign Pamudpod
+        (0x017b6, 0x017b6,),  # Khmer Vowel Sign Aa
+        (0x017be, 0x017c5,),  # Khmer Vowel Sign Oe     ..Khmer Vowel Sign Au
+        (0x017c7, 0x017c8,),  # Khmer Sign Reahmuk      ..Khmer Sign Yuukaleapintu
+        (0x01923, 0x01926,),  # Limbu Vowel Sign Ee     ..Limbu Vowel Sign Au
+        (0x01929, 0x0192b,),  # Limbu Subjoined Letter Y..Limbu Subjoined Letter W
+        (0x01930, 0x01931,),  # Limbu Small Letter Ka   ..Limbu Small Letter Nga
+        (0x01933, 0x01938,),  # Limbu Small Letter Ta   ..Limbu Small Letter La
+        (0x01a19, 0x01a1a,),  # Buginese Vowel Sign E   ..Buginese Vowel Sign O
+        (0x01a55, 0x01a55,),  # Tai Tham Consonant Sign Medial Ra
+        (0x01a57, 0x01a57,),  # Tai Tham Consonant Sign La Tang Lai
+        (0x01a61, 0x01a61,),  # Tai Tham Vowel Sign A
+        (0x01a63, 0x01a64,),  # Tai Tham Vowel Sign Aa  ..Tai Tham Vowel Sign Tall
+        (0x01a6d, 0x01a72,),  # Tai Tham Vowel Sign Oy  ..Tai Tham Vowel Sign Tham
+        (0x01b04, 0x01b04,),  # Balinese Sign Bisah
+        (0x01b35, 0x01b35,),  # Balinese Vowel Sign Tedung
+        (0x01b3b, 0x01b3b,),  # Balinese Vowel Sign Ra Repa Tedung
+        (0x01b3d, 0x01b41,),  # Balinese Vowel Sign La L..Balinese Vowel Sign Tali
+        (0x01b43, 0x01b44,),  # Balinese Vowel Sign Pepe..Balinese Adeg Adeg
+        (0x01b82, 0x01b82,),  # Sundanese Sign Pangwisad
+        (0x01ba1, 0x01ba1,),  # Sundanese Consonant Sign Pamingkal
+        (0x01ba6, 0x01ba7,),  # Sundanese Vowel Sign Pan..Sundanese Vowel Sign Pan
+        (0x01baa, 0x01baa,),  # Sundanese Sign Pamaaeh
+        (0x01be7, 0x01be7,),  # Batak Vowel Sign E
+        (0x01bea, 0x01bec,),  # Batak Vowel Sign I      ..Batak Vowel Sign O
+        (0x01bee, 0x01bee,),  # Batak Vowel Sign U
+        (0x01bf2, 0x01bf3,),  # Batak Pangolat          ..Batak Panongonan
+        (0x01c24, 0x01c2b,),  # Lepcha Subjoined Letter ..Lepcha Vowel Sign Uu
+        (0x01c34, 0x01c35,),  # Lepcha Consonant Sign Ny..Lepcha Consonant Sign Ka
+        (0x01ce1, 0x01ce1,),  # Vedic Tone Atharvavedic Independent Svarita
+        (0x01cf7, 0x01cf7,),  # Vedic Sign Atikrama
+        (0x0302e, 0x0302f,),  # Hangul Single Dot Tone M..Hangul Double Dot Tone M
+        (0x0a823, 0x0a824,),  # Syloti Nagri Vowel Sign ..Syloti Nagri Vowel Sign
+        (0x0a827, 0x0a827,),  # Syloti Nagri Vowel Sign Oo
+        (0x0a880, 0x0a881,),  # Saurashtra Sign Anusvara..Saurashtra Sign Visarga
+        (0x0a8b4, 0x0a8c3,),  # Saurashtra Consonant Sig..Saurashtra Vowel Sign Au
+        (0x0a952, 0x0a953,),  # Rejang Consonant Sign H ..Rejang Virama
+        (0x0a983, 0x0a983,),  # Javanese Sign Wignyan
+        (0x0a9b4, 0x0a9b5,),  # Javanese Vowel Sign Taru..Javanese Vowel Sign Tolo
+        (0x0a9ba, 0x0a9bb,),  # Javanese Vowel Sign Tali..Javanese Vowel Sign Dirg
+        (0x0a9be, 0x0a9c0,),  # Javanese Consonant Sign ..Javanese Pangkon
+        (0x0aa2f, 0x0aa30,),  # Cham Vowel Sign O       ..Cham Vowel Sign Ai
+        (0x0aa33, 0x0aa34,),  # Cham Consonant Sign Ya  ..Cham Consonant Sign Ra
+        (0x0aa4d, 0x0aa4d,),  # Cham Consonant Sign Final H
+        (0x0aa7b, 0x0aa7b,),  # Myanmar Sign Pao Karen Tone
+        (0x0aa7d, 0x0aa7d,),  # Myanmar Sign Tai Laing Tone-5
+        (0x0aaeb, 0x0aaeb,),  # Meetei Mayek Vowel Sign Ii
+        (0x0aaee, 0x0aaef,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+        (0x0aaf5, 0x0aaf5,),  # Meetei Mayek Vowel Sign Visarga
+        (0x0abe3, 0x0abe4,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+        (0x0abe6, 0x0abe7,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+        (0x0abe9, 0x0abea,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+        (0x0abec, 0x0abec,),  # Meetei Mayek Lum Iyek
+        (0x11000, 0x11000,),  # Brahmi Sign Candrabindu
+        (0x11002, 0x11002,),  # Brahmi Sign Visarga
+        (0x11082, 0x11082,),  # Kaithi Sign Visarga
+        (0x110b0, 0x110b2,),  # Kaithi Vowel Sign Aa    ..Kaithi Vowel Sign Ii
+        (0x110b7, 0x110b8,),  # Kaithi Vowel Sign O     ..Kaithi Vowel Sign Au
+        (0x1112c, 0x1112c,),  # Chakma Vowel Sign E
+        (0x11145, 0x11146,),  # Chakma Vowel Sign Aa    ..Chakma Vowel Sign Ei
+        (0x11182, 0x11182,),  # Sharada Sign Visarga
+        (0x111b3, 0x111b5,),  # Sharada Vowel Sign Aa   ..Sharada Vowel Sign Ii
+        (0x111bf, 0x111c0,),  # Sharada Vowel Sign Au   ..Sharada Sign Virama
+        (0x111ce, 0x111ce,),  # Sharada Vowel Sign Prishthamatra E
+        (0x1122c, 0x1122e,),  # Khojki Vowel Sign Aa    ..Khojki Vowel Sign Ii
+        (0x11232, 0x11233,),  # Khojki Vowel Sign O     ..Khojki Vowel Sign Au
+        (0x11235, 0x11235,),  # Khojki Sign Virama
+        (0x112e0, 0x112e2,),  # Khudawadi Vowel Sign Aa ..Khudawadi Vowel Sign Ii
+        (0x11302, 0x11303,),  # Grantha Sign Anusvara   ..Grantha Sign Visarga
+        (0x1133e, 0x1133f,),  # Grantha Vowel Sign Aa   ..Grantha Vowel Sign I
+        (0x11341, 0x11344,),  # Grantha Vowel Sign U    ..Grantha Vowel Sign Vocal
+        (0x11347, 0x11348,),  # Grantha Vowel Sign Ee   ..Grantha Vowel Sign Ai
+        (0x1134b, 0x1134d,),  # Grantha Vowel Sign Oo   ..Grantha Sign Virama
+        (0x11357, 0x11357,),  # Grantha Au Length Mark
+        (0x11362, 0x11363,),  # Grantha Vowel Sign Vocal..Grantha Vowel Sign Vocal
+        (0x113b8, 0x113ba,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Vowel Sign
+        (0x113c2, 0x113c2,),  # Tulu-tigalari Vowel Sign Ee
+        (0x113c5, 0x113c5,),  # Tulu-tigalari Vowel Sign Ai
+        (0x113c7, 0x113ca,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Sign Candr
+        (0x113cc, 0x113cd,),  # Tulu-tigalari Sign Anusv..Tulu-tigalari Sign Visar
+        (0x113cf, 0x113cf,),  # Tulu-tigalari Sign Looped Virama
+        (0x11435, 0x11437,),  # Newa Vowel Sign Aa      ..Newa Vowel Sign Ii
+        (0x11440, 0x11441,),  # Newa Vowel Sign O       ..Newa Vowel Sign Au
+        (0x11445, 0x11445,),  # Newa Sign Visarga
+        (0x114b0, 0x114b2,),  # Tirhuta Vowel Sign Aa   ..Tirhuta Vowel Sign Ii
+        (0x114b9, 0x114b9,),  # Tirhuta Vowel Sign E
+        (0x114bb, 0x114be,),  # Tirhuta Vowel Sign Ai   ..Tirhuta Vowel Sign Au
+        (0x114c1, 0x114c1,),  # Tirhuta Sign Visarga
+        (0x115af, 0x115b1,),  # Siddham Vowel Sign Aa   ..Siddham Vowel Sign Ii
+        (0x115b8, 0x115bb,),  # Siddham Vowel Sign E    ..Siddham Vowel Sign Au
+        (0x115be, 0x115be,),  # Siddham Sign Visarga
+        (0x11630, 0x11632,),  # Modi Vowel Sign Aa      ..Modi Vowel Sign Ii
+        (0x1163b, 0x1163c,),  # Modi Vowel Sign O       ..Modi Vowel Sign Au
+        (0x1163e, 0x1163e,),  # Modi Sign Visarga
+        (0x116ac, 0x116ac,),  # Takri Sign Visarga
+        (0x116ae, 0x116af,),  # Takri Vowel Sign I      ..Takri Vowel Sign Ii
+        (0x116b6, 0x116b6,),  # Takri Sign Virama
+        (0x1171e, 0x1171e,),  # Ahom Consonant Sign Medial Ra
+        (0x11720, 0x11721,),  # Ahom Vowel Sign A       ..Ahom Vowel Sign Aa
+        (0x11726, 0x11726,),  # Ahom Vowel Sign E
+        (0x1182c, 0x1182e,),  # Dogra Vowel Sign Aa     ..Dogra Vowel Sign Ii
+        (0x11838, 0x11838,),  # Dogra Sign Visarga
+        (0x11930, 0x11935,),  # Dives Akuru Vowel Sign A..Dives Akuru Vowel Sign E
+        (0x11937, 0x11938,),  # Dives Akuru Vowel Sign A..Dives Akuru Vowel Sign O
+        (0x1193d, 0x1193d,),  # Dives Akuru Sign Halanta
+        (0x11940, 0x11940,),  # Dives Akuru Medial Ya
+        (0x11942, 0x11942,),  # Dives Akuru Medial Ra
+        (0x119d1, 0x119d3,),  # Nandinagari Vowel Sign A..Nandinagari Vowel Sign I
+        (0x119dc, 0x119df,),  # Nandinagari Vowel Sign O..Nandinagari Sign Visarga
+        (0x119e4, 0x119e4,),  # Nandinagari Vowel Sign Prishthamatra E
+        (0x11a39, 0x11a39,),  # Zanabazar Square Sign Visarga
+        (0x11a57, 0x11a58,),  # Soyombo Vowel Sign Ai   ..Soyombo Vowel Sign Au
+        (0x11a97, 0x11a97,),  # Soyombo Sign Visarga
+        (0x11b61, 0x11b61,),  # Sharada Vowel Sign Ooe
+        (0x11b65, 0x11b65,),  # Sharada Vowel Sign Short O
+        (0x11b67, 0x11b67,),  # Sharada Vowel Sign Candra O
+        (0x11c2f, 0x11c2f,),  # Bhaiksuki Vowel Sign Aa
+        (0x11c3e, 0x11c3e,),  # Bhaiksuki Sign Visarga
+        (0x11ca9, 0x11ca9,),  # Marchen Subjoined Letter Ya
+        (0x11cb1, 0x11cb1,),  # Marchen Vowel Sign I
+        (0x11cb4, 0x11cb4,),  # Marchen Vowel Sign O
+        (0x11d8a, 0x11d8e,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+        (0x11d93, 0x11d94,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+        (0x11d96, 0x11d96,),  # Gunjala Gondi Sign Visarga
+        (0x11ef5, 0x11ef6,),  # Makasar Vowel Sign E    ..Makasar Vowel Sign O
+        (0x11f03, 0x11f03,),  # Kawi Sign Visarga
+        (0x11f34, 0x11f35,),  # Kawi Vowel Sign Aa      ..Kawi Vowel Sign Alternat
+        (0x11f3e, 0x11f3f,),  # Kawi Vowel Sign E       ..Kawi Vowel Sign Ai
+        (0x11f41, 0x11f41,),  # Kawi Sign Killer
+        (0x1612a, 0x1612c,),  # Gurung Khema Consonant S..Gurung Khema Consonant S
+        (0x16f51, 0x16f87,),  # Miao Sign Aspiration    ..Miao Vowel Sign Ui
+        (0x16ff0, 0x16ff1,),  # Vietnamese Alternate Rea..Vietnamese Alternate Rea
+        (0x1d165, 0x1d166,),  # Musical Symbol Combining..Musical Symbol Combining
+        (0x1d16d, 0x1d172,),  # Musical Symbol Combining..Musical Symbol Combining
+    ),
+}
diff --git a/lib/wcwidth/table_vs16.py b/lib/wcwidth/table_vs16.py
new file mode 100644
index 0000000..70e4a73
--- /dev/null
+++ b/lib/wcwidth/table_vs16.py
@@ -0,0 +1,126 @@
+"""
+Exports VS16_NARROW_TO_WIDE table keyed by supporting unicode version level.
+
+This code generated by wcwidth/bin/update-tables.py on 2025-09-15 16:57:50 UTC.
+"""
+# pylint: disable=duplicate-code
+VS16_NARROW_TO_WIDE = {
+    '9.0.0': (
+        # Source: 9.0.0
+        # Date: 2025-01-30, 21:48:29 GMT
+        #
+        (0x00023, 0x00023,),  # Number Sign
+        (0x0002a, 0x0002a,),  # Asterisk
+        (0x00030, 0x00039,),  # Digit Zero              ..Digit Nine
+        (0x000a9, 0x000a9,),  # Copyright Sign
+        (0x000ae, 0x000ae,),  # Registered Sign
+        (0x0203c, 0x0203c,),  # Double Exclamation Mark
+        (0x02049, 0x02049,),  # Exclamation Question Mark
+        (0x02122, 0x02122,),  # Trade Mark Sign
+        (0x02139, 0x02139,),  # Information Source
+        (0x02194, 0x02199,),  # Left Right Arrow        ..South West Arrow
+        (0x021a9, 0x021aa,),  # Leftwards Arrow With Hoo..Rightwards Arrow With Ho
+        (0x02328, 0x02328,),  # Keyboard
+        (0x023cf, 0x023cf,),  # Eject Symbol
+        (0x023ed, 0x023ef,),  # Black Right-pointing Dou..Black Right-pointing Tri
+        (0x023f1, 0x023f2,),  # Stopwatch               ..Timer Clock
+        (0x023f8, 0x023fa,),  # Double Vertical Bar     ..Black Circle For Record
+        (0x024c2, 0x024c2,),  # Circled Latin Capital Letter M
+        (0x025aa, 0x025ab,),  # Black Small Square      ..White Small Square
+        (0x025b6, 0x025b6,),  # Black Right-pointing Triangle
+        (0x025c0, 0x025c0,),  # Black Left-pointing Triangle
+        (0x025fb, 0x025fc,),  # White Medium Square     ..Black Medium Square
+        (0x02600, 0x02604,),  # Black Sun With Rays     ..Comet
+        (0x0260e, 0x0260e,),  # Black Telephone
+        (0x02611, 0x02611,),  # Ballot Box With Check
+        (0x02618, 0x02618,),  # Shamrock
+        (0x0261d, 0x0261d,),  # White Up Pointing Index
+        (0x02620, 0x02620,),  # Skull And Crossbones
+        (0x02622, 0x02623,),  # Radioactive Sign        ..Biohazard Sign
+        (0x02626, 0x02626,),  # Orthodox Cross
+        (0x0262a, 0x0262a,),  # Star And Crescent
+        (0x0262e, 0x0262f,),  # Peace Symbol            ..Yin Yang
+        (0x02638, 0x0263a,),  # Wheel Of Dharma         ..White Smiling Face
+        (0x02640, 0x02640,),  # Female Sign
+        (0x02642, 0x02642,),  # Male Sign
+        (0x0265f, 0x02660,),  # Black Chess Pawn        ..Black Spade Suit
+        (0x02663, 0x02663,),  # Black Club Suit
+        (0x02665, 0x02666,),  # Black Heart Suit        ..Black Diamond Suit
+        (0x02668, 0x02668,),  # Hot Springs
+        (0x0267b, 0x0267b,),  # Black Universal Recycling Symbol
+        (0x0267e, 0x0267e,),  # Permanent Paper Sign
+        (0x02692, 0x02692,),  # Hammer And Pick
+        (0x02694, 0x02697,),  # Crossed Swords          ..Alembic
+        (0x02699, 0x02699,),  # Gear
+        (0x0269b, 0x0269c,),  # Atom Symbol             ..Fleur-de-lis
+        (0x026a0, 0x026a0,),  # Warning Sign
+        (0x026a7, 0x026a7,),  # Male With Stroke And Male And Female Sign
+        (0x026b0, 0x026b1,),  # Coffin                  ..Funeral Urn
+        (0x026c8, 0x026c8,),  # Thunder Cloud And Rain
+        (0x026cf, 0x026cf,),  # Pick
+        (0x026d1, 0x026d1,),  # Helmet With White Cross
+        (0x026d3, 0x026d3,),  # Chains
+        (0x026e9, 0x026e9,),  # Shinto Shrine
+        (0x026f0, 0x026f1,),  # Mountain                ..Umbrella On Ground
+        (0x026f4, 0x026f4,),  # Ferry
+        (0x026f7, 0x026f9,),  # Skier                   ..Person With Ball
+        (0x02702, 0x02702,),  # Black Scissors
+        (0x02708, 0x02709,),  # Airplane                ..Envelope
+        (0x0270c, 0x0270d,),  # Victory Hand            ..Writing Hand
+        (0x0270f, 0x0270f,),  # Pencil
+        (0x02712, 0x02712,),  # Black Nib
+        (0x02714, 0x02714,),  # Heavy Check Mark
+        (0x02716, 0x02716,),  # Heavy Multiplication X
+        (0x0271d, 0x0271d,),  # Latin Cross
+        (0x02721, 0x02721,),  # Star Of David
+        (0x02733, 0x02734,),  # Eight Spoked Asterisk   ..Eight Pointed Black Star
+        (0x02744, 0x02744,),  # Snowflake
+        (0x02747, 0x02747,),  # Sparkle
+        (0x02763, 0x02764,),  # Heavy Heart Exclamation ..Heavy Black Heart
+        (0x027a1, 0x027a1,),  # Black Rightwards Arrow
+        (0x02934, 0x02935,),  # Arrow Pointing Rightward..Arrow Pointing Rightward
+        (0x02b05, 0x02b07,),  # Leftwards Black Arrow   ..Downwards Black Arrow
+        (0x1f170, 0x1f171,),  # Negative Squared Latin C..Negative Squared Latin C
+        (0x1f17e, 0x1f17f,),  # Negative Squared Latin C..Negative Squared Latin C
+        (0x1f321, 0x1f321,),  # Thermometer
+        (0x1f324, 0x1f32c,),  # White Sun With Small Clo..Wind Blowing Face
+        (0x1f336, 0x1f336,),  # Hot Pepper
+        (0x1f37d, 0x1f37d,),  # Fork And Knife With Plate
+        (0x1f396, 0x1f397,),  # Military Medal          ..Reminder Ribbon
+        (0x1f399, 0x1f39b,),  # Studio Microphone       ..Control Knobs
+        (0x1f39e, 0x1f39f,),  # Film Frames             ..Admission Tickets
+        (0x1f3cb, 0x1f3ce,),  # Weight Lifter           ..Racing Car
+        (0x1f3d4, 0x1f3df,),  # Snow Capped Mountain    ..Stadium
+        (0x1f3f3, 0x1f3f3,),  # Waving White Flag
+        (0x1f3f5, 0x1f3f5,),  # Rosette
+        (0x1f3f7, 0x1f3f7,),  # Label
+        (0x1f43f, 0x1f43f,),  # Chipmunk
+        (0x1f441, 0x1f441,),  # Eye
+        (0x1f4fd, 0x1f4fd,),  # Film Projector
+        (0x1f549, 0x1f54a,),  # Om Symbol               ..Dove Of Peace
+        (0x1f56f, 0x1f570,),  # Candle                  ..Mantelpiece Clock
+        (0x1f573, 0x1f579,),  # Hole                    ..Joystick
+        (0x1f587, 0x1f587,),  # Linked Paperclips
+        (0x1f58a, 0x1f58d,),  # Lower Left Ballpoint Pen..Lower Left Crayon
+        (0x1f590, 0x1f590,),  # Raised Hand With Fingers Splayed
+        (0x1f5a5, 0x1f5a5,),  # Desktop Computer
+        (0x1f5a8, 0x1f5a8,),  # Printer
+        (0x1f5b1, 0x1f5b2,),  # Three Button Mouse      ..Trackball
+        (0x1f5bc, 0x1f5bc,),  # Frame With Picture
+        (0x1f5c2, 0x1f5c4,),  # Card Index Dividers     ..File Cabinet
+        (0x1f5d1, 0x1f5d3,),  # Wastebasket             ..Spiral Calendar Pad
+        (0x1f5dc, 0x1f5de,),  # Compression             ..Rolled-up Newspaper
+        (0x1f5e1, 0x1f5e1,),  # Dagger Knife
+        (0x1f5e3, 0x1f5e3,),  # Speaking Head In Silhouette
+        (0x1f5e8, 0x1f5e8,),  # Left Speech Bubble
+        (0x1f5ef, 0x1f5ef,),  # Right Anger Bubble
+        (0x1f5f3, 0x1f5f3,),  # Ballot Box With Ballot
+        (0x1f5fa, 0x1f5fa,),  # World Map
+        (0x1f6cb, 0x1f6cb,),  # Couch And Lamp
+        (0x1f6cd, 0x1f6cf,),  # Shopping Bags           ..Bed
+        (0x1f6e0, 0x1f6e5,),  # Hammer And Wrench       ..Motor Boat
+        (0x1f6e9, 0x1f6e9,),  # Small Airplane
+        (0x1f6f0, 0x1f6f0,),  # Satellite
+        (0x1f6f3, 0x1f6f3,),  # Passenger Ship
+    ),
+}
diff --git a/lib/wcwidth/table_wide.py b/lib/wcwidth/table_wide.py
new file mode 100644
index 0000000..ed6f48a
--- /dev/null
+++ b/lib/wcwidth/table_wide.py
@@ -0,0 +1,138 @@
+"""
+Exports WIDE_EASTASIAN table keyed by supporting unicode version level.
+
+This code generated by wcwidth/bin/update-tables.py on 2026-01-30 00:58:17 UTC.
+"""
+# pylint: disable=duplicate-code
+WIDE_EASTASIAN = {
+    '17.0.0': (
+        # Source: EastAsianWidth-17.0.0.txt
+        # Date: 2025-07-24, 00:12:54 GMT
+        #
+        (0x01100, 0x0115f,),  # Hangul Choseong Kiyeok  ..Hangul Choseong Filler
+        (0x0231a, 0x0231b,),  # Watch                   ..Hourglass
+        (0x02329, 0x0232a,),  # Left-pointing Angle Brac..Right-pointing Angle Bra
+        (0x023e9, 0x023ec,),  # Black Right-pointing Dou..Black Down-pointing Doub
+        (0x023f0, 0x023f0,),  # Alarm Clock
+        (0x023f3, 0x023f3,),  # Hourglass With Flowing Sand
+        (0x025fd, 0x025fe,),  # White Medium Small Squar..Black Medium Small Squar
+        (0x02614, 0x02615,),  # Umbrella With Rain Drops..Hot Beverage
+        (0x02630, 0x02637,),  # Trigram For Heaven      ..Trigram For Earth
+        (0x02648, 0x02653,),  # Aries                   ..Pisces
+        (0x0267f, 0x0267f,),  # Wheelchair Symbol
+        (0x0268a, 0x0268f,),  # Monogram For Yang       ..Digram For Greater Yin
+        (0x02693, 0x02693,),  # Anchor
+        (0x026a1, 0x026a1,),  # High Voltage Sign
+        (0x026aa, 0x026ab,),  # Medium White Circle     ..Medium Black Circle
+        (0x026bd, 0x026be,),  # Soccer Ball             ..Baseball
+        (0x026c4, 0x026c5,),  # Snowman Without Snow    ..Sun Behind Cloud
+        (0x026ce, 0x026ce,),  # Ophiuchus
+        (0x026d4, 0x026d4,),  # No Entry
+        (0x026ea, 0x026ea,),  # Church
+        (0x026f2, 0x026f3,),  # Fountain                ..Flag In Hole
+        (0x026f5, 0x026f5,),  # Sailboat
+        (0x026fa, 0x026fa,),  # Tent
+        (0x026fd, 0x026fd,),  # Fuel Pump
+        (0x02705, 0x02705,),  # White Heavy Check Mark
+        (0x0270a, 0x0270b,),  # Raised Fist             ..Raised Hand
+        (0x02728, 0x02728,),  # Sparkles
+        (0x0274c, 0x0274c,),  # Cross Mark
+        (0x0274e, 0x0274e,),  # Negative Squared Cross Mark
+        (0x02753, 0x02755,),  # Black Question Mark Orna..White Exclamation Mark O
+        (0x02757, 0x02757,),  # Heavy Exclamation Mark Symbol
+        (0x02795, 0x02797,),  # Heavy Plus Sign         ..Heavy Division Sign
+        (0x027b0, 0x027b0,),  # Curly Loop
+        (0x027bf, 0x027bf,),  # Double Curly Loop
+        (0x02b1b, 0x02b1c,),  # Black Large Square      ..White Large Square
+        (0x02b50, 0x02b50,),  # White Medium Star
+        (0x02b55, 0x02b55,),  # Heavy Large Circle
+        (0x02e80, 0x02e99,),  # Cjk Radical Repeat      ..Cjk Radical Rap
+        (0x02e9b, 0x02ef3,),  # Cjk Radical Choke       ..Cjk Radical C-simplified
+        (0x02f00, 0x02fd5,),  # Kangxi Radical One      ..Kangxi Radical Flute
+        (0x02ff0, 0x03029,),  # Ideographic Description ..Hangzhou Numeral Nine
+        (0x03030, 0x0303e,),  # Wavy Dash               ..Ideographic Variation In
+        (0x03041, 0x03096,),  # Hiragana Letter Small A ..Hiragana Letter Small Ke
+        (0x0309b, 0x030ff,),  # Katakana-hiragana Voiced..Katakana Digraph Koto
+        (0x03105, 0x0312f,),  # Bopomofo Letter B       ..Bopomofo Letter Nn
+        (0x03131, 0x03163,),  # Hangul Letter Kiyeok    ..Hangul Letter I
+        (0x03165, 0x0318e,),  # Hangul Letter Ssangnieun..Hangul Letter Araeae
+        (0x03190, 0x031e5,),  # Ideographic Annotation L..Cjk Stroke Szp
+        (0x031ef, 0x0321e,),  # Ideographic Description ..Parenthesized Korean Cha
+        (0x03220, 0x03247,),  # Parenthesized Ideograph ..Circled Ideograph Koto
+        (0x03250, 0x0a48c,),  # Partnership Sign        ..Yi Syllable Yyr
+        (0x0a490, 0x0a4c6,),  # Yi Radical Qot          ..Yi Radical Ke
+        (0x0a960, 0x0a97c,),  # Hangul Choseong Tikeut-m..Hangul Choseong Ssangyeo
+        (0x0ac00, 0x0d7a3,),  # Hangul Syllable Ga      ..Hangul Syllable Hih
+        (0x0f900, 0x0faff,),  # Cjk Compatibility Ideogr..(nil)
+        (0x0fe10, 0x0fe19,),  # Presentation Form For Ve..Presentation Form For Ve
+        (0x0fe30, 0x0fe52,),  # Presentation Form For Ve..Small Full Stop
+        (0x0fe54, 0x0fe66,),  # Small Semicolon         ..Small Equals Sign
+        (0x0fe68, 0x0fe6b,),  # Small Reverse Solidus   ..Small Commercial At
+        (0x0ff01, 0x0ff60,),  # Fullwidth Exclamation Ma..Fullwidth Right White Pa
+        (0x0ffe0, 0x0ffe6,),  # Fullwidth Cent Sign     ..Fullwidth Won Sign
+        (0x16fe0, 0x16fe3,),  # Tangut Iteration Mark   ..Old Chinese Iteration Ma
+        (0x16ff2, 0x16ff6,),  # Chinese Small Simplified..Yangqin Sign Slow Two Be
+        (0x17000, 0x18cd5,),  # (nil)                   ..Khitan Small Script Char
+        (0x18cff, 0x18d1e,),  # Khitan Small Script Char..(nil)
+        (0x18d80, 0x18df2,),  # Tangut Component-769    ..Tangut Component-883
+        (0x1aff0, 0x1aff3,),  # Katakana Letter Minnan T..Katakana Letter Minnan T
+        (0x1aff5, 0x1affb,),  # Katakana Letter Minnan T..Katakana Letter Minnan N
+        (0x1affd, 0x1affe,),  # Katakana Letter Minnan N..Katakana Letter Minnan N
+        (0x1b000, 0x1b122,),  # Katakana Letter Archaic ..Katakana Letter Archaic
+        (0x1b132, 0x1b132,),  # Hiragana Letter Small Ko
+        (0x1b150, 0x1b152,),  # Hiragana Letter Small Wi..Hiragana Letter Small Wo
+        (0x1b155, 0x1b155,),  # Katakana Letter Small Ko
+        (0x1b164, 0x1b167,),  # Katakana Letter Small Wi..Katakana Letter Small N
+        (0x1b170, 0x1b2fb,),  # Nushu Character-1b170   ..Nushu Character-1b2fb
+        (0x1d300, 0x1d356,),  # Monogram For Earth      ..Tetragram For Fostering
+        (0x1d360, 0x1d376,),  # Counting Rod Unit Digit ..Ideographic Tally Mark F
+        (0x1f004, 0x1f004,),  # Mahjong Tile Red Dragon
+        (0x1f0cf, 0x1f0cf,),  # Playing Card Black Joker
+        (0x1f18e, 0x1f18e,),  # Negative Squared Ab
+        (0x1f191, 0x1f19a,),  # Squared Cl              ..Squared Vs
+        (0x1f1e6, 0x1f202,),  # Regional Indicator Symbo..Squared Katakana Sa
+        (0x1f210, 0x1f23b,),  # Squared Cjk Unified Ideo..Squared Cjk Unified Ideo
+        (0x1f240, 0x1f248,),  # Tortoise Shell Bracketed..Tortoise Shell Bracketed
+        (0x1f250, 0x1f251,),  # Circled Ideograph Advant..Circled Ideograph Accept
+        (0x1f260, 0x1f265,),  # Rounded Symbol For Fu   ..Rounded Symbol For Cai
+        (0x1f300, 0x1f320,),  # Cyclone                 ..Shooting Star
+        (0x1f32d, 0x1f335,),  # Hot Dog                 ..Cactus
+        (0x1f337, 0x1f37c,),  # Tulip                   ..Baby Bottle
+        (0x1f37e, 0x1f393,),  # Bottle With Popping Cork..Graduation Cap
+        (0x1f3a0, 0x1f3ca,),  # Carousel Horse          ..Swimmer
+        (0x1f3cf, 0x1f3d3,),  # Cricket Bat And Ball    ..Table Tennis Paddle And
+        (0x1f3e0, 0x1f3f0,),  # House Building          ..European Castle
+        (0x1f3f4, 0x1f3f4,),  # Waving Black Flag
+        (0x1f3f8, 0x1f43e,),  # Badminton Racquet And Sh..Paw Prints
+        (0x1f440, 0x1f440,),  # Eyes
+        (0x1f442, 0x1f4fc,),  # Ear                     ..Videocassette
+        (0x1f4ff, 0x1f53d,),  # Prayer Beads            ..Down-pointing Small Red
+        (0x1f54b, 0x1f54e,),  # Kaaba                   ..Menorah With Nine Branch
+        (0x1f550, 0x1f567,),  # Clock Face One Oclock   ..Clock Face Twelve-thirty
+        (0x1f57a, 0x1f57a,),  # Man Dancing
+        (0x1f595, 0x1f596,),  # Reversed Hand With Middl..Raised Hand With Part Be
+        (0x1f5a4, 0x1f5a4,),  # Black Heart
+        (0x1f5fb, 0x1f64f,),  # Mount Fuji              ..Person With Folded Hands
+        (0x1f680, 0x1f6c5,),  # Rocket                  ..Left Luggage
+        (0x1f6cc, 0x1f6cc,),  # Sleeping Accommodation
+        (0x1f6d0, 0x1f6d2,),  # Place Of Worship        ..Shopping Trolley
+        (0x1f6d5, 0x1f6d8,),  # Hindu Temple            ..Landslide
+        (0x1f6dc, 0x1f6df,),  # Wireless                ..Ring Buoy
+        (0x1f6eb, 0x1f6ec,),  # Airplane Departure      ..Airplane Arriving
+        (0x1f6f4, 0x1f6fc,),  # Scooter                 ..Roller Skate
+        (0x1f7e0, 0x1f7eb,),  # Large Orange Circle     ..Large Brown Square
+        (0x1f7f0, 0x1f7f0,),  # Heavy Equals Sign
+        (0x1f90c, 0x1f93a,),  # Pinched Fingers         ..Fencer
+        (0x1f93c, 0x1f945,),  # Wrestlers               ..Goal Net
+        (0x1f947, 0x1f9ff,),  # First Place Medal       ..Nazar Amulet
+        (0x1fa70, 0x1fa7c,),  # Ballet Shoes            ..Crutch
+        (0x1fa80, 0x1fa8a,),  # Yo-yo                   ..Trombone
+        (0x1fa8e, 0x1fac6,),  # Treasure Chest          ..Fingerprint
+        (0x1fac8, 0x1fac8,),  # Hairy Creature
+        (0x1facd, 0x1fadc,),  # Orca                    ..Root Vegetable
+        (0x1fadf, 0x1faea,),  # Splatter                ..Distorted Face
+        (0x1faef, 0x1faf8,),  # Fight Cloud             ..Rightwards Pushing Hand
+        (0x20000, 0x2fffd,),  # Cjk Unified Ideograph-20..(nil)
+        (0x30000, 0x3fffd,),  # Cjk Unified Ideograph-30..(nil)
+    ),
+}
diff --git a/lib/wcwidth/table_zero.py b/lib/wcwidth/table_zero.py
new file mode 100644
index 0000000..c440bfc
--- /dev/null
+++ b/lib/wcwidth/table_zero.py
@@ -0,0 +1,350 @@
+"""
+Exports ZERO_WIDTH table keyed by supporting unicode version level.
+
+This code generated by wcwidth/bin/update-tables.py on 2026-01-30 00:48:24 UTC.
+"""
+# pylint: disable=duplicate-code
+ZERO_WIDTH = {
+    '17.0.0': (
+        # Source: DerivedGeneralCategory-17.0.0.txt
+        # Date: 2025-07-24, 00:12:50 GMT
+        #
+        (0x00000, 0x00000,),  # (nil)
+        (0x00300, 0x0036f,),  # Combining Grave Accent  ..Combining Latin Small Le
+        (0x00483, 0x00489,),  # Combining Cyrillic Titlo..Combining Cyrillic Milli
+        (0x00591, 0x005bd,),  # Hebrew Accent Etnahta   ..Hebrew Point Meteg
+        (0x005bf, 0x005bf,),  # Hebrew Point Rafe
+        (0x005c1, 0x005c2,),  # Hebrew Point Shin Dot   ..Hebrew Point Sin Dot
+        (0x005c4, 0x005c5,),  # Hebrew Mark Upper Dot   ..Hebrew Mark Lower Dot
+        (0x005c7, 0x005c7,),  # Hebrew Point Qamats Qatan
+        (0x00610, 0x0061a,),  # Arabic Sign Sallallahou ..Arabic Small Kasra
+        (0x0061c, 0x0061c,),  # Arabic Letter Mark
+        (0x0064b, 0x0065f,),  # Arabic Fathatan         ..Arabic Wavy Hamza Below
+        (0x00670, 0x00670,),  # Arabic Letter Superscript Alef
+        (0x006d6, 0x006dc,),  # Arabic Small High Ligatu..Arabic Small High Seen
+        (0x006df, 0x006e4,),  # Arabic Small High Rounde..Arabic Small High Madda
+        (0x006e7, 0x006e8,),  # Arabic Small High Yeh   ..Arabic Small High Noon
+        (0x006ea, 0x006ed,),  # Arabic Empty Centre Low ..Arabic Small Low Meem
+        (0x00711, 0x00711,),  # Syriac Letter Superscript Alaph
+        (0x00730, 0x0074a,),  # Syriac Pthaha Above     ..Syriac Barrekh
+        (0x007a6, 0x007b0,),  # Thaana Abafili          ..Thaana Sukun
+        (0x007eb, 0x007f3,),  # Nko Combining Short High..Nko Combining Double Dot
+        (0x007fd, 0x007fd,),  # Nko Dantayalan
+        (0x00816, 0x00819,),  # Samaritan Mark In       ..Samaritan Mark Dagesh
+        (0x0081b, 0x00823,),  # Samaritan Mark Epentheti..Samaritan Vowel Sign A
+        (0x00825, 0x00827,),  # Samaritan Vowel Sign Sho..Samaritan Vowel Sign U
+        (0x00829, 0x0082d,),  # Samaritan Vowel Sign Lon..Samaritan Mark Nequdaa
+        (0x00859, 0x0085b,),  # Mandaic Affrication Mark..Mandaic Gemination Mark
+        (0x00897, 0x0089f,),  # Arabic Pepet            ..Arabic Half Madda Over M
+        (0x008ca, 0x008e1,),  # Arabic Small High Farsi ..Arabic Small High Sign S
+        (0x008e3, 0x00903,),  # Arabic Turned Damma Belo..Devanagari Sign Visarga
+        (0x0093a, 0x0093c,),  # Devanagari Vowel Sign Oe..Devanagari Sign Nukta
+        (0x0093e, 0x0094f,),  # Devanagari Vowel Sign Aa..Devanagari Vowel Sign Aw
+        (0x00951, 0x00957,),  # Devanagari Stress Sign U..Devanagari Vowel Sign Uu
+        (0x00962, 0x00963,),  # Devanagari Vowel Sign Vo..Devanagari Vowel Sign Vo
+        (0x00981, 0x00983,),  # Bengali Sign Candrabindu..Bengali Sign Visarga
+        (0x009bc, 0x009bc,),  # Bengali Sign Nukta
+        (0x009be, 0x009c4,),  # Bengali Vowel Sign Aa   ..Bengali Vowel Sign Vocal
+        (0x009c7, 0x009c8,),  # Bengali Vowel Sign E    ..Bengali Vowel Sign Ai
+        (0x009cb, 0x009cd,),  # Bengali Vowel Sign O    ..Bengali Sign Virama
+        (0x009d7, 0x009d7,),  # Bengali Au Length Mark
+        (0x009e2, 0x009e3,),  # Bengali Vowel Sign Vocal..Bengali Vowel Sign Vocal
+        (0x009fe, 0x009fe,),  # Bengali Sandhi Mark
+        (0x00a01, 0x00a03,),  # Gurmukhi Sign Adak Bindi..Gurmukhi Sign Visarga
+        (0x00a3c, 0x00a3c,),  # Gurmukhi Sign Nukta
+        (0x00a3e, 0x00a42,),  # Gurmukhi Vowel Sign Aa  ..Gurmukhi Vowel Sign Uu
+        (0x00a47, 0x00a48,),  # Gurmukhi Vowel Sign Ee  ..Gurmukhi Vowel Sign Ai
+        (0x00a4b, 0x00a4d,),  # Gurmukhi Vowel Sign Oo  ..Gurmukhi Sign Virama
+        (0x00a51, 0x00a51,),  # Gurmukhi Sign Udaat
+        (0x00a70, 0x00a71,),  # Gurmukhi Tippi          ..Gurmukhi Addak
+        (0x00a75, 0x00a75,),  # Gurmukhi Sign Yakash
+        (0x00a81, 0x00a83,),  # Gujarati Sign Candrabind..Gujarati Sign Visarga
+        (0x00abc, 0x00abc,),  # Gujarati Sign Nukta
+        (0x00abe, 0x00ac5,),  # Gujarati Vowel Sign Aa  ..Gujarati Vowel Sign Cand
+        (0x00ac7, 0x00ac9,),  # Gujarati Vowel Sign E   ..Gujarati Vowel Sign Cand
+        (0x00acb, 0x00acd,),  # Gujarati Vowel Sign O   ..Gujarati Sign Virama
+        (0x00ae2, 0x00ae3,),  # Gujarati Vowel Sign Voca..Gujarati Vowel Sign Voca
+        (0x00afa, 0x00aff,),  # Gujarati Sign Sukun     ..Gujarati Sign Two-circle
+        (0x00b01, 0x00b03,),  # Oriya Sign Candrabindu  ..Oriya Sign Visarga
+        (0x00b3c, 0x00b3c,),  # Oriya Sign Nukta
+        (0x00b3e, 0x00b44,),  # Oriya Vowel Sign Aa     ..Oriya Vowel Sign Vocalic
+        (0x00b47, 0x00b48,),  # Oriya Vowel Sign E      ..Oriya Vowel Sign Ai
+        (0x00b4b, 0x00b4d,),  # Oriya Vowel Sign O      ..Oriya Sign Virama
+        (0x00b55, 0x00b57,),  # Oriya Sign Overline     ..Oriya Au Length Mark
+        (0x00b62, 0x00b63,),  # Oriya Vowel Sign Vocalic..Oriya Vowel Sign Vocalic
+        (0x00b82, 0x00b82,),  # Tamil Sign Anusvara
+        (0x00bbe, 0x00bc2,),  # Tamil Vowel Sign Aa     ..Tamil Vowel Sign Uu
+        (0x00bc6, 0x00bc8,),  # Tamil Vowel Sign E      ..Tamil Vowel Sign Ai
+        (0x00bca, 0x00bcd,),  # Tamil Vowel Sign O      ..Tamil Sign Virama
+        (0x00bd7, 0x00bd7,),  # Tamil Au Length Mark
+        (0x00c00, 0x00c04,),  # Telugu Sign Combining Ca..Telugu Sign Combining An
+        (0x00c3c, 0x00c3c,),  # Telugu Sign Nukta
+        (0x00c3e, 0x00c44,),  # Telugu Vowel Sign Aa    ..Telugu Vowel Sign Vocali
+        (0x00c46, 0x00c48,),  # Telugu Vowel Sign E     ..Telugu Vowel Sign Ai
+        (0x00c4a, 0x00c4d,),  # Telugu Vowel Sign O     ..Telugu Sign Virama
+        (0x00c55, 0x00c56,),  # Telugu Length Mark      ..Telugu Ai Length Mark
+        (0x00c62, 0x00c63,),  # Telugu Vowel Sign Vocali..Telugu Vowel Sign Vocali
+        (0x00c81, 0x00c83,),  # Kannada Sign Candrabindu..Kannada Sign Visarga
+        (0x00cbc, 0x00cbc,),  # Kannada Sign Nukta
+        (0x00cbe, 0x00cc4,),  # Kannada Vowel Sign Aa   ..Kannada Vowel Sign Vocal
+        (0x00cc6, 0x00cc8,),  # Kannada Vowel Sign E    ..Kannada Vowel Sign Ai
+        (0x00cca, 0x00ccd,),  # Kannada Vowel Sign O    ..Kannada Sign Virama
+        (0x00cd5, 0x00cd6,),  # Kannada Length Mark     ..Kannada Ai Length Mark
+        (0x00ce2, 0x00ce3,),  # Kannada Vowel Sign Vocal..Kannada Vowel Sign Vocal
+        (0x00cf3, 0x00cf3,),  # Kannada Sign Combining Anusvara Above Right
+        (0x00d00, 0x00d03,),  # Malayalam Sign Combining..Malayalam Sign Visarga
+        (0x00d3b, 0x00d3c,),  # Malayalam Sign Vertical ..Malayalam Sign Circular
+        (0x00d3e, 0x00d44,),  # Malayalam Vowel Sign Aa ..Malayalam Vowel Sign Voc
+        (0x00d46, 0x00d48,),  # Malayalam Vowel Sign E  ..Malayalam Vowel Sign Ai
+        (0x00d4a, 0x00d4d,),  # Malayalam Vowel Sign O  ..Malayalam Sign Virama
+        (0x00d57, 0x00d57,),  # Malayalam Au Length Mark
+        (0x00d62, 0x00d63,),  # Malayalam Vowel Sign Voc..Malayalam Vowel Sign Voc
+        (0x00d81, 0x00d83,),  # Sinhala Sign Candrabindu..Sinhala Sign Visargaya
+        (0x00dca, 0x00dca,),  # Sinhala Sign Al-lakuna
+        (0x00dcf, 0x00dd4,),  # Sinhala Vowel Sign Aela-..Sinhala Vowel Sign Ketti
+        (0x00dd6, 0x00dd6,),  # Sinhala Vowel Sign Diga Paa-pilla
+        (0x00dd8, 0x00ddf,),  # Sinhala Vowel Sign Gaett..Sinhala Vowel Sign Gayan
+        (0x00df2, 0x00df3,),  # Sinhala Vowel Sign Diga ..Sinhala Vowel Sign Diga
+        (0x00e31, 0x00e31,),  # Thai Character Mai Han-akat
+        (0x00e34, 0x00e3a,),  # Thai Character Sara I   ..Thai Character Phinthu
+        (0x00e47, 0x00e4e,),  # Thai Character Maitaikhu..Thai Character Yamakkan
+        (0x00eb1, 0x00eb1,),  # Lao Vowel Sign Mai Kan
+        (0x00eb4, 0x00ebc,),  # Lao Vowel Sign I        ..Lao Semivowel Sign Lo
+        (0x00ec8, 0x00ece,),  # Lao Tone Mai Ek         ..Lao Yamakkan
+        (0x00f18, 0x00f19,),  # Tibetan Astrological Sig..Tibetan Astrological Sig
+        (0x00f35, 0x00f35,),  # Tibetan Mark Ngas Bzung Nyi Zla
+        (0x00f37, 0x00f37,),  # Tibetan Mark Ngas Bzung Sgor Rtags
+        (0x00f39, 0x00f39,),  # Tibetan Mark Tsa -phru
+        (0x00f3e, 0x00f3f,),  # Tibetan Sign Yar Tshes  ..Tibetan Sign Mar Tshes
+        (0x00f71, 0x00f84,),  # Tibetan Vowel Sign Aa   ..Tibetan Mark Halanta
+        (0x00f86, 0x00f87,),  # Tibetan Sign Lci Rtags  ..Tibetan Sign Yang Rtags
+        (0x00f8d, 0x00f97,),  # Tibetan Subjoined Sign L..Tibetan Subjoined Letter
+        (0x00f99, 0x00fbc,),  # Tibetan Subjoined Letter..Tibetan Subjoined Letter
+        (0x00fc6, 0x00fc6,),  # Tibetan Symbol Padma Gdan
+        (0x0102b, 0x0103e,),  # Myanmar Vowel Sign Tall ..Myanmar Consonant Sign M
+        (0x01056, 0x01059,),  # Myanmar Vowel Sign Vocal..Myanmar Vowel Sign Vocal
+        (0x0105e, 0x01060,),  # Myanmar Consonant Sign M..Myanmar Consonant Sign M
+        (0x01062, 0x01064,),  # Myanmar Vowel Sign Sgaw ..Myanmar Tone Mark Sgaw K
+        (0x01067, 0x0106d,),  # Myanmar Vowel Sign Weste..Myanmar Sign Western Pwo
+        (0x01071, 0x01074,),  # Myanmar Vowel Sign Geba ..Myanmar Vowel Sign Kayah
+        (0x01082, 0x0108d,),  # Myanmar Consonant Sign S..Myanmar Sign Shan Counci
+        (0x0108f, 0x0108f,),  # Myanmar Sign Rumai Palaung Tone-5
+        (0x0109a, 0x0109d,),  # Myanmar Sign Khamti Tone..Myanmar Vowel Sign Aiton
+        (0x01160, 0x011ff,),  # Hangul Jungseong Filler ..Hangul Jongseong Ssangni
+        (0x0135d, 0x0135f,),  # Ethiopic Combining Gemin..Ethiopic Combining Gemin
+        (0x01712, 0x01715,),  # Tagalog Vowel Sign I    ..Tagalog Sign Pamudpod
+        (0x01732, 0x01734,),  # Hanunoo Vowel Sign I    ..Hanunoo Sign Pamudpod
+        (0x01752, 0x01753,),  # Buhid Vowel Sign I      ..Buhid Vowel Sign U
+        (0x01772, 0x01773,),  # Tagbanwa Vowel Sign I   ..Tagbanwa Vowel Sign U
+        (0x017b4, 0x017d3,),  # Khmer Vowel Inherent Aq ..Khmer Sign Bathamasat
+        (0x017dd, 0x017dd,),  # Khmer Sign Atthacan
+        (0x0180b, 0x0180f,),  # Mongolian Free Variation..Mongolian Free Variation
+        (0x01885, 0x01886,),  # Mongolian Letter Ali Gal..Mongolian Letter Ali Gal
+        (0x018a9, 0x018a9,),  # Mongolian Letter Ali Gali Dagalga
+        (0x01920, 0x0192b,),  # Limbu Vowel Sign A      ..Limbu Subjoined Letter W
+        (0x01930, 0x0193b,),  # Limbu Small Letter Ka   ..Limbu Sign Sa-i
+        (0x01a17, 0x01a1b,),  # Buginese Vowel Sign I   ..Buginese Vowel Sign Ae
+        (0x01a55, 0x01a5e,),  # Tai Tham Consonant Sign ..Tai Tham Consonant Sign
+        (0x01a60, 0x01a7c,),  # Tai Tham Sign Sakot     ..Tai Tham Sign Khuen-lue
+        (0x01a7f, 0x01a7f,),  # Tai Tham Combining Cryptogrammic Dot
+        (0x01ab0, 0x01add,),  # Combining Doubled Circum..Combining Dot-and-ring B
+        (0x01ae0, 0x01aeb,),  # Combining Left Tack Abov..Combining Double Rightwa
+        (0x01b00, 0x01b04,),  # Balinese Sign Ulu Ricem ..Balinese Sign Bisah
+        (0x01b34, 0x01b44,),  # Balinese Sign Rerekan   ..Balinese Adeg Adeg
+        (0x01b6b, 0x01b73,),  # Balinese Musical Symbol ..Balinese Musical Symbol
+        (0x01b80, 0x01b82,),  # Sundanese Sign Panyecek ..Sundanese Sign Pangwisad
+        (0x01ba1, 0x01bad,),  # Sundanese Consonant Sign..Sundanese Consonant Sign
+        (0x01be6, 0x01bf3,),  # Batak Sign Tompi        ..Batak Panongonan
+        (0x01c24, 0x01c37,),  # Lepcha Subjoined Letter ..Lepcha Sign Nukta
+        (0x01cd0, 0x01cd2,),  # Vedic Tone Karshana     ..Vedic Tone Prenkha
+        (0x01cd4, 0x01ce8,),  # Vedic Sign Yajurvedic Mi..Vedic Sign Visarga Anuda
+        (0x01ced, 0x01ced,),  # Vedic Sign Tiryak
+        (0x01cf4, 0x01cf4,),  # Vedic Tone Candra Above
+        (0x01cf7, 0x01cf9,),  # Vedic Sign Atikrama     ..Vedic Tone Double Ring A
+        (0x01dc0, 0x01dff,),  # Combining Dotted Grave A..Combining Right Arrowhea
+        (0x0200b, 0x0200f,),  # Zero Width Space        ..Right-to-left Mark
+        (0x02028, 0x0202e,),  # Line Separator          ..Right-to-left Override
+        (0x02060, 0x0206f,),  # Word Joiner             ..Nominal Digit Shapes
+        (0x020d0, 0x020f0,),  # Combining Left Harpoon A..Combining Asterisk Above
+        (0x02cef, 0x02cf1,),  # Coptic Combining Ni Abov..Coptic Combining Spiritu
+        (0x02d7f, 0x02d7f,),  # Tifinagh Consonant Joiner
+        (0x02de0, 0x02dff,),  # Combining Cyrillic Lette..Combining Cyrillic Lette
+        (0x0302a, 0x0302f,),  # Ideographic Level Tone M..Hangul Double Dot Tone M
+        (0x03099, 0x0309a,),  # Combining Katakana-hirag..Combining Katakana-hirag
+        (0x03164, 0x03164,),  # Hangul Filler
+        (0x0a66f, 0x0a672,),  # Combining Cyrillic Vzmet..Combining Cyrillic Thous
+        (0x0a674, 0x0a67d,),  # Combining Cyrillic Lette..Combining Cyrillic Payer
+        (0x0a69e, 0x0a69f,),  # Combining Cyrillic Lette..Combining Cyrillic Lette
+        (0x0a6f0, 0x0a6f1,),  # Bamum Combining Mark Koq..Bamum Combining Mark Tuk
+        (0x0a802, 0x0a802,),  # Syloti Nagri Sign Dvisvara
+        (0x0a806, 0x0a806,),  # Syloti Nagri Sign Hasanta
+        (0x0a80b, 0x0a80b,),  # Syloti Nagri Sign Anusvara
+        (0x0a823, 0x0a827,),  # Syloti Nagri Vowel Sign ..Syloti Nagri Vowel Sign
+        (0x0a82c, 0x0a82c,),  # Syloti Nagri Sign Alternate Hasanta
+        (0x0a880, 0x0a881,),  # Saurashtra Sign Anusvara..Saurashtra Sign Visarga
+        (0x0a8b4, 0x0a8c5,),  # Saurashtra Consonant Sig..Saurashtra Sign Candrabi
+        (0x0a8e0, 0x0a8f1,),  # Combining Devanagari Dig..Combining Devanagari Sig
+        (0x0a8ff, 0x0a8ff,),  # Devanagari Vowel Sign Ay
+        (0x0a926, 0x0a92d,),  # Kayah Li Vowel Ue       ..Kayah Li Tone Calya Plop
+        (0x0a947, 0x0a953,),  # Rejang Vowel Sign I     ..Rejang Virama
+        (0x0a980, 0x0a983,),  # Javanese Sign Panyangga ..Javanese Sign Wignyan
+        (0x0a9b3, 0x0a9c0,),  # Javanese Sign Cecak Telu..Javanese Pangkon
+        (0x0a9e5, 0x0a9e5,),  # Myanmar Sign Shan Saw
+        (0x0aa29, 0x0aa36,),  # Cham Vowel Sign Aa      ..Cham Consonant Sign Wa
+        (0x0aa43, 0x0aa43,),  # Cham Consonant Sign Final Ng
+        (0x0aa4c, 0x0aa4d,),  # Cham Consonant Sign Fina..Cham Consonant Sign Fina
+        (0x0aa7b, 0x0aa7d,),  # Myanmar Sign Pao Karen T..Myanmar Sign Tai Laing T
+        (0x0aab0, 0x0aab0,),  # Tai Viet Mai Kang
+        (0x0aab2, 0x0aab4,),  # Tai Viet Vowel I        ..Tai Viet Vowel U
+        (0x0aab7, 0x0aab8,),  # Tai Viet Mai Khit       ..Tai Viet Vowel Ia
+        (0x0aabe, 0x0aabf,),  # Tai Viet Vowel Am       ..Tai Viet Tone Mai Ek
+        (0x0aac1, 0x0aac1,),  # Tai Viet Tone Mai Tho
+        (0x0aaeb, 0x0aaef,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+        (0x0aaf5, 0x0aaf6,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Virama
+        (0x0abe3, 0x0abea,),  # Meetei Mayek Vowel Sign ..Meetei Mayek Vowel Sign
+        (0x0abec, 0x0abed,),  # Meetei Mayek Lum Iyek   ..Meetei Mayek Apun Iyek
+        (0x0d7b0, 0x0d7ff,),  # Hangul Jungseong O-yeo  ..(nil)
+        (0x0fb1e, 0x0fb1e,),  # Hebrew Point Judeo-spanish Varika
+        (0x0fe00, 0x0fe0f,),  # Variation Selector-1    ..Variation Selector-16
+        (0x0fe20, 0x0fe2f,),  # Combining Ligature Left ..Combining Cyrillic Titlo
+        (0x0feff, 0x0feff,),  # Zero Width No-break Space
+        (0x0ffa0, 0x0ffa0,),  # Halfwidth Hangul Filler
+        (0x0fff0, 0x0fffb,),  # (nil)                   ..Interlinear Annotation T
+        (0x101fd, 0x101fd,),  # Phaistos Disc Sign Combining Oblique Stroke
+        (0x102e0, 0x102e0,),  # Coptic Epact Thousands Mark
+        (0x10376, 0x1037a,),  # Combining Old Permic Let..Combining Old Permic Let
+        (0x10a01, 0x10a03,),  # Kharoshthi Vowel Sign I ..Kharoshthi Vowel Sign Vo
+        (0x10a05, 0x10a06,),  # Kharoshthi Vowel Sign E ..Kharoshthi Vowel Sign O
+        (0x10a0c, 0x10a0f,),  # Kharoshthi Vowel Length ..Kharoshthi Sign Visarga
+        (0x10a38, 0x10a3a,),  # Kharoshthi Sign Bar Abov..Kharoshthi Sign Dot Belo
+        (0x10a3f, 0x10a3f,),  # Kharoshthi Virama
+        (0x10ae5, 0x10ae6,),  # Manichaean Abbreviation ..Manichaean Abbreviation
+        (0x10d24, 0x10d27,),  # Hanifi Rohingya Sign Har..Hanifi Rohingya Sign Tas
+        (0x10d69, 0x10d6d,),  # Garay Vowel Sign E      ..Garay Consonant Nasaliza
+        (0x10eab, 0x10eac,),  # Yezidi Combining Hamza M..Yezidi Combining Madda M
+        (0x10efa, 0x10eff,),  # Arabic Double Vertical B..Arabic Small Low Word Ma
+        (0x10f46, 0x10f50,),  # Sogdian Combining Dot Be..Sogdian Combining Stroke
+        (0x10f82, 0x10f85,),  # Old Uyghur Combining Dot..Old Uyghur Combining Two
+        (0x11000, 0x11002,),  # Brahmi Sign Candrabindu ..Brahmi Sign Visarga
+        (0x11038, 0x11046,),  # Brahmi Vowel Sign Aa    ..Brahmi Virama
+        (0x11070, 0x11070,),  # Brahmi Sign Old Tamil Virama
+        (0x11073, 0x11074,),  # Brahmi Vowel Sign Old Ta..Brahmi Vowel Sign Old Ta
+        (0x1107f, 0x11082,),  # Brahmi Number Joiner    ..Kaithi Sign Visarga
+        (0x110b0, 0x110ba,),  # Kaithi Vowel Sign Aa    ..Kaithi Sign Nukta
+        (0x110c2, 0x110c2,),  # Kaithi Vowel Sign Vocalic R
+        (0x11100, 0x11102,),  # Chakma Sign Candrabindu ..Chakma Sign Visarga
+        (0x11127, 0x11134,),  # Chakma Vowel Sign A     ..Chakma Maayyaa
+        (0x11145, 0x11146,),  # Chakma Vowel Sign Aa    ..Chakma Vowel Sign Ei
+        (0x11173, 0x11173,),  # Mahajani Sign Nukta
+        (0x11180, 0x11182,),  # Sharada Sign Candrabindu..Sharada Sign Visarga
+        (0x111b3, 0x111c0,),  # Sharada Vowel Sign Aa   ..Sharada Sign Virama
+        (0x111c9, 0x111cc,),  # Sharada Sandhi Mark     ..Sharada Extra Short Vowe
+        (0x111ce, 0x111cf,),  # Sharada Vowel Sign Prish..Sharada Sign Inverted Ca
+        (0x1122c, 0x11237,),  # Khojki Vowel Sign Aa    ..Khojki Sign Shadda
+        (0x1123e, 0x1123e,),  # Khojki Sign Sukun
+        (0x11241, 0x11241,),  # Khojki Vowel Sign Vocalic R
+        (0x112df, 0x112ea,),  # Khudawadi Sign Anusvara ..Khudawadi Sign Virama
+        (0x11300, 0x11303,),  # Grantha Sign Combining A..Grantha Sign Visarga
+        (0x1133b, 0x1133c,),  # Combining Bindu Below   ..Grantha Sign Nukta
+        (0x1133e, 0x11344,),  # Grantha Vowel Sign Aa   ..Grantha Vowel Sign Vocal
+        (0x11347, 0x11348,),  # Grantha Vowel Sign Ee   ..Grantha Vowel Sign Ai
+        (0x1134b, 0x1134d,),  # Grantha Vowel Sign Oo   ..Grantha Sign Virama
+        (0x11357, 0x11357,),  # Grantha Au Length Mark
+        (0x11362, 0x11363,),  # Grantha Vowel Sign Vocal..Grantha Vowel Sign Vocal
+        (0x11366, 0x1136c,),  # Combining Grantha Digit ..Combining Grantha Digit
+        (0x11370, 0x11374,),  # Combining Grantha Letter..Combining Grantha Letter
+        (0x113b8, 0x113c0,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Vowel Sign
+        (0x113c2, 0x113c2,),  # Tulu-tigalari Vowel Sign Ee
+        (0x113c5, 0x113c5,),  # Tulu-tigalari Vowel Sign Ai
+        (0x113c7, 0x113ca,),  # Tulu-tigalari Vowel Sign..Tulu-tigalari Sign Candr
+        (0x113cc, 0x113d0,),  # Tulu-tigalari Sign Anusv..Tulu-tigalari Conjoiner
+        (0x113d2, 0x113d2,),  # Tulu-tigalari Gemination Mark
+        (0x113e1, 0x113e2,),  # Tulu-tigalari Vedic Tone..Tulu-tigalari Vedic Tone
+        (0x11435, 0x11446,),  # Newa Vowel Sign Aa      ..Newa Sign Nukta
+        (0x1145e, 0x1145e,),  # Newa Sandhi Mark
+        (0x114b0, 0x114c3,),  # Tirhuta Vowel Sign Aa   ..Tirhuta Sign Nukta
+        (0x115af, 0x115b5,),  # Siddham Vowel Sign Aa   ..Siddham Vowel Sign Vocal
+        (0x115b8, 0x115c0,),  # Siddham Vowel Sign E    ..Siddham Sign Nukta
+        (0x115dc, 0x115dd,),  # Siddham Vowel Sign Alter..Siddham Vowel Sign Alter
+        (0x11630, 0x11640,),  # Modi Vowel Sign Aa      ..Modi Sign Ardhacandra
+        (0x116ab, 0x116b7,),  # Takri Sign Anusvara     ..Takri Sign Nukta
+        (0x1171d, 0x1172b,),  # Ahom Consonant Sign Medi..Ahom Sign Killer
+        (0x1182c, 0x1183a,),  # Dogra Vowel Sign Aa     ..Dogra Sign Nukta
+        (0x11930, 0x11935,),  # Dives Akuru Vowel Sign A..Dives Akuru Vowel Sign E
+        (0x11937, 0x11938,),  # Dives Akuru Vowel Sign A..Dives Akuru Vowel Sign O
+        (0x1193b, 0x1193e,),  # Dives Akuru Sign Anusvar..Dives Akuru Virama
+        (0x11940, 0x11940,),  # Dives Akuru Medial Ya
+        (0x11942, 0x11943,),  # Dives Akuru Medial Ra   ..Dives Akuru Sign Nukta
+        (0x119d1, 0x119d7,),  # Nandinagari Vowel Sign A..Nandinagari Vowel Sign V
+        (0x119da, 0x119e0,),  # Nandinagari Vowel Sign E..Nandinagari Sign Virama
+        (0x119e4, 0x119e4,),  # Nandinagari Vowel Sign Prishthamatra E
+        (0x11a01, 0x11a0a,),  # Zanabazar Square Vowel S..Zanabazar Square Vowel L
+        (0x11a33, 0x11a39,),  # Zanabazar Square Final C..Zanabazar Square Sign Vi
+        (0x11a3b, 0x11a3e,),  # Zanabazar Square Cluster..Zanabazar Square Cluster
+        (0x11a47, 0x11a47,),  # Zanabazar Square Subjoiner
+        (0x11a51, 0x11a5b,),  # Soyombo Vowel Sign I    ..Soyombo Vowel Length Mar
+        (0x11a8a, 0x11a99,),  # Soyombo Final Consonant ..Soyombo Subjoiner
+        (0x11b60, 0x11b67,),  # Sharada Vowel Sign Oe   ..Sharada Vowel Sign Candr
+        (0x11c2f, 0x11c36,),  # Bhaiksuki Vowel Sign Aa ..Bhaiksuki Vowel Sign Voc
+        (0x11c38, 0x11c3f,),  # Bhaiksuki Vowel Sign E  ..Bhaiksuki Sign Virama
+        (0x11c92, 0x11ca7,),  # Marchen Subjoined Letter..Marchen Subjoined Letter
+        (0x11ca9, 0x11cb6,),  # Marchen Subjoined Letter..Marchen Sign Candrabindu
+        (0x11d31, 0x11d36,),  # Masaram Gondi Vowel Sign..Masaram Gondi Vowel Sign
+        (0x11d3a, 0x11d3a,),  # Masaram Gondi Vowel Sign E
+        (0x11d3c, 0x11d3d,),  # Masaram Gondi Vowel Sign..Masaram Gondi Vowel Sign
+        (0x11d3f, 0x11d45,),  # Masaram Gondi Vowel Sign..Masaram Gondi Virama
+        (0x11d47, 0x11d47,),  # Masaram Gondi Ra-kara
+        (0x11d8a, 0x11d8e,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+        (0x11d90, 0x11d91,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Vowel Sign
+        (0x11d93, 0x11d97,),  # Gunjala Gondi Vowel Sign..Gunjala Gondi Virama
+        (0x11ef3, 0x11ef6,),  # Makasar Vowel Sign I    ..Makasar Vowel Sign O
+        (0x11f00, 0x11f01,),  # Kawi Sign Candrabindu   ..Kawi Sign Anusvara
+        (0x11f03, 0x11f03,),  # Kawi Sign Visarga
+        (0x11f34, 0x11f3a,),  # Kawi Vowel Sign Aa      ..Kawi Vowel Sign Vocalic
+        (0x11f3e, 0x11f42,),  # Kawi Vowel Sign E       ..Kawi Conjoiner
+        (0x11f5a, 0x11f5a,),  # Kawi Sign Nukta
+        (0x13430, 0x13440,),  # Egyptian Hieroglyph Vert..Egyptian Hieroglyph Mirr
+        (0x13447, 0x13455,),  # Egyptian Hieroglyph Modi..Egyptian Hieroglyph Modi
+        (0x1611e, 0x1612f,),  # Gurung Khema Vowel Sign ..Gurung Khema Sign Tholho
+        (0x16af0, 0x16af4,),  # Bassa Vah Combining High..Bassa Vah Combining High
+        (0x16b30, 0x16b36,),  # Pahawh Hmong Mark Cim Tu..Pahawh Hmong Mark Cim Ta
+        (0x16f4f, 0x16f4f,),  # Miao Sign Consonant Modifier Bar
+        (0x16f51, 0x16f87,),  # Miao Sign Aspiration    ..Miao Vowel Sign Ui
+        (0x16f8f, 0x16f92,),  # Miao Tone Right         ..Miao Tone Below
+        (0x16fe4, 0x16fe4,),  # Khitan Small Script Filler
+        (0x16ff0, 0x16ff1,),  # Vietnamese Alternate Rea..Vietnamese Alternate Rea
+        (0x1bc9d, 0x1bc9e,),  # Duployan Thick Letter Se..Duployan Double Mark
+        (0x1bca0, 0x1bca3,),  # Shorthand Format Letter ..Shorthand Format Up Step
+        (0x1cf00, 0x1cf2d,),  # Znamenny Combining Mark ..Znamenny Combining Mark
+        (0x1cf30, 0x1cf46,),  # Znamenny Combining Tonal..Znamenny Priznak Modifie
+        (0x1d165, 0x1d169,),  # Musical Symbol Combining..Musical Symbol Combining
+        (0x1d16d, 0x1d182,),  # Musical Symbol Combining..Musical Symbol Combining
+        (0x1d185, 0x1d18b,),  # Musical Symbol Combining..Musical Symbol Combining
+        (0x1d1aa, 0x1d1ad,),  # Musical Symbol Combining..Musical Symbol Combining
+        (0x1d242, 0x1d244,),  # Combining Greek Musical ..Combining Greek Musical
+        (0x1da00, 0x1da36,),  # Signwriting Head Rim    ..Signwriting Air Sucking
+        (0x1da3b, 0x1da6c,),  # Signwriting Mouth Closed..Signwriting Excitement
+        (0x1da75, 0x1da75,),  # Signwriting Upper Body Tilting From Hip Joints
+        (0x1da84, 0x1da84,),  # Signwriting Location Head Neck
+        (0x1da9b, 0x1da9f,),  # Signwriting Fill Modifie..Signwriting Fill Modifie
+        (0x1daa1, 0x1daaf,),  # Signwriting Rotation Mod..Signwriting Rotation Mod
+        (0x1e000, 0x1e006,),  # Combining Glagolitic Let..Combining Glagolitic Let
+        (0x1e008, 0x1e018,),  # Combining Glagolitic Let..Combining Glagolitic Let
+        (0x1e01b, 0x1e021,),  # Combining Glagolitic Let..Combining Glagolitic Let
+        (0x1e023, 0x1e024,),  # Combining Glagolitic Let..Combining Glagolitic Let
+        (0x1e026, 0x1e02a,),  # Combining Glagolitic Let..Combining Glagolitic Let
+        (0x1e08f, 0x1e08f,),  # Combining Cyrillic Small Letter Byelorussian-ukr
+        (0x1e130, 0x1e136,),  # Nyiakeng Puachue Hmong T..Nyiakeng Puachue Hmong T
+        (0x1e2ae, 0x1e2ae,),  # Toto Sign Rising Tone
+        (0x1e2ec, 0x1e2ef,),  # Wancho Tone Tup         ..Wancho Tone Koini
+        (0x1e4ec, 0x1e4ef,),  # Nag Mundari Sign Muhor  ..Nag Mundari Sign Sutuh
+        (0x1e5ee, 0x1e5ef,),  # Ol Onal Sign Mu         ..Ol Onal Sign Ikir
+        (0x1e6e3, 0x1e6e3,),  # Tai Yo Sign Ue
+        (0x1e6e6, 0x1e6e6,),  # Tai Yo Sign Au
+        (0x1e6ee, 0x1e6ef,),  # Tai Yo Sign Ay          ..Tai Yo Sign Ang
+        (0x1e6f5, 0x1e6f5,),  # Tai Yo Sign Om
+        (0x1e8d0, 0x1e8d6,),  # Mende Kikakui Combining ..Mende Kikakui Combining
+        (0x1e944, 0x1e94a,),  # Adlam Alif Lengthener   ..Adlam Nukta
+        (0xe0000, 0xe0fff,),  # (nil)
+    ),
+}
diff --git a/lib/wcwidth/textwrap.py b/lib/wcwidth/textwrap.py
new file mode 100644
index 0000000..4582cd5
--- /dev/null
+++ b/lib/wcwidth/textwrap.py
@@ -0,0 +1,656 @@
+"""
+Sequence-aware text wrapping functions.
+
+This module provides functions for wrapping text that may contain terminal escape sequences, with
+proper handling of Unicode grapheme clusters and character display widths.
+"""
+from __future__ import annotations
+
+# std imports
+import re
+import secrets
+import textwrap
+
+from typing import TYPE_CHECKING, NamedTuple
+
+# local
+from .wcwidth import width as _width
+from .wcwidth import iter_sequences
+from .grapheme import iter_graphemes
+from .sgr_state import propagate_sgr as _propagate_sgr
+from .escape_sequences import ZERO_WIDTH_PATTERN
+
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Any, Literal
+
+
+class _HyperlinkState(NamedTuple):
+    """State for tracking an open OSC 8 hyperlink across line breaks."""
+
+    url: str  # hyperlink target URL
+    params: str  # id=xxx and other key=value pairs separated by :
+    terminator: str  # BEL (\x07) or ST (\x1b\\)
+
+
+# Hyperlink parsing: captures (params, url, terminator)
+_HYPERLINK_OPEN_RE = re.compile(r'\x1b]8;([^;]*);([^\x07\x1b]*)(\x07|\x1b\\)')
+
+
+def _parse_hyperlink_open(seq: str) -> _HyperlinkState | None:
+    """Parse OSC 8 open sequence, return state or None."""
+    if (m := _HYPERLINK_OPEN_RE.match(seq)):
+        return _HyperlinkState(url=m.group(2), params=m.group(1), terminator=m.group(3))
+    return None
+
+
+def _make_hyperlink_open(url: str, params: str, terminator: str) -> str:
+    """Generate OSC 8 open sequence."""
+    return f'\x1b]8;{params};{url}{terminator}'
+
+
+def _make_hyperlink_close(terminator: str) -> str:
+    """Generate OSC 8 close sequence."""
+    return f'\x1b]8;;{terminator}'
+
+
+class SequenceTextWrapper(textwrap.TextWrapper):
+    """
+    Sequence-aware text wrapper extending :class:`textwrap.TextWrapper`.
+
+    This wrapper properly handles terminal escape sequences and Unicode grapheme clusters when
+    calculating text width for wrapping.
+
+    This implementation is based on the SequenceTextWrapper from the 'blessed' library, with
+    contributions from Avram Lubkin and grayjk.
+
+    The key difference from the blessed implementation is the addition of grapheme cluster support
+    via :func:`~.iter_graphemes`, providing width calculation for ZWJ emoji sequences, VS-16 emojis
+    and variations, regional indicator flags, and combining characters.
+
+    OSC 8 hyperlinks are handled specially: when a hyperlink must span multiple lines, each line
+    receives complete open/close sequences with a shared ``id`` parameter, ensuring terminals
+    treat the fragments as a single hyperlink for hover underlining. If the original hyperlink
+    already has an ``id`` parameter, it is preserved; otherwise, one is generated.
+    """
+
+    def __init__(self, width: int = 70, *,
+                 control_codes: Literal['parse', 'strict', 'ignore'] = 'parse',
+                 tabsize: int = 8,
+                 ambiguous_width: int = 1,
+                 **kwargs: Any) -> None:
+        """
+        Initialize the wrapper.
+
+        :param width: Maximum line width in display cells.
+        :param control_codes: How to handle control sequences (see :func:`~.width`).
+        :param tabsize: Tab stop width for tab expansion.
+        :param ambiguous_width: Width to use for East Asian Ambiguous (A) characters.
+        :param kwargs: Additional arguments passed to :class:`textwrap.TextWrapper`.
+        """
+        super().__init__(width=width, **kwargs)
+        self.control_codes = control_codes
+        self.tabsize = tabsize
+        self.ambiguous_width = ambiguous_width
+
+    @staticmethod
+    def _next_hyperlink_id() -> str:
+        """Generate unique hyperlink id as 8-character hex string."""
+        return secrets.token_hex(4)
+
+    def _width(self, text: str) -> int:
+        """Measure text width accounting for sequences."""
+        return _width(text, control_codes=self.control_codes, tabsize=self.tabsize,
+                      ambiguous_width=self.ambiguous_width)
+
+    def _strip_sequences(self, text: str) -> str:
+        """Strip all terminal sequences from text."""
+        result = []
+        for segment, is_seq in iter_sequences(text):
+            if not is_seq:
+                result.append(segment)
+        return ''.join(result)
+
+    def _extract_sequences(self, text: str) -> str:
+        """Extract only terminal sequences from text."""
+        result = []
+        for segment, is_seq in iter_sequences(text):
+            if is_seq:
+                result.append(segment)
+        return ''.join(result)
+
+    def _split(self, text: str) -> list[str]:  # pylint: disable=too-many-locals
+        r"""
+        Sequence-aware variant of :meth:`textwrap.TextWrapper._split`.
+
+        This method ensures that terminal escape sequences don't interfere with the text splitting
+        logic, particularly for hyphen-based word breaking. It builds a position mapping from
+        stripped text to original text, calls the parent's _split on stripped text, then maps chunks
+        back.
+
+        OSC hyperlink sequences are treated as word boundaries::
+
+            >>> wrap('foo \x1b]8;;https://example.com\x07link\x1b]8;;\x07 bar', 6)
+            ['foo', '\x1b]8;;https://example.com\x07link\x1b]8;;\x07', 'bar']
+
+        Both BEL (``\x07``) and ST (``\x1b\\``) terminators are supported.
+        """
+        # pylint: disable=too-many-locals,too-many-branches
+        # Build a mapping from stripped text positions to original text positions.
+        #
+        # Track where each character ENDS so that sequences between characters
+        # attach to the following text (not preceding text). This ensures sequences
+        # aren't lost when whitespace is dropped.
+        #
+        # char_end[i] = position in original text right after the i-th stripped char
+        char_end: list[int] = []
+        stripped_text = ''
+        original_pos = 0
+        prev_was_hyperlink_close = False
+
+        for segment, is_seq in iter_sequences(text):
+            if not is_seq:
+                # Conditionally insert space after hyperlink close to force word boundary
+                if prev_was_hyperlink_close and segment and not segment[0].isspace():
+                    stripped_text += ' '
+                    char_end.append(original_pos)
+                for char in segment:
+                    original_pos += 1
+                    char_end.append(original_pos)
+                    stripped_text += char
+                prev_was_hyperlink_close = False
+            else:
+                is_hyperlink_close = segment.startswith(('\x1b]8;;\x1b\\', '\x1b]8;;\x07'))
+
+                # Conditionally insert space before OSC sequences to artificially create word
+                # boundary, but *not* before hyperlink close sequences, to ensure hyperlink is
+                # terminated on the same line.
+                if (segment.startswith('\x1b]') and stripped_text and not
+                        stripped_text[-1].isspace()):
+                    if not is_hyperlink_close:
+                        stripped_text += ' '
+                        char_end.append(original_pos)
+
+                # Escape sequences advance position but don't add to stripped text
+                original_pos += len(segment)
+                prev_was_hyperlink_close = is_hyperlink_close
+
+        # Add sentinel for final position
+        char_end.append(original_pos)
+
+        # Use parent's _split on the stripped text
+        # pylint: disable-next=protected-access
+        stripped_chunks = textwrap.TextWrapper._split(self, stripped_text)
+
+        # Handle text that contains only sequences (no visible characters).
+        # Return the sequences as a single chunk to preserve them.
+        if not stripped_chunks and text:
+            return [text]
+
+        # Map the chunks back to the original text with sequences
+        result: list[str] = []
+        stripped_pos = 0
+        num_chunks = len(stripped_chunks)
+
+        for idx, chunk in enumerate(stripped_chunks):
+            chunk_len = len(chunk)
+
+            # Start is where previous character ended (or 0 for first chunk)
+            start_orig = 0 if stripped_pos == 0 else char_end[stripped_pos - 1]
+
+            # End is where next character starts. For last chunk, use sentinel
+            # to include any trailing sequences.
+            if idx == num_chunks - 1:
+                end_orig = char_end[-1]  # sentinel includes trailing sequences
+            else:
+                end_orig = char_end[stripped_pos + chunk_len - 1]
+
+            # Extract the corresponding portion from the original text
+            # Skip empty chunks (from virtual spaces inserted at OSC boundaries)
+            if start_orig != end_orig:
+                result.append(text[start_orig:end_orig])
+            stripped_pos += chunk_len
+
+        return result
+
+    def _wrap_chunks(self, chunks: list[str]) -> list[str]:  # pylint: disable=too-many-branches
+        """
+        Wrap chunks into lines using sequence-aware width.
+
+        Override TextWrapper._wrap_chunks to use _width instead of len. Follows stdlib's algorithm:
+        greedily fill lines, handle long words.  Also handle OSC hyperlink processing. When
+        hyperlinks span multiple lines, each line gets complete open/close sequences with matching
+        id parameters for hover underlining continuity per OSC 8 spec.
+        """
+        # pylint: disable=too-many-branches,too-many-statements,too-complex,too-many-locals
+        # pylint: disable=too-many-nested-blocks
+        # the hyperlink code in particular really pushes the complexity rating of this method.
+        # preferring to keep it "all in one method" because of so much local state and manipulation.
+        if not chunks:
+            return []
+
+        if self.max_lines is not None:
+            if self.max_lines > 1:
+                indent = self.subsequent_indent
+            else:
+                indent = self.initial_indent
+            if (self._width(indent)
+                    + self._width(self.placeholder.lstrip())
+                    > self.width):
+                raise ValueError("placeholder too large for max width")
+
+        lines: list[str] = []
+        is_first_line = True
+
+        hyperlink_state: _HyperlinkState | None = None
+        # Track the id we're using for the current hyperlink continuation
+        current_hyperlink_id: str | None = None
+
+        # Arrange in reverse order so items can be efficiently popped
+        chunks = list(reversed(chunks))
+
+        while chunks:
+            current_line: list[str] = []
+            current_width = 0
+
+            # Get the indent and available width for current line
+            indent = self.initial_indent if is_first_line else self.subsequent_indent
+            line_width = self.width - self._width(indent)
+
+            # If continuing a hyperlink from previous line, prepend open sequence
+            if hyperlink_state is not None:
+                open_seq = _make_hyperlink_open(
+                    hyperlink_state.url, hyperlink_state.params, hyperlink_state.terminator)
+                chunks[-1] = open_seq + chunks[-1]
+
+            # Drop leading whitespace (except at very start)
+            # When dropping, transfer any sequences to the next chunk.
+            # Only drop if there's actual whitespace text, not if it's only sequences.
+            stripped = self._strip_sequences(chunks[-1])
+            if self.drop_whitespace and lines and stripped and not stripped.strip():
+                sequences = self._extract_sequences(chunks[-1])
+                del chunks[-1]
+                if sequences and chunks:
+                    chunks[-1] = sequences + chunks[-1]
+
+            # Greedily add chunks that fit
+            while chunks:
+                chunk = chunks[-1]
+                chunk_width = self._width(chunk)
+
+                if current_width + chunk_width <= line_width:
+                    current_line.append(chunks.pop())
+                    current_width += chunk_width
+                else:
+                    break
+
+            # Handle chunk that's too long for any line
+            if chunks and self._width(chunks[-1]) > line_width:
+                self._handle_long_word(
+                    chunks, current_line, current_width, line_width
+                )
+                current_width = self._width(''.join(current_line))
+                # Remove any empty chunks left by _handle_long_word
+                while chunks and not chunks[-1]:
+                    del chunks[-1]
+
+            # Drop trailing whitespace
+            # When dropping, transfer any sequences to the previous chunk.
+            # Only drop if there's actual whitespace text, not if it's only sequences.
+            stripped_last = self._strip_sequences(current_line[-1]) if current_line else ''
+            if (self.drop_whitespace and current_line and
+                    stripped_last and not stripped_last.strip()):
+                sequences = self._extract_sequences(current_line[-1])
+                current_width -= self._width(current_line[-1])
+                del current_line[-1]
+                if sequences and current_line:
+                    current_line[-1] = current_line[-1] + sequences
+
+            if current_line:
+                # Check whether this is a normal append or max_lines
+                # truncation. Matches stdlib textwrap precedence:
+                # normal if max_lines not set, not yet reached, or no
+                # remaining visible content that would need truncation.
+                no_more_content = (
+                    not chunks or
+                    self.drop_whitespace and
+                    len(chunks) == 1 and
+                    not self._strip_sequences(chunks[0]).strip()
+                )
+                if (self.max_lines is None or
+                        len(lines) + 1 < self.max_lines or
+                        no_more_content
+                        and current_width <= line_width):
+                    line_content = ''.join(current_line)
+
+                    # Track hyperlink state through this line's content
+                    new_state = self._track_hyperlink_state(line_content, hyperlink_state)
+
+                    # If we end inside a hyperlink, append close sequence
+                    if new_state is not None:
+                        # Ensure we have an id for continuation
+                        if current_hyperlink_id is None:
+                            if 'id=' in new_state.params:
+                                current_hyperlink_id = new_state.params
+                            elif new_state.params:
+                                # Prepend id to existing params (per OSC 8 spec, params can have
+                                # multiple key=value pairs separated by :)
+                                current_hyperlink_id = (
+                                    f'id={self._next_hyperlink_id()}:{new_state.params}')
+                            else:
+                                current_hyperlink_id = f'id={self._next_hyperlink_id()}'
+                        line_content += _make_hyperlink_close(new_state.terminator)
+
+                        # Also need to inject the id into the opening
+                        # sequence if it didn't have one
+                        if 'id=' not in new_state.params:
+                            # Find and replace the original open sequence with one that has id
+                            old_open = _make_hyperlink_open(
+                                new_state.url, new_state.params, new_state.terminator)
+                            new_open = _make_hyperlink_open(
+                                new_state.url, current_hyperlink_id, new_state.terminator)
+                            line_content = line_content.replace(old_open, new_open, 1)
+
+                        # Update state for next line, using computed id
+                        hyperlink_state = _HyperlinkState(
+                            new_state.url, current_hyperlink_id, new_state.terminator)
+                    else:
+                        hyperlink_state = None
+                        current_hyperlink_id = None  # Reset id when hyperlink closes
+
+                    # Strip trailing whitespace when drop_whitespace is enabled
+                    # (matches CPython #140627 fix behavior)
+                    if self.drop_whitespace:
+                        line_content = line_content.rstrip()
+                    lines.append(indent + line_content)
+                    is_first_line = False
+                else:
+                    # max_lines reached with remaining content —
+                    # pop chunks until placeholder fits, then break.
+                    placeholder_w = self._width(self.placeholder)
+                    while current_line:
+                        last_text = self._strip_sequences(current_line[-1])
+                        if (last_text.strip()
+                                and current_width + placeholder_w <= line_width):
+                            line_content = ''.join(current_line)
+                            new_state = self._track_hyperlink_state(
+                                line_content, hyperlink_state)
+                            if new_state is not None:
+                                line_content += _make_hyperlink_close(
+                                    new_state.terminator)
+                            lines.append(indent + line_content + self.placeholder)
+                            break
+                        current_width -= self._width(current_line[-1])
+                        del current_line[-1]
+                    else:
+                        if lines:
+                            prev_line = self._rstrip_visible(lines[-1])
+                            if (self._width(prev_line) + placeholder_w
+                                    <= self.width):
+                                lines[-1] = prev_line + self.placeholder
+                                break
+                        lines.append(indent + self.placeholder.lstrip())
+                    break
+
+        return lines
+
+    def _track_hyperlink_state(
+            self, text: str,
+            state: _HyperlinkState | None) -> _HyperlinkState | None:
+        """
+        Track hyperlink state through text.
+
+        :param text: Text to scan for hyperlink sequences.
+        :param state: Current state or None if outside hyperlink.
+        :returns: Updated state after processing text.
+        """
+        for segment, is_seq in iter_sequences(text):
+            if is_seq:
+                parsed_link = _parse_hyperlink_open(segment)
+                if parsed_link is not None and parsed_link.url:  # has URL = open
+                    state = parsed_link
+                elif segment.startswith(('\x1b]8;;\x1b\\', '\x1b]8;;\x07')):  # close
+                    state = None
+        return state
+
+    def _handle_long_word(self, reversed_chunks: list[str],
+                          cur_line: list[str], cur_len: int,
+                          width: int) -> None:
+        """
+        Sequence-aware :meth:`textwrap.TextWrapper._handle_long_word`.
+
+        This method ensures that word boundaries are not broken mid-sequence, and respects grapheme
+        cluster boundaries when breaking long words.
+        """
+        if width < 1:
+            space_left = 1
+        else:
+            space_left = width - cur_len
+
+        chunk = reversed_chunks[-1]
+
+        if self.break_long_words:
+            break_at_hyphen = False
+            hyphen_end = 0
+
+            # Handle break_on_hyphens: find last hyphen within space_left
+            if self.break_on_hyphens:
+                # Strip sequences to find hyphen in logical text
+                stripped = self._strip_sequences(chunk)
+                if len(stripped) > space_left:
+                    # Find last hyphen in the portion that fits
+                    hyphen_pos = stripped.rfind('-', 0, space_left)
+                    if hyphen_pos > 0 and any(c != '-' for c in stripped[:hyphen_pos]):
+                        # Map back to original position including sequences
+                        hyphen_end = self._map_stripped_pos_to_original(chunk, hyphen_pos + 1)
+                        break_at_hyphen = True
+
+            # Break at grapheme boundaries to avoid splitting multi-codepoint characters
+            if break_at_hyphen:
+                actual_end = hyphen_end
+            else:
+                actual_end = self._find_break_position(chunk, space_left)
+                # If no progress possible (e.g., wide char exceeds line width),
+                # force at least one grapheme to avoid infinite loop.
+                # Only force when cur_line is empty; if line has content,
+                # appending nothing is safe and the line will be committed.
+                if actual_end == 0 and not cur_line:
+                    actual_end = self._find_first_grapheme_end(chunk)
+            cur_line.append(chunk[:actual_end])
+            reversed_chunks[-1] = chunk[actual_end:]
+
+        elif not cur_line:
+            cur_line.append(reversed_chunks.pop())
+
+    def _map_stripped_pos_to_original(self, text: str, stripped_pos: int) -> int:
+        """Map a position in stripped text back to original text position."""
+        stripped_idx = 0
+        original_idx = 0
+
+        for segment, is_seq in iter_sequences(text):
+            if is_seq:
+                original_idx += len(segment)
+            elif stripped_idx + len(segment) > stripped_pos:
+                # Position is within this segment
+                return original_idx + (stripped_pos - stripped_idx)
+            else:
+                stripped_idx += len(segment)
+                original_idx += len(segment)
+
+        # Caller guarantees stripped_pos < total stripped chars, so we always
+        # return from within the loop. This line satisfies the type checker.
+        return original_idx  # pragma: no cover
+
+    def _find_break_position(self, text: str, max_width: int) -> int:
+        """Find string index in text that fits within max_width cells."""
+        idx = 0
+        width_so_far = 0
+
+        while idx < len(text):
+            char = text[idx]
+
+            # Skip escape sequences (they don't add width)
+            if char == '\x1b':
+                match = ZERO_WIDTH_PATTERN.match(text, idx)
+                if match:
+                    idx = match.end()
+                    continue
+
+            # Get grapheme (use start= to avoid slice allocation)
+            grapheme = next(iter_graphemes(text, start=idx))
+
+            grapheme_width = self._width(grapheme)
+            if width_so_far + grapheme_width > max_width:
+                return idx  # Found break point
+
+            width_so_far += grapheme_width
+            idx += len(grapheme)
+
+        # Caller guarantees chunk_width > max_width, so a grapheme always
+        # exceeds and we return from within the loop. Type checker requires this.
+        return idx  # pragma: no cover
+
+    def _find_first_grapheme_end(self, text: str) -> int:
+        """Find the end position of the first grapheme."""
+        return len(next(iter_graphemes(text)))
+
+    def _rstrip_visible(self, text: str) -> str:
+        """Strip trailing visible whitespace, preserving trailing sequences."""
+        segments = list(iter_sequences(text))
+        last_vis = -1
+        for i, (segment, is_seq) in enumerate(segments):
+            if not is_seq and segment.rstrip():
+                last_vis = i
+        if last_vis == -1:
+            return ''
+        result = []
+        for i, (segment, is_seq) in enumerate(segments):
+            if i < last_vis:
+                result.append(segment)
+            elif i == last_vis:
+                result.append(segment.rstrip())
+            elif is_seq:
+                result.append(segment)
+        return ''.join(result)
+
+
+def wrap(text: str, width: int = 70, *,
+         control_codes: Literal['parse', 'strict', 'ignore'] = 'parse',
+         tabsize: int = 8,
+         expand_tabs: bool = True,
+         replace_whitespace: bool = True,
+         ambiguous_width: int = 1,
+         initial_indent: str = '',
+         subsequent_indent: str = '',
+         fix_sentence_endings: bool = False,
+         break_long_words: bool = True,
+         break_on_hyphens: bool = True,
+         drop_whitespace: bool = True,
+         max_lines: int | None = None,
+         placeholder: str = ' [...]',
+         propagate_sgr: bool = True) -> list[str]:
+    r"""
+    Wrap text to fit within given width, returning a list of wrapped lines.
+
+    Like :func:`textwrap.wrap`, but measures width in display cells rather than
+    characters, correctly handling wide characters, combining marks, and terminal
+    escape sequences.
+
+    :param text: Text to wrap, may contain terminal sequences.
+    :param width: Maximum line width in display cells.
+    :param control_codes: How to handle terminal sequences (see :func:`~.width`).
+    :param tabsize: Tab stop width for tab expansion.
+    :param expand_tabs: If True (default), tab characters are expanded
+        to spaces using ``tabsize``.
+    :param replace_whitespace: If True (default), each whitespace character
+        is replaced with a single space after tab expansion. When False,
+        control whitespace like ``\n`` has zero display width (unlike
+        :func:`textwrap.wrap` which counts ``len()``), so wrap points
+        may differ from stdlib for non-space whitespace characters.
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :param initial_indent: String prepended to first line.
+    :param subsequent_indent: String prepended to subsequent lines.
+    :param fix_sentence_endings: If True, ensure sentences are always
+        separated by exactly two spaces.
+    :param break_long_words: If True, break words longer than width.
+    :param break_on_hyphens: If True, allow breaking at hyphens.
+    :param drop_whitespace: If True (default), whitespace at the beginning
+        and end of each line (after wrapping but before indenting) is dropped.
+        Set to False to preserve whitespace.
+    :param max_lines: If set, output contains at most this many lines, with
+        ``placeholder`` appended to the last line if the text was truncated.
+    :param placeholder: String appended to the last line when text is
+        truncated by ``max_lines``. Default is ``' [...]'``.
+    :param propagate_sgr: If True (default), SGR (terminal styling) sequences
+        are propagated across wrapped lines. Each line ends with a reset
+        sequence and the next line begins with the active style restored.
+    :returns: List of wrapped lines without trailing newlines.
+
+    SGR (terminal styling) sequences are propagated across wrapped lines
+    by default. Each line ends with a reset sequence and the next line
+    begins with the active style restored::
+
+        >>> wrap('\x1b[1;34mHello world\x1b[0m', width=6)
+        ['\x1b[1;34mHello\x1b[0m', '\x1b[1;34mworld\x1b[0m']
+
+    Set ``propagate_sgr=False`` to disable this behavior.
+
+    Like :func:`textwrap.wrap`, newlines in the input text are treated as
+    whitespace and collapsed. To preserve paragraph breaks, wrap each
+    paragraph separately::
+
+        >>> text = 'First line.\nSecond line.'
+        >>> wrap(text, 40)  # newline collapsed to space
+        ['First line. Second line.']
+        >>> [line for para in text.split('\n')
+        ...  for line in (wrap(para, 40) if para else [''])]
+        ['First line.', 'Second line.']
+
+    .. seealso::
+
+       :func:`textwrap.wrap`, :class:`textwrap.TextWrapper`
+           Standard library text wrapping (character-based).
+
+       :class:`.SequenceTextWrapper`
+           Class interface for advanced wrapping options.
+
+    .. versionadded:: 0.3.0
+
+    .. versionchanged:: 0.5.0
+       Added ``propagate_sgr`` parameter (default True).
+
+    .. versionchanged:: 0.6.0
+       Added ``expand_tabs``, ``replace_whitespace``, ``fix_sentence_endings``,
+       ``drop_whitespace``, ``max_lines``, and ``placeholder`` parameters.
+
+    Example::
+
+        >>> from wcwidth import wrap
+        >>> wrap('hello world', 5)
+        ['hello', 'world']
+        >>> wrap('中文字符', 4)  # CJK characters (2 cells each)
+        ['中文', '字符']
+    """
+    # pylint: disable=too-many-arguments,too-many-locals
+    wrapper = SequenceTextWrapper(
+        width=width,
+        control_codes=control_codes,
+        tabsize=tabsize,
+        expand_tabs=expand_tabs,
+        replace_whitespace=replace_whitespace,
+        ambiguous_width=ambiguous_width,
+        initial_indent=initial_indent,
+        subsequent_indent=subsequent_indent,
+        fix_sentence_endings=fix_sentence_endings,
+        break_long_words=break_long_words,
+        break_on_hyphens=break_on_hyphens,
+        drop_whitespace=drop_whitespace,
+        max_lines=max_lines,
+        placeholder=placeholder,
+    )
+    lines = wrapper.wrap(text)
+
+    if propagate_sgr:
+        lines = _propagate_sgr(lines)
+
+    return lines
diff --git a/lib/wcwidth/unicode_versions.py b/lib/wcwidth/unicode_versions.py
new file mode 100644
index 0000000..e848411
--- /dev/null
+++ b/lib/wcwidth/unicode_versions.py
@@ -0,0 +1,21 @@
+"""
+Exports function list_versions() for unicode version level support.
+
+This code generated by wcwidth/bin/update-tables.py on 2026-01-27 00:41:01 UTC.
+"""
+
+from __future__ import annotations
+
+
+def list_versions() -> tuple[str, ...]:
+    """
+    Return Unicode version levels supported by this module release.
+
+    .. versionchanged:: 0.5.0
+       Now returns a single-element tuple containing only the latest version.
+
+    :returns: Supported Unicode version numbers in ascending sorted order.
+    """
+    return (
+        "17.0.0",
+    )
diff --git a/lib/wcwidth/wcwidth.py b/lib/wcwidth/wcwidth.py
new file mode 100644
index 0000000..98e7a63
--- /dev/null
+++ b/lib/wcwidth/wcwidth.py
@@ -0,0 +1,1030 @@
+"""
+This is a python implementation of wcwidth() and wcswidth().
+
+https://github.com/jquast/wcwidth
+
+from Markus Kuhn's C code, retrieved from:
+
+    http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+
+This is an implementation of wcwidth() and wcswidth() (defined in
+IEEE Std 1002.1-2001) for Unicode.
+
+http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
+http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
+
+In fixed-width output devices, Latin characters all occupy a single
+"cell" position of equal width, whereas ideographic CJK characters
+occupy two such cells. Interoperability between terminal-line
+applications and (teletype-style) character terminals using the
+UTF-8 encoding requires agreement on which character should advance
+the cursor by how many cell positions. No established formal
+standards exist at present on which Unicode character shall occupy
+how many cell positions on character terminals. These routines are
+a first attempt of defining such behavior based on simple rules
+applied to data provided by the Unicode Consortium.
+
+For some graphical characters, the Unicode standard explicitly
+defines a character-cell width via the definition of the East Asian
+FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
+In all these cases, there is no ambiguity about which width a
+terminal shall use. For characters in the East Asian Ambiguous (A)
+class, the width choice depends purely on a preference of backward
+compatibility with either historic CJK or Western practice.
+Choosing single-width for these characters is easy to justify as
+the appropriate long-term solution, as the CJK practice of
+displaying these characters as double-width comes from historic
+implementation simplicity (8-bit encoded characters were displayed
+single-width and 16-bit ones double-width, even for Greek,
+Cyrillic, etc.) and not any typographic considerations.
+
+Much less clear is the choice of width for the Not East Asian
+(Neutral) class. Existing practice does not dictate a width for any
+of these characters. It would nevertheless make sense
+typographically to allocate two character cells to characters such
+as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
+represented adequately with a single-width glyph. The following
+routines at present merely assign a single-cell width to all
+neutral characters, in the interest of simplicity. This is not
+entirely satisfactory and should be reconsidered before
+establishing a formal standard in this area. At the moment, the
+decision which Not East Asian (Neutral) characters should be
+represented by double-width glyphs cannot yet be answered by
+applying a simple rule from the Unicode database content. Setting
+up a proper standard for the behavior of UTF-8 character terminals
+will require a careful analysis not only of each Unicode character,
+but also of each presentation form, something the author of these
+routines has avoided to do so far.
+
+http://www.unicode.org/unicode/reports/tr11/
+
+Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+"""
+
+from __future__ import annotations
+
+# std imports
+from functools import lru_cache
+
+from typing import TYPE_CHECKING
+
+# local
+from .bisearch import bisearch as _bisearch
+from .grapheme import iter_graphemes
+from .table_mc import CATEGORY_MC
+from .sgr_state import (_SGR_PATTERN,
+                        _SGR_STATE_DEFAULT,
+                        _sgr_state_update,
+                        _sgr_state_is_active,
+                        _sgr_state_to_sequence)
+from .table_vs16 import VS16_NARROW_TO_WIDE
+from .table_wide import WIDE_EASTASIAN
+from .table_zero import ZERO_WIDTH
+from .control_codes import ILLEGAL_CTRL, VERTICAL_CTRL, HORIZONTAL_CTRL, ZERO_WIDTH_CTRL
+from .table_grapheme import ISC_CONSONANT, EXTENDED_PICTOGRAPHIC, GRAPHEME_REGIONAL_INDICATOR
+from .table_ambiguous import AMBIGUOUS_EASTASIAN
+from .escape_sequences import (ZERO_WIDTH_PATTERN,
+                               CURSOR_LEFT_SEQUENCE,
+                               CURSOR_RIGHT_SEQUENCE,
+                               INDETERMINATE_EFFECT_SEQUENCE)
+from .unicode_versions import list_versions
+
+if TYPE_CHECKING:  # pragma: no cover
+    # std imports
+    from collections.abc import Iterator
+
+    from typing import Literal
+
+# Pre-compute table references for the latest (and only) Unicode version.
+_LATEST_VERSION = list_versions()[-1]
+_ZERO_WIDTH_TABLE = ZERO_WIDTH[_LATEST_VERSION]
+_WIDE_EASTASIAN_TABLE = WIDE_EASTASIAN[_LATEST_VERSION]
+_AMBIGUOUS_TABLE = AMBIGUOUS_EASTASIAN[next(iter(AMBIGUOUS_EASTASIAN))]
+_CATEGORY_MC_TABLE = CATEGORY_MC[_LATEST_VERSION]
+_REGIONAL_INDICATOR_SET = frozenset(
+    range(GRAPHEME_REGIONAL_INDICATOR[0][0], GRAPHEME_REGIONAL_INDICATOR[0][1] + 1)
+)
+_EMOJI_ZWJ_SET = frozenset(
+    cp for lo, hi in EXTENDED_PICTOGRAPHIC for cp in range(lo, hi + 1)
+) | _REGIONAL_INDICATOR_SET
+_FITZPATRICK_RANGE = (0x1F3FB, 0x1F3FF)
+# Indic_Syllabic_Category=Virama codepoints, from IndicSyllabicCategory.txt.
+# These are structurally tied to their scripts and not expected to change.
+# https://www.unicode.org/Public/UCD/latest/ucd/IndicSyllabicCategory.txt
+_ISC_VIRAMA_SET = frozenset((
+    0x094D,   # DEVANAGARI SIGN VIRAMA
+    0x09CD,   # BENGALI SIGN VIRAMA
+    0x0A4D,   # GURMUKHI SIGN VIRAMA
+    0x0ACD,   # GUJARATI SIGN VIRAMA
+    0x0B4D,   # ORIYA SIGN VIRAMA
+    0x0BCD,   # TAMIL SIGN VIRAMA
+    0x0C4D,   # TELUGU SIGN VIRAMA
+    0x0CCD,   # KANNADA SIGN VIRAMA
+    0x0D4D,   # MALAYALAM SIGN VIRAMA
+    0x0DCA,   # SINHALA SIGN AL-LAKUNA
+    0x1B44,   # BALINESE ADEG ADEG
+    0xA806,   # SYLOTI NAGRI SIGN HASANTA
+    0xA8C4,   # SAURASHTRA SIGN VIRAMA
+    0xA9C0,   # JAVANESE PANGKON
+    0x11046,  # BRAHMI VIRAMA
+    0x110B9,  # KAITHI SIGN VIRAMA
+    0x111C0,  # SHARADA SIGN VIRAMA
+    0x11235,  # KHOJKI SIGN VIRAMA
+    0x1134D,  # GRANTHA SIGN VIRAMA
+    0x11442,  # NEWA SIGN VIRAMA
+    0x114C2,  # TIRHUTA SIGN VIRAMA
+    0x115BF,  # SIDDHAM SIGN VIRAMA
+    0x1163F,  # MODI SIGN VIRAMA
+    0x116B6,  # TAKRI SIGN VIRAMA
+    0x11839,  # DOGRA SIGN VIRAMA
+    0x119E0,  # NANDINAGARI SIGN VIRAMA
+    0x11C3F,  # BHAIKSUKI SIGN VIRAMA
+))
+_ISC_CONSONANT_TABLE = ISC_CONSONANT
+
+# In 'parse' mode, strings longer than this are checked for cursor-movement
+# controls (BS, TAB, CR, cursor sequences); when absent, mode downgrades to
+# 'ignore' to skip character-by-character parsing. The detection scan cost is
+# negligible for long strings but wasted on short ones like labels or headings.
+_WIDTH_FAST_PATH_MIN_LEN = 20
+
+# Translation table to strip C0/C1 control characters for fast 'ignore' mode.
+_CONTROL_CHAR_TABLE = str.maketrans('', '', (
+    ''.join(chr(c) for c in range(0x00, 0x20)) +   # C0: NUL through US (including tab)
+    '\x7f' +                                       # DEL
+    ''.join(chr(c) for c in range(0x80, 0xa0))     # C1: U+0080-U+009F
+))
+
+# Unlike wcwidth.__all__, wcwidth.wcwidth.__all__ is NOT for the purpose of defining a public API,
+# or what we prefer to be imported with statement, "from wcwidth.wcwidth import *".  Explicitly
+# re-export imports here for no other reason than to satisfy the type checkers (mypy). Yak shavings.
+__all__ = (
+    'ZERO_WIDTH',
+    'WIDE_EASTASIAN',
+    'AMBIGUOUS_EASTASIAN',
+    'VS16_NARROW_TO_WIDE',
+    'list_versions',
+    'wcwidth',
+    'wcswidth',
+    'width',
+    'iter_sequences',
+    'ljust',
+    'rjust',
+    'center',
+    'clip',
+    'strip_sequences',
+    '_wcmatch_version',
+    '_wcversion_value',
+)
+
+
+# maxsize=1024: western scripts need ~64 unique codepoints per session, but
+# CJK sessions may use ~2000 of ~3500 common hanzi/kanji. 1024 accommodates
+# heavy CJK use. Performance floor at 32; bisearch is ~100ns per miss.
+
+@lru_cache(maxsize=1024)
+def wcwidth(wc: str, unicode_version: str = 'auto', ambiguous_width: int = 1) -> int:  # pylint: disable=unused-argument
+    r"""
+    Given one Unicode codepoint, return its printable length on a terminal.
+
+    :param wc: A single Unicode character.
+    :param unicode_version: Ignored. Retained for backwards compatibility.
+
+        .. deprecated:: 0.3.0
+           Only the latest Unicode version is now shipped.
+
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts
+        where ambiguous characters display as double-width. See
+        :ref:`ambiguous_width` for details.
+    :returns: The width, in cells, necessary to display the character of
+        Unicode string character, ``wc``.  Returns 0 if the ``wc`` argument has
+        no printable effect on a terminal (such as NUL '\0'), -1 if ``wc`` is
+        not printable, or has an indeterminate effect on the terminal, such as
+        a control character.  Otherwise, the number of column positions the
+        character occupies on a graphic terminal (1 or 2) is returned.
+
+    See :ref:`Specification` for details of cell measurement.
+    """
+    ucs = ord(wc) if wc else 0
+
+    # small optimization: early return of 1 for printable ASCII, this provides
+    # approximately 40% performance improvement for mostly-ascii documents, with
+    # less than 1% impact to others.
+    if 32 <= ucs < 0x7f:
+        return 1
+
+    # C0/C1 control characters are -1 for compatibility with POSIX-like calls
+    if ucs and ucs < 32 or 0x07F <= ucs < 0x0A0:
+        return -1
+
+    # Zero width
+    if _bisearch(ucs, _ZERO_WIDTH_TABLE):
+        return 0
+
+    # Wide (F/W categories)
+    if _bisearch(ucs, _WIDE_EASTASIAN_TABLE):
+        return 2
+
+    # Ambiguous width (A category) - only when ambiguous_width=2
+    if ambiguous_width == 2 and _bisearch(ucs, _AMBIGUOUS_TABLE):
+        return 2
+
+    return 1
+
+
+def wcswidth(
+    pwcs: str,
+    n: int | None = None,
+    unicode_version: str = 'auto',
+    ambiguous_width: int = 1,
+) -> int:
+    """
+    Given a unicode string, return its printable length on a terminal.
+
+    :param pwcs: Measure width of given unicode string.
+    :param n: When ``n`` is None (default), return the length of the entire
+        string, otherwise only the first ``n`` characters are measured.
+
+        Better to use string slicing capability, ``wcswidth(pwcs[:n])``, instead,
+        for performance.  This argument is a holdover from the POSIX function for
+        matching signatures. Be careful that ``n`` is at grapheme boundaries.
+
+    :param unicode_version: Ignored. Retained for backwards compatibility.
+
+        .. deprecated:: 0.3.0
+           Only the latest Unicode version is now shipped.
+
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :returns: The width, in cells, needed to display the first ``n`` characters
+        of the unicode string ``pwcs``.  Returns ``-1`` for C0 and C1 control
+        characters!
+
+    See :ref:`Specification` for details of cell measurement.
+    """
+    # pylint: disable=unused-argument,too-many-locals,too-many-statements
+    # pylint: disable=too-complex,too-many-branches
+    # This function intentionally kept long without delegating functions to reduce function calls in
+    # "hot path", the overhead per-character adds up.
+
+    # Fast path: pure ASCII printable strings are always width == length
+    if n is None and pwcs.isascii() and pwcs.isprintable():
+        return len(pwcs)
+
+    # Select wcwidth call pattern for best lru_cache performance:
+    # - ambiguous_width=1 (default): single-arg calls share cache with direct wcwidth() calls
+    # - ambiguous_width=2: full positional args needed (results differ, separate cache is correct)
+    _wcwidth = wcwidth if ambiguous_width == 1 else lambda c: wcwidth(c, 'auto', ambiguous_width)
+
+    end = len(pwcs) if n is None else n
+    total_width = 0
+    idx = 0
+    last_measured_idx = -2  # Track index of last measured char for VS16
+    last_measured_ucs = -1  # Codepoint of last measured char (for deferred emoji check)
+    last_was_virama = False  # Virama conjunct formation state
+    conjunct_pending = False  # Deferred +1 for bare conjuncts (no trailing Mc)
+    while idx < end:
+        char = pwcs[idx]
+        ucs = ord(char)
+        if ucs == 0x200D:
+            if last_was_virama:
+                # ZWJ after virama requests explicit half-form rendering but
+                # does not change cell count — consume ZWJ only, let the next
+                # consonant be handled by the virama conjunct rule.
+                idx += 1
+            elif idx + 1 < end:
+                # Emoji ZWJ: skip next character unconditionally.
+                idx += 2
+                last_was_virama = False
+            else:
+                idx += 1
+                last_was_virama = False
+            continue
+        if ucs == 0xFE0F and last_measured_idx >= 0:
+            # VS16 following a measured character: add 1 if that character is
+            # known to be converted from narrow to wide by VS16.
+            total_width += _bisearch(ord(pwcs[last_measured_idx]),
+                                     VS16_NARROW_TO_WIDE["9.0.0"])
+            last_measured_idx = -2  # Prevent double application
+            # VS16 preserves emoji context: last_measured_ucs stays as the base
+            idx += 1
+            continue
+        # Regional Indicator & Fitzpatrick: both above BMP (U+1F1E6+)
+        if ucs > 0xFFFF:
+            if ucs in _REGIONAL_INDICATOR_SET:
+                # Lazy RI pairing: count preceding consecutive RIs only when the last one is
+                # received, because RI's are received so rarely its better than per-loop tracking of
+                # 'last char was an RI'.
+                ri_before = 0
+                j = idx - 1
+                while j >= 0 and ord(pwcs[j]) in _REGIONAL_INDICATOR_SET:
+                    ri_before += 1
+                    j -= 1
+                if ri_before % 2 == 1:
+                    # Second RI in pair: contributes 0 (pair = one 2-cell flag) using an even-or-odd
+                    # check to determine, 'CAUS' would be two flags, but 'CAU' would be 1 flag
+                    # and wide 'U'.
+                    idx += 1
+                    last_measured_ucs = ucs
+                    continue
+                # First or unpaired RI: measured normally (width 2 from table)
+            # Fitzpatrick modifier: zero-width when following emoji base
+            elif (_FITZPATRICK_RANGE[0] <= ucs <= _FITZPATRICK_RANGE[1]
+                  and last_measured_ucs in _EMOJI_ZWJ_SET):
+                idx += 1
+                continue
+        # Virama conjunct formation: consonant following virama contributes 0 width.
+        # See https://www.unicode.org/reports/tr44/#Indic_Syllabic_Category
+        if last_was_virama and _bisearch(ucs, _ISC_CONSONANT_TABLE):
+            last_measured_idx = idx
+            last_measured_ucs = ucs
+            last_was_virama = False
+            conjunct_pending = True
+            idx += 1
+            continue
+        wcw = _wcwidth(char)
+        if wcw < 0:
+            # early return -1 on C0 and C1 control characters
+            return wcw
+        if wcw > 0:
+            if conjunct_pending:
+                total_width += 1
+                conjunct_pending = False
+            last_measured_idx = idx
+            last_measured_ucs = ucs
+            last_was_virama = False
+        elif last_measured_idx >= 0 and _bisearch(ucs, _CATEGORY_MC_TABLE):
+            # Spacing Combining Mark (Mc) following a base character adds 1
+            wcw = 1
+            last_measured_idx = -2
+            last_was_virama = False
+            conjunct_pending = False
+        else:
+            last_was_virama = ucs in _ISC_VIRAMA_SET
+        total_width += wcw
+        idx += 1
+    if conjunct_pending:
+        total_width += 1
+    return total_width
+
+
+# NOTE: _wcversion_value and _wcmatch_version are no longer used internally
+# by wcwidth since version 0.5.0 (only the latest Unicode version is shipped).
+#
+# They are retained for API compatibility with external tools like ucs-detect
+# that may use these private functions.
+
+
+@lru_cache(maxsize=128)
+def _wcversion_value(ver_string: str) -> tuple[int, ...]:  # pragma: no cover
+    """
+    Integer-mapped value of given dotted version string.
+
+    .. deprecated:: 0.3.0
+
+        This function is no longer used internally by wcwidth but is retained
+        for API compatibility with external tools.
+
+    :param ver_string: Unicode version string, of form ``n.n.n``.
+    :returns: tuple of digit tuples, ``tuple(int, [...])``.
+    """
+    retval = tuple(map(int, (ver_string.split('.'))))
+    return retval
+
+
+@lru_cache(maxsize=8)
+def _wcmatch_version(given_version: str) -> str:  # pylint: disable=unused-argument
+    """
+    Return the supported Unicode version level.
+
+    .. deprecated:: 0.3.0
+        This function now always returns the latest version.
+
+        This function is no longer used internally by wcwidth but is retained
+        for API compatibility with external tools.
+
+    :param given_version: Ignored. Any value is accepted for compatibility.
+    :returns: The latest unicode version string.
+    """
+    return _LATEST_VERSION
+
+
+def iter_sequences(text: str) -> Iterator[tuple[str, bool]]:
+    r"""
+    Iterate through text, yielding segments with sequence identification.
+
+    This generator yields tuples of ``(segment, is_sequence)`` for each part
+    of the input text, where ``is_sequence`` is ``True`` if the segment is
+    a recognized terminal escape sequence.
+
+    :param text: String to iterate through.
+    :returns: Iterator of (segment, is_sequence) tuples.
+
+    .. versionadded:: 0.3.0
+
+    Example::
+
+        >>> list(iter_sequences('hello'))
+        [('hello', False)]
+        >>> list(iter_sequences('\x1b[31mred'))
+        [('\x1b[31m', True), ('red', False)]
+        >>> list(iter_sequences('\x1b[1m\x1b[31m'))
+        [('\x1b[1m', True), ('\x1b[31m', True)]
+    """
+    idx = 0
+    text_len = len(text)
+    segment_start = 0
+
+    while idx < text_len:
+        char = text[idx]
+
+        if char == '\x1b':
+            # Yield any accumulated non-sequence text
+            if idx > segment_start:
+                yield (text[segment_start:idx], False)
+
+            # Try to match an escape sequence
+            match = ZERO_WIDTH_PATTERN.match(text, idx)
+            if match:
+                yield (match.group(), True)
+                idx = match.end()
+            else:
+                # Lone ESC or unrecognized - yield as sequence anyway
+                yield (char, True)
+                idx += 1
+            segment_start = idx
+        else:
+            idx += 1
+
+    # Yield any remaining text
+    if segment_start < text_len:
+        yield (text[segment_start:], False)
+
+
+def _width_ignored_codes(text: str, ambiguous_width: int = 1) -> int:
+    """
+    Fast path for width() with control_codes='ignore'.
+
+    Strips escape sequences and control characters, then measures remaining text.
+    """
+    return wcswidth(
+        strip_sequences(text).translate(_CONTROL_CHAR_TABLE),
+        ambiguous_width=ambiguous_width
+    )
+
+
+def width(
+    text: str,
+    *,
+    control_codes: Literal['parse', 'strict', 'ignore'] = 'parse',
+    tabsize: int = 8,
+    ambiguous_width: int = 1,
+) -> int:
+    r"""
+    Return printable width of text containing many kinds of control codes and sequences.
+
+    Unlike :func:`wcswidth`, this function handles most control characters and many popular terminal
+    output sequences.  Never returns -1.
+
+    :param text: String to measure.
+    :param control_codes: How to handle control characters and sequences:
+
+        - ``'parse'`` (default): Track horizontal cursor movement from BS ``\b``, CR ``\r``, TAB
+          ``\t``, and cursor left and right movement sequences.  Vertical movement (LF, VT, FF) and
+          indeterminate sequences are zero-width. Never raises.
+        - ``'strict'``: Like parse, but raises :exc:`ValueError` on control characters with
+          indeterminate results of the screen or cursor, like clear or vertical movement. Generally,
+          these should be handled with a virtual terminal emulator (like 'pyte').
+        - ``'ignore'``: All C0 and C1 control characters and escape sequences are measured as
+          width 0. This is the fastest measurement for text already filtered or known not to contain
+          any kinds of control codes or sequences. TAB ``\t`` is zero-width; for tab expansion,
+          pre-process: ``text.replace('\t', ' ' * 8)``.
+
+    :param tabsize: Tab stop width for ``'parse'`` and ``'strict'`` modes. Default is 8.
+        Must be positive. Has no effect when ``control_codes='ignore'``.
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :returns: Maximum cursor position reached, "extent", accounting for cursor movement sequences
+        present in ``text`` according to given parameters.  This represents the rightmost column the
+        cursor reaches.  Always a non-negative integer.
+
+    :raises ValueError: If ``control_codes='strict'`` and control characters with indeterminate
+        effects, such as vertical movement or clear sequences are encountered, or on unexpected
+        C0 or C1 control code. Also raised when ``control_codes`` is not one of the valid values.
+
+    .. versionadded:: 0.3.0
+
+    Examples::
+
+        >>> width('hello')
+        5
+        >>> width('コンニチハ')
+        10
+        >>> width('\x1b[31mred\x1b[0m')
+        3
+        >>> width('\x1b[31mred\x1b[0m', control_codes='ignore')  # same result (ignored)
+        3
+        >>> width('123\b4')     # backspace overwrites previous cell (outputs '124')
+        3
+        >>> width('abc\t')      # tab caused cursor to move to column 8
+        8
+        >>> width('1\x1b[10C')  # '1' + cursor right 10, cursor ends on column 11
+        11
+        >>> width('1\x1b[10C', control_codes='ignore')   # faster but wrong in this case
+        1
+    """
+    # pylint: disable=too-complex,too-many-branches,too-many-statements,too-many-locals
+    # This could be broken into sub-functions (#1, #3, and 6 especially), but for reduced overhead
+    # considering this function is a likely "hot path", they are inlined, breaking many of our
+    # complexity rules.
+
+    # Fast path for ASCII printable (no tabs, escapes, or control chars)
+    if text.isascii() and text.isprintable():
+        return len(text)
+
+    # Fast parse: if no horizontal cursor movements are possible, switch to 'ignore' mode.
+    # Only check for longer strings - the detection overhead hurts short string performance.
+    if control_codes == 'parse' and len(text) > _WIDTH_FAST_PATH_MIN_LEN:
+        # Check for cursor-affecting control characters
+        if '\b' not in text and '\t' not in text and '\r' not in text:
+            # Check for escape sequences - if none, or only non-cursor-movement sequences
+            if '\x1b' not in text or (
+                not CURSOR_RIGHT_SEQUENCE.search(text) and
+                not CURSOR_LEFT_SEQUENCE.search(text)
+            ):
+                control_codes = 'ignore'
+
+    # Fast path for ignore mode -- this is useful if you know the text is already "clean"
+    if control_codes == 'ignore':
+        return _width_ignored_codes(text, ambiguous_width)
+
+    strict = control_codes == 'strict'
+    # Track absolute positions: tab stops need modulo on absolute column, CR resets to 0.
+    # Initialize max_extent to 0 so backward movement (CR, BS) won't yield negative width.
+    current_col = 0
+    max_extent = 0
+    idx = 0
+    last_measured_idx = -2  # Track index of last measured char for VS16; -2 can never match idx-1
+    last_measured_ucs = -1  # Codepoint of last measured char (for deferred emoji check)
+    last_was_virama = False  # Virama conjunct formation state
+    conjunct_pending = False  # Deferred +1 for bare conjuncts (no trailing Mc)
+    text_len = len(text)
+
+    # Select wcwidth call pattern for best lru_cache performance:
+    # - ambiguous_width=1 (default): single-arg calls share cache with direct wcwidth() calls
+    # - ambiguous_width=2: full positional args needed (results differ, separate cache is correct)
+    _wcwidth = wcwidth if ambiguous_width == 1 else lambda c: wcwidth(c, 'auto', ambiguous_width)
+
+    while idx < text_len:
+        char = text[idx]
+
+        # 1. Handle ESC sequences
+        if char == '\x1b':
+            match = ZERO_WIDTH_PATTERN.match(text, idx)
+            if match:
+                seq = match.group()
+                if strict and INDETERMINATE_EFFECT_SEQUENCE.match(seq):
+                    raise ValueError(f"Indeterminate cursor sequence at position {idx}")
+                # Apply cursor movement
+                right = CURSOR_RIGHT_SEQUENCE.match(seq)
+                if right:
+                    current_col += int(right.group(1) or 1)
+                else:
+                    left = CURSOR_LEFT_SEQUENCE.match(seq)
+                    if left:
+                        current_col = max(0, current_col - int(left.group(1) or 1))
+                idx = match.end()
+            else:
+                idx += 1
+            max_extent = max(max_extent, current_col)
+            continue
+
+        # 2. Handle illegal and vertical control characters (zero width, error in strict)
+        if char in ILLEGAL_CTRL:
+            if strict:
+                raise ValueError(f"Illegal control character {ord(char):#x} at position {idx}")
+            idx += 1
+            continue
+
+        if char in VERTICAL_CTRL:
+            if strict:
+                raise ValueError(f"Vertical movement character {ord(char):#x} at position {idx}")
+            idx += 1
+            continue
+
+        # 3. Handle horizontal movement characters
+        if char in HORIZONTAL_CTRL:
+            if char == '\x09' and tabsize > 0:  # Tab
+                current_col += tabsize - (current_col % tabsize)
+            elif char == '\x08':  # Backspace
+                if current_col > 0:
+                    current_col -= 1
+            elif char == '\x0d':  # Carriage return
+                current_col = 0
+            max_extent = max(max_extent, current_col)
+            idx += 1
+            continue
+
+        # 4. Handle ZWJ
+        if char == '\u200D':
+            if last_was_virama:
+                # ZWJ after virama requests explicit half-form rendering but
+                # does not change cell count — consume ZWJ only, let the next
+                # consonant be handled by the virama conjunct rule.
+                idx += 1
+            elif idx + 1 < text_len:
+                # Emoji ZWJ: skip next character unconditionally.
+                idx += 2
+                last_was_virama = False
+            else:
+                idx += 1
+                last_was_virama = False
+            continue
+
+        # 5. Handle other zero-width characters (control chars)
+        if char in ZERO_WIDTH_CTRL:
+            idx += 1
+            continue
+
+        ucs = ord(char)
+
+        # 6. Handle VS16: converts preceding narrow character to wide
+        if ucs == 0xFE0F:
+            if last_measured_idx == idx - 1:
+                if _bisearch(ord(text[last_measured_idx]), VS16_NARROW_TO_WIDE["9.0.0"]):
+                    current_col += 1
+                    max_extent = max(max_extent, current_col)
+            # VS16 preserves emoji context: last_measured_ucs stays as the base
+            idx += 1
+            continue
+
+        # 6b. Regional Indicator & Fitzpatrick: both above BMP (U+1F1E6+)
+        if ucs > 0xFFFF:
+            if ucs in _REGIONAL_INDICATOR_SET:
+                # Lazy RI pairing: count preceding consecutive RIs
+                ri_before = 0
+                j = idx - 1
+                while j >= 0 and ord(text[j]) in _REGIONAL_INDICATOR_SET:
+                    ri_before += 1
+                    j -= 1
+                if ri_before % 2 == 1:
+                    last_measured_ucs = ucs
+                    idx += 1
+                    continue
+            # 6c. Fitzpatrick modifier: zero-width when following emoji base
+            elif (_FITZPATRICK_RANGE[0] <= ucs <= _FITZPATRICK_RANGE[1]
+                  and last_measured_ucs in _EMOJI_ZWJ_SET):
+                idx += 1
+                continue
+
+        # 7. Virama conjunct formation: consonant following virama contributes 0 width.
+        # See https://www.unicode.org/reports/tr44/#Indic_Syllabic_Category
+        if last_was_virama and _bisearch(ucs, _ISC_CONSONANT_TABLE):
+            last_measured_idx = idx
+            last_measured_ucs = ucs
+            last_was_virama = False
+            conjunct_pending = True
+            idx += 1
+            continue
+
+        # 8. Normal characters: measure with wcwidth
+        w = _wcwidth(char)
+        if w > 0:
+            if conjunct_pending:
+                current_col += 1
+                conjunct_pending = False
+            current_col += w
+            max_extent = max(max_extent, current_col)
+            last_measured_idx = idx
+            last_measured_ucs = ucs
+            last_was_virama = False
+        elif last_measured_idx >= 0 and _bisearch(ucs, _CATEGORY_MC_TABLE):
+            # Spacing Combining Mark (Mc) following a base character adds 1
+            current_col += 1
+            max_extent = max(max_extent, current_col)
+            last_measured_idx = -2
+            last_was_virama = False
+            conjunct_pending = False
+        else:
+            last_was_virama = ucs in _ISC_VIRAMA_SET
+        idx += 1
+
+    if conjunct_pending:
+        current_col += 1
+        max_extent = max(max_extent, current_col)
+    return max_extent
+
+
+def ljust(
+    text: str,
+    dest_width: int,
+    fillchar: str = ' ',
+    *,
+    control_codes: Literal['parse', 'strict', 'ignore'] = 'parse',
+    ambiguous_width: int = 1,
+) -> str:
+    r"""
+    Return text left-justified in a string of given display width.
+
+    :param text: String to justify, may contain terminal sequences.
+    :param dest_width: Total display width of result in terminal cells.
+    :param fillchar: Single character for padding (default space). Must have
+        display width of 1 (not wide, not zero-width, not combining). Unicode
+        characters like ``'·'`` are acceptable. The width is not validated.
+    :param control_codes: How to handle control sequences when measuring.
+        Passed to :func:`width` for measurement.
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :returns: Text padded on the right to reach ``dest_width``.
+
+    .. versionadded:: 0.3.0
+
+    Example::
+
+        >>> wcwidth.ljust('hi', 5)
+        'hi   '
+        >>> wcwidth.ljust('\x1b[31mhi\x1b[0m', 5)
+        '\x1b[31mhi\x1b[0m   '
+        >>> wcwidth.ljust('\U0001F468\u200D\U0001F469\u200D\U0001F467', 6)
+        '👨‍👩‍👧    '
+    """
+    if text.isascii() and text.isprintable():
+        text_width = len(text)
+    else:
+        text_width = width(text, control_codes=control_codes, ambiguous_width=ambiguous_width)
+    padding_cells = max(0, dest_width - text_width)
+    return text + fillchar * padding_cells
+
+
+def rjust(
+    text: str,
+    dest_width: int,
+    fillchar: str = ' ',
+    *,
+    control_codes: Literal['parse', 'strict', 'ignore'] = 'parse',
+    ambiguous_width: int = 1,
+) -> str:
+    r"""
+    Return text right-justified in a string of given display width.
+
+    :param text: String to justify, may contain terminal sequences.
+    :param dest_width: Total display width of result in terminal cells.
+    :param fillchar: Single character for padding (default space). Must have
+        display width of 1 (not wide, not zero-width, not combining). Unicode
+        characters like ``'·'`` are acceptable. The width is not validated.
+    :param control_codes: How to handle control sequences when measuring.
+        Passed to :func:`width` for measurement.
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :returns: Text padded on the left to reach ``dest_width``.
+
+    .. versionadded:: 0.3.0
+
+    Example::
+
+        >>> wcwidth.rjust('hi', 5)
+        '   hi'
+        >>> wcwidth.rjust('\x1b[31mhi\x1b[0m', 5)
+        '   \x1b[31mhi\x1b[0m'
+        >>> wcwidth.rjust('\U0001F468\u200D\U0001F469\u200D\U0001F467', 6)
+        '    👨‍👩‍👧'
+    """
+    if text.isascii() and text.isprintable():
+        text_width = len(text)
+    else:
+        text_width = width(text, control_codes=control_codes, ambiguous_width=ambiguous_width)
+    padding_cells = max(0, dest_width - text_width)
+    return fillchar * padding_cells + text
+
+
+def center(
+    text: str,
+    dest_width: int,
+    fillchar: str = ' ',
+    *,
+    control_codes: Literal['parse', 'strict', 'ignore'] = 'parse',
+    ambiguous_width: int = 1,
+) -> str:
+    r"""
+    Return text centered in a string of given display width.
+
+    :param text: String to center, may contain terminal sequences.
+    :param dest_width: Total display width of result in terminal cells.
+    :param fillchar: Single character for padding (default space). Must have
+        display width of 1 (not wide, not zero-width, not combining). Unicode
+        characters like ``'·'`` are acceptable. The width is not validated.
+    :param control_codes: How to handle control sequences when measuring.
+        Passed to :func:`width` for measurement.
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :returns: Text padded on both sides to reach ``dest_width``.
+
+    For odd-width padding, the extra cell goes on the right (matching
+    Python's :meth:`str.center` behavior).
+
+    .. versionadded:: 0.3.0
+
+    Example::
+
+        >>> wcwidth.center('hi', 6)
+        '  hi  '
+        >>> wcwidth.center('\x1b[31mhi\x1b[0m', 6)
+        '  \x1b[31mhi\x1b[0m  '
+        >>> wcwidth.center('\U0001F468\u200D\U0001F469\u200D\U0001F467', 6)
+        '  👨‍👩‍👧  '
+    """
+    if text.isascii() and text.isprintable():
+        text_width = len(text)
+    else:
+        text_width = width(text, control_codes=control_codes, ambiguous_width=ambiguous_width)
+    total_padding = max(0, dest_width - text_width)
+    # matching https://jazcap53.github.io/pythons-eccentric-strcenter.html
+    left_pad = total_padding // 2 + (total_padding & dest_width & 1)
+    right_pad = total_padding - left_pad
+    return fillchar * left_pad + text + fillchar * right_pad
+
+
+def strip_sequences(text: str) -> str:
+    r"""
+    Return text with all terminal escape sequences removed.
+
+    Unknown or incomplete ESC sequences are preserved.
+
+    :param text: String that may contain terminal escape sequences.
+    :returns: The input text with all escape sequences stripped.
+
+    .. versionadded:: 0.3.0
+
+    Example::
+
+        >>> strip_sequences('\x1b[31mred\x1b[0m')
+        'red'
+        >>> strip_sequences('hello')
+        'hello'
+        >>> strip_sequences('\x1b[1m\x1b[31mbold red\x1b[0m text')
+        'bold red text'
+    """
+    return ZERO_WIDTH_PATTERN.sub('', text)
+
+
+def clip(
+    text: str,
+    start: int,
+    end: int,
+    *,
+    fillchar: str = ' ',
+    tabsize: int = 8,
+    ambiguous_width: int = 1,
+    propagate_sgr: bool = True,
+) -> str:
+    r"""
+    Clip text to display columns ``(start, end)`` while preserving all terminal sequences.
+
+    This function extracts a substring based on visible column positions rather than
+    character indices. Terminal escape sequences are preserved in the output since
+    they have zero display width. If a wide character (width 2) would be split at
+    either boundary, it is replaced with ``fillchar``.
+
+    TAB characters (``\t``) are expanded to spaces up to the next tab stop,
+    controlled by the ``tabsize`` parameter.
+
+    Other cursor movement characters (backspace, carriage return) and cursor
+    movement sequences are passed through unchanged as zero-width.
+
+    :param text: String to clip, may contain terminal escape sequences.
+    :param start: Absolute starting column (inclusive, 0-indexed).
+    :param end: Absolute ending column (exclusive).
+    :param fillchar: Character to use when a wide character must be split at
+        a boundary (default space). Must have display width of 1.
+    :param tabsize: Tab stop width (default 8). Set to 0 to pass tabs through
+        as zero-width (preserved in output but don't advance column position).
+    :param ambiguous_width: Width to use for East Asian Ambiguous (A)
+        characters. Default is ``1`` (narrow). Set to ``2`` for CJK contexts.
+    :param propagate_sgr: If True (default), SGR (terminal styling) sequences
+        are propagated. The result begins with any active style at the start
+        position and ends with a reset sequence if styles are active.
+    :returns: Substring of ``text`` spanning display columns ``(start, end)``,
+        with all terminal sequences preserved and wide characters at boundaries
+        replaced with ``fillchar``.
+
+    SGR (terminal styling) sequences are propagated by default. The result
+    begins with any active style and ends with a reset::
+
+        >>> clip('\x1b[1;34mHello world\x1b[0m', 6, 11)
+        '\x1b[1;34mworld\x1b[0m'
+
+    Set ``propagate_sgr=False`` to disable this behavior.
+
+    .. versionadded:: 0.3.0
+
+    .. versionchanged:: 0.5.0
+       Added ``propagate_sgr`` parameter (default True).
+
+    Example::
+
+        >>> clip('hello world', 0, 5)
+        'hello'
+        >>> clip('中文字', 0, 3)  # Wide char split at column 3
+        '中 '
+        >>> clip('a\tb', 0, 10)  # Tab expanded to spaces
+        'a       b'
+    """
+    # pylint: disable=too-complex,too-many-locals,too-many-branches,too-many-statements,too-many-nested-blocks
+    # Again, for 'hot path', we avoid additional delegate functions and accept the cost
+    # of complexity for improved python performance.
+    start = max(start, 0)
+    if end <= start:
+        return ''
+
+    # Fast path: printable ASCII only (no tabs, escape sequences, or wide or zero-width chars)
+    if text.isascii() and text.isprintable():
+        return text[start:end]
+
+    # Fast path: no escape sequences means no SGR tracking needed
+    if propagate_sgr and '\x1b' not in text:
+        propagate_sgr = False
+
+    # SGR tracking state (only when propagate_sgr=True)
+    sgr_at_clip_start = None  # state when first visible char emitted (None = not yet)
+    if propagate_sgr:
+        sgr = _SGR_STATE_DEFAULT  # current SGR state, updated by all sequences
+
+    output: list[str] = []
+    col = 0
+    idx = 0
+
+    while idx < len(text):
+        char = text[idx]
+
+        # Early exit: past visible region, SGR captured, no escape ahead
+        if col >= end and sgr_at_clip_start is not None and char != '\x1b':
+            break
+
+        # Handle escape sequences
+        if char == '\x1b' and (match := ZERO_WIDTH_PATTERN.match(text, idx)):
+            seq = match.group()
+            if propagate_sgr and _SGR_PATTERN.match(seq):
+                # Update SGR state; will be applied as prefix when visible content starts
+                sgr = _sgr_state_update(sgr, seq)
+            else:
+                # Non-SGR sequences always preserved
+                output.append(seq)
+            idx = match.end()
+            continue
+
+        # Handle bare ESC (not a valid sequence)
+        if char == '\x1b':
+            output.append(char)
+            idx += 1
+            continue
+
+        # TAB expansion
+        if char == '\t':
+            if tabsize > 0:
+                next_tab = col + (tabsize - (col % tabsize))
+                while col < next_tab:
+                    if start <= col < end:
+                        output.append(' ')
+                        if propagate_sgr and sgr_at_clip_start is None:
+                            sgr_at_clip_start = sgr
+                    col += 1
+            else:
+                output.append(char)
+            idx += 1
+            continue
+
+        # Grapheme clustering for everything else
+        grapheme = next(iter_graphemes(text, start=idx))
+        w = width(grapheme, ambiguous_width=ambiguous_width)
+
+        if w == 0:
+            if start <= col < end:
+                output.append(grapheme)
+        elif col >= start and col + w <= end:
+            # Fully visible
+            output.append(grapheme)
+            if propagate_sgr and sgr_at_clip_start is None:
+                sgr_at_clip_start = sgr
+            col += w
+        elif col < end and col + w > start:
+            # Partially visible (wide char at boundary)
+            output.append(fillchar * (min(end, col + w) - max(start, col)))
+            if propagate_sgr and sgr_at_clip_start is None:
+                sgr_at_clip_start = sgr
+            col += w
+        else:
+            col += w
+
+        idx += len(grapheme)
+
+    result = ''.join(output)
+
+    # Apply SGR prefix/suffix
+    if sgr_at_clip_start is not None:
+        if prefix := _sgr_state_to_sequence(sgr_at_clip_start):
+            result = prefix + result
+        if _sgr_state_is_active(sgr_at_clip_start):
+            result += '\x1b[0m'
+
+    return result
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..cd6e3c8
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,23 @@
+import lib.questionary as qs
+import os
+
+
+def clear():
+    os.subprocess('cls' if os.name == 'nt' else 'clear')
+
+
+class setupMenu:
+    @staticmethod
+    def main():
+        
+        print("Welcome... Setup Global Config:")
+        sel = qs.select("Add Server", "Save Global Config")
+
+
+
+
+
+
+def initialSetup():
+    setupMenu.main() 
+
diff --git a/sftp_debug.log b/sftp_debug.log
new file mode 100644
index 0000000..d8f5e35
--- /dev/null
+++ b/sftp_debug.log
@@ -0,0 +1,8300 @@
+2026-02-12 01:49:31,089 - DEBUG - Using selector: EpollSelector
+2026-02-12 01:49:38,960 - INFO - Initiating SSH connection to 192.168.1.5:22
+2026-02-12 01:49:38,960 - INFO - Attempting login for user: kevin
+2026-02-12 01:49:38,963 - DEBUG - starting thread (client mode): 0xd3f3ea50
+2026-02-12 01:49:38,964 - DEBUG - Local version/idstring: SSH-2.0-paramiko_4.0.0
+2026-02-12 01:49:38,977 - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.11
+2026-02-12 01:49:38,977 - INFO - Connected (version 2.0, client OpenSSH_9.6p1)
+2026-02-12 01:49:38,979 - DEBUG - === Key exchange possibilities ===
+2026-02-12 01:49:38,979 - DEBUG - kex algos: sntrup761x25519-sha512@openssh.com, curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, ext-info-s, kex-strict-s-v00@openssh.com
+2026-02-12 01:49:38,979 - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519
+2026-02-12 01:49:38,979 - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 01:49:38,980 - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 01:49:38,980 - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 01:49:38,980 - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 01:49:38,980 - DEBUG - client compress: none, zlib@openssh.com
+2026-02-12 01:49:38,980 - DEBUG - server compress: none, zlib@openssh.com
+2026-02-12 01:49:38,980 - DEBUG - client lang: 
+2026-02-12 01:49:38,980 - DEBUG - server lang: 
+2026-02-12 01:49:38,980 - DEBUG - kex follows: False
+2026-02-12 01:49:38,980 - DEBUG - === Key exchange agreements ===
+2026-02-12 01:49:38,980 - DEBUG - Strict kex mode: True
+2026-02-12 01:49:38,980 - DEBUG - Kex: curve25519-sha256@libssh.org
+2026-02-12 01:49:38,981 - DEBUG - HostKey: ssh-ed25519
+2026-02-12 01:49:38,981 - DEBUG - Cipher: aes128-ctr
+2026-02-12 01:49:38,981 - DEBUG - MAC: hmac-sha2-256
+2026-02-12 01:49:38,981 - DEBUG - Compression: none
+2026-02-12 01:49:38,981 - DEBUG - === End of kex handshake ===
+2026-02-12 01:49:38,991 - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode
+2026-02-12 01:49:38,991 - DEBUG - kex engine KexCurve25519 specified hash_algo 
+2026-02-12 01:49:38,992 - DEBUG - Switch to new keys ...
+2026-02-12 01:49:38,993 - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode
+2026-02-12 01:49:38,993 - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256', 'publickey-hostbound@openssh.com': b'0', 'ping@openssh.com': b'0'}
+2026-02-12 01:49:38,993 - DEBUG - Adding ssh-ed25519 host key for 192.168.1.5: b'05b4c881009e843d53172fd7a680af53'
+2026-02-12 01:49:39,032 - DEBUG - userauth is OK
+2026-02-12 01:49:39,054 - INFO - Authentication (password) successful!
+2026-02-12 01:49:39,054 - INFO - SSH connection established. Opening SFTP session...
+2026-02-12 01:49:39,054 - DEBUG - [chan 0] Max packet in: 32768 bytes
+2026-02-12 01:49:39,499 - DEBUG - Received global request "hostkeys-00@openssh.com"
+2026-02-12 01:49:39,499 - DEBUG - Rejecting "hostkeys-00@openssh.com" global request from server.
+2026-02-12 01:49:39,539 - DEBUG - [chan 0] Max packet out: 32768 bytes
+2026-02-12 01:49:39,539 - DEBUG - Secsh channel 0 opened.
+2026-02-12 01:49:39,541 - DEBUG - [chan 0] Sesch channel 0 request ok
+2026-02-12 01:49:39,546 - INFO - [chan 0] Opened sftp connection (server version 3)
+2026-02-12 01:49:39,546 - INFO - SFTP session opened successfully.
+2026-02-12 01:49:39,546 - DEBUG - [chan 0] mkdir(b'/home/kevin/test', 511)
+2026-02-12 01:49:39,547 - INFO - Closing SFTP and SSH connections.
+2026-02-12 01:49:39,547 - INFO - [chan 0] sftp session closed.
+2026-02-12 01:49:39,547 - DEBUG - [chan 0] EOF sent (0)
+2026-02-12 01:52:54,416 - DEBUG - Using selector: EpollSelector
+2026-02-12 01:53:03,985 - DEBUG - Using selector: EpollSelector
+2026-02-12 01:53:08,286 - INFO - Initiating SSH connection to 192.168.1.5:22
+2026-02-12 01:53:08,286 - INFO - Attempting login for user: kevin
+2026-02-12 01:53:08,288 - DEBUG - starting thread (client mode): 0x66c1a900
+2026-02-12 01:53:08,289 - DEBUG - Local version/idstring: SSH-2.0-paramiko_4.0.0
+2026-02-12 01:53:08,303 - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.11
+2026-02-12 01:53:08,303 - INFO - Connected (version 2.0, client OpenSSH_9.6p1)
+2026-02-12 01:53:08,305 - DEBUG - === Key exchange possibilities ===
+2026-02-12 01:53:08,305 - DEBUG - kex algos: sntrup761x25519-sha512@openssh.com, curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, ext-info-s, kex-strict-s-v00@openssh.com
+2026-02-12 01:53:08,305 - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519
+2026-02-12 01:53:08,305 - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 01:53:08,305 - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 01:53:08,305 - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 01:53:08,305 - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 01:53:08,305 - DEBUG - client compress: none, zlib@openssh.com
+2026-02-12 01:53:08,305 - DEBUG - server compress: none, zlib@openssh.com
+2026-02-12 01:53:08,306 - DEBUG - client lang: 
+2026-02-12 01:53:08,306 - DEBUG - server lang: 
+2026-02-12 01:53:08,306 - DEBUG - kex follows: False
+2026-02-12 01:53:08,306 - DEBUG - === Key exchange agreements ===
+2026-02-12 01:53:08,306 - DEBUG - Strict kex mode: True
+2026-02-12 01:53:08,306 - DEBUG - Kex: curve25519-sha256@libssh.org
+2026-02-12 01:53:08,306 - DEBUG - HostKey: ssh-ed25519
+2026-02-12 01:53:08,306 - DEBUG - Cipher: aes128-ctr
+2026-02-12 01:53:08,306 - DEBUG - MAC: hmac-sha2-256
+2026-02-12 01:53:08,306 - DEBUG - Compression: none
+2026-02-12 01:53:08,307 - DEBUG - === End of kex handshake ===
+2026-02-12 01:53:08,316 - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode
+2026-02-12 01:53:08,316 - DEBUG - kex engine KexCurve25519 specified hash_algo 
+2026-02-12 01:53:08,317 - DEBUG - Switch to new keys ...
+2026-02-12 01:53:08,317 - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode
+2026-02-12 01:53:08,317 - DEBUG - Adding ssh-ed25519 host key for 192.168.1.5: b'05b4c881009e843d53172fd7a680af53'
+2026-02-12 01:53:08,317 - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256', 'publickey-hostbound@openssh.com': b'0', 'ping@openssh.com': b'0'}
+2026-02-12 01:53:08,358 - DEBUG - userauth is OK
+2026-02-12 01:53:08,379 - INFO - Authentication (password) successful!
+2026-02-12 01:53:08,380 - INFO - SSH connection established. Opening SFTP session...
+2026-02-12 01:53:08,381 - DEBUG - [chan 0] Max packet in: 32768 bytes
+2026-02-12 01:53:08,821 - DEBUG - Received global request "hostkeys-00@openssh.com"
+2026-02-12 01:53:08,821 - DEBUG - Rejecting "hostkeys-00@openssh.com" global request from server.
+2026-02-12 01:53:08,862 - DEBUG - [chan 0] Max packet out: 32768 bytes
+2026-02-12 01:53:08,862 - DEBUG - Secsh channel 0 opened.
+2026-02-12 01:53:08,864 - DEBUG - [chan 0] Sesch channel 0 request ok
+2026-02-12 01:53:08,868 - INFO - [chan 0] Opened sftp connection (server version 3)
+2026-02-12 01:53:08,868 - INFO - SFTP session opened successfully.
+2026-02-12 01:53:08,868 - INFO - Scanning directory: core
+2026-02-12 01:53:08,869 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/core', 511)
+2026-02-12 01:53:08,870 - INFO - Created remote folder: /home/kevin/test/core
+2026-02-12 01:53:08,870 - INFO - Closing SFTP and SSH connections.
+2026-02-12 01:53:08,870 - INFO - [chan 0] sftp session closed.
+2026-02-12 01:53:08,870 - DEBUG - [chan 0] EOF sent (0)
+2026-02-12 01:53:23,851 - DEBUG - Using selector: EpollSelector
+2026-02-12 01:53:31,597 - INFO - Initiating SSH connection to 192.168.1.5:22
+2026-02-12 01:53:31,597 - INFO - Attempting login for user: kevin
+2026-02-12 01:53:31,602 - DEBUG - starting thread (client mode): 0x4be1acf0
+2026-02-12 01:53:31,602 - DEBUG - Local version/idstring: SSH-2.0-paramiko_4.0.0
+2026-02-12 01:53:31,616 - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.11
+2026-02-12 01:53:31,616 - INFO - Connected (version 2.0, client OpenSSH_9.6p1)
+2026-02-12 01:53:31,618 - DEBUG - === Key exchange possibilities ===
+2026-02-12 01:53:31,618 - DEBUG - kex algos: sntrup761x25519-sha512@openssh.com, curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, ext-info-s, kex-strict-s-v00@openssh.com
+2026-02-12 01:53:31,618 - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519
+2026-02-12 01:53:31,618 - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 01:53:31,618 - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 01:53:31,619 - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 01:53:31,619 - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 01:53:31,619 - DEBUG - client compress: none, zlib@openssh.com
+2026-02-12 01:53:31,619 - DEBUG - server compress: none, zlib@openssh.com
+2026-02-12 01:53:31,619 - DEBUG - client lang: 
+2026-02-12 01:53:31,619 - DEBUG - server lang: 
+2026-02-12 01:53:31,619 - DEBUG - kex follows: False
+2026-02-12 01:53:31,619 - DEBUG - === Key exchange agreements ===
+2026-02-12 01:53:31,620 - DEBUG - Strict kex mode: True
+2026-02-12 01:53:31,620 - DEBUG - Kex: curve25519-sha256@libssh.org
+2026-02-12 01:53:31,620 - DEBUG - HostKey: ssh-ed25519
+2026-02-12 01:53:31,620 - DEBUG - Cipher: aes128-ctr
+2026-02-12 01:53:31,620 - DEBUG - MAC: hmac-sha2-256
+2026-02-12 01:53:31,620 - DEBUG - Compression: none
+2026-02-12 01:53:31,620 - DEBUG - === End of kex handshake ===
+2026-02-12 01:53:31,630 - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode
+2026-02-12 01:53:31,630 - DEBUG - kex engine KexCurve25519 specified hash_algo 
+2026-02-12 01:53:31,630 - DEBUG - Switch to new keys ...
+2026-02-12 01:53:31,630 - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode
+2026-02-12 01:53:31,631 - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256', 'publickey-hostbound@openssh.com': b'0', 'ping@openssh.com': b'0'}
+2026-02-12 01:53:31,631 - DEBUG - Adding ssh-ed25519 host key for 192.168.1.5: b'05b4c881009e843d53172fd7a680af53'
+2026-02-12 01:53:31,671 - DEBUG - userauth is OK
+2026-02-12 01:53:31,692 - INFO - Authentication (password) successful!
+2026-02-12 01:53:31,693 - INFO - SSH connection established. Opening SFTP session...
+2026-02-12 01:53:31,693 - DEBUG - [chan 0] Max packet in: 32768 bytes
+2026-02-12 01:53:31,761 - DEBUG - Received global request "hostkeys-00@openssh.com"
+2026-02-12 01:53:31,761 - DEBUG - Rejecting "hostkeys-00@openssh.com" global request from server.
+2026-02-12 01:53:31,801 - DEBUG - [chan 0] Max packet out: 32768 bytes
+2026-02-12 01:53:31,801 - DEBUG - Secsh channel 0 opened.
+2026-02-12 01:53:31,803 - DEBUG - [chan 0] Sesch channel 0 request ok
+2026-02-12 01:53:31,807 - INFO - [chan 0] Opened sftp connection (server version 3)
+2026-02-12 01:53:31,807 - INFO - SFTP session opened successfully.
+2026-02-12 01:53:31,807 - INFO - Scanning directory: lib
+2026-02-12 01:53:31,807 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib', 511)
+2026-02-12 01:53:31,808 - INFO - Created remote folder: /home/kevin/test/lib
+2026-02-12 01:53:31,809 - INFO - Starting upload: lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so -> /home/kevin/test/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so (344664 bytes)
+2026-02-12 01:53:31,814 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so', 'wb')
+2026-02-12 01:53:31,815 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so', 'wb') -> 00000000
+2026-02-12 01:53:31,837 - DEBUG - Progress: 262144/344664 bytes
+2026-02-12 01:53:31,841 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,864 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/_cffi_backend.cpython-314-x86_64-linux-gnu.so')
+2026-02-12 01:53:31,867 - INFO - Upload completed successfully: _cffi_backend.cpython-314-x86_64-linux-gnu.so
+2026-02-12 01:53:31,867 - INFO - Starting upload: lib/ftp.py -> /home/kevin/test/lib/ftp.py (7041 bytes)
+2026-02-12 01:53:31,871 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/ftp.py', 'wb')
+2026-02-12 01:53:31,872 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/ftp.py', 'wb') -> 00000000
+2026-02-12 01:53:31,872 - DEBUG - Progress: 7041/7041 bytes
+2026-02-12 01:53:31,872 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,874 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/ftp.py')
+2026-02-12 01:53:31,878 - INFO - Upload completed successfully: ftp.py
+2026-02-12 01:53:31,878 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/questionary-2.1.1.dist-info', 511)
+2026-02-12 01:53:31,879 - INFO - Created remote folder: /home/kevin/test/lib/questionary-2.1.1.dist-info
+2026-02-12 01:53:31,879 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/RECORD -> /home/kevin/test/lib/questionary-2.1.1.dist-info/RECORD (3363 bytes)
+2026-02-12 01:53:31,883 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/RECORD', 'wb')
+2026-02-12 01:53:31,884 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:31,884 - DEBUG - Progress: 3363/3363 bytes
+2026-02-12 01:53:31,884 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,885 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/RECORD')
+2026-02-12 01:53:31,889 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:31,889 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/REQUESTED -> /home/kevin/test/lib/questionary-2.1.1.dist-info/REQUESTED (0 bytes)
+2026-02-12 01:53:31,893 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/REQUESTED', 'wb')
+2026-02-12 01:53:31,894 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/REQUESTED', 'wb') -> 00000000
+2026-02-12 01:53:31,894 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,895 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/REQUESTED')
+2026-02-12 01:53:31,898 - INFO - Upload completed successfully: REQUESTED
+2026-02-12 01:53:31,899 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/INSTALLER -> /home/kevin/test/lib/questionary-2.1.1.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:31,903 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:31,904 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:31,904 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:31,904 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,905 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/INSTALLER')
+2026-02-12 01:53:31,908 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:31,908 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/WHEEL -> /home/kevin/test/lib/questionary-2.1.1.dist-info/WHEEL (88 bytes)
+2026-02-12 01:53:31,912 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:31,913 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:31,913 - DEBUG - Progress: 88/88 bytes
+2026-02-12 01:53:31,913 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,914 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/WHEEL')
+2026-02-12 01:53:31,918 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:31,918 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/NOTICE -> /home/kevin/test/lib/questionary-2.1.1.dist-info/NOTICE (2419 bytes)
+2026-02-12 01:53:31,922 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/NOTICE', 'wb')
+2026-02-12 01:53:31,923 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/NOTICE', 'wb') -> 00000000
+2026-02-12 01:53:31,923 - DEBUG - Progress: 2419/2419 bytes
+2026-02-12 01:53:31,924 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,925 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/NOTICE')
+2026-02-12 01:53:31,928 - INFO - Upload completed successfully: NOTICE
+2026-02-12 01:53:31,928 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/METADATA -> /home/kevin/test/lib/questionary-2.1.1.dist-info/METADATA (5402 bytes)
+2026-02-12 01:53:31,931 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/METADATA', 'wb')
+2026-02-12 01:53:31,932 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:31,932 - DEBUG - Progress: 5402/5402 bytes
+2026-02-12 01:53:31,932 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,934 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/METADATA')
+2026-02-12 01:53:31,935 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:31,935 - INFO - Starting upload: lib/questionary-2.1.1.dist-info/LICENSE -> /home/kevin/test/lib/questionary-2.1.1.dist-info/LICENSE (1070 bytes)
+2026-02-12 01:53:31,936 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/LICENSE', 'wb')
+2026-02-12 01:53:31,937 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:31,937 - DEBUG - Progress: 1070/1070 bytes
+2026-02-12 01:53:31,937 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,938 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary-2.1.1.dist-info/LICENSE')
+2026-02-12 01:53:31,940 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:31,940 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/questionary', 511)
+2026-02-12 01:53:31,940 - INFO - Created remote folder: /home/kevin/test/lib/questionary
+2026-02-12 01:53:31,940 - INFO - Starting upload: lib/questionary/version.py -> /home/kevin/test/lib/questionary/version.py (22 bytes)
+2026-02-12 01:53:31,942 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/version.py', 'wb')
+2026-02-12 01:53:31,942 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/version.py', 'wb') -> 00000000
+2026-02-12 01:53:31,943 - DEBUG - Progress: 22/22 bytes
+2026-02-12 01:53:31,943 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,943 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/version.py')
+2026-02-12 01:53:31,945 - INFO - Upload completed successfully: version.py
+2026-02-12 01:53:31,945 - INFO - Starting upload: lib/questionary/utils.py -> /home/kevin/test/lib/questionary/utils.py (2300 bytes)
+2026-02-12 01:53:31,946 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/utils.py', 'wb')
+2026-02-12 01:53:31,947 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:31,947 - DEBUG - Progress: 2300/2300 bytes
+2026-02-12 01:53:31,947 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,948 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/utils.py')
+2026-02-12 01:53:31,950 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:31,950 - INFO - Starting upload: lib/questionary/styles.py -> /home/kevin/test/lib/questionary/styles.py (595 bytes)
+2026-02-12 01:53:31,951 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/styles.py', 'wb')
+2026-02-12 01:53:31,952 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/styles.py', 'wb') -> 00000000
+2026-02-12 01:53:31,952 - DEBUG - Progress: 595/595 bytes
+2026-02-12 01:53:31,952 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,952 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/styles.py')
+2026-02-12 01:53:31,954 - INFO - Upload completed successfully: styles.py
+2026-02-12 01:53:31,954 - INFO - Starting upload: lib/questionary/question.py -> /home/kevin/test/lib/questionary/question.py (3998 bytes)
+2026-02-12 01:53:31,956 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/question.py', 'wb')
+2026-02-12 01:53:31,956 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/question.py', 'wb') -> 00000000
+2026-02-12 01:53:31,956 - DEBUG - Progress: 3998/3998 bytes
+2026-02-12 01:53:31,956 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,957 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/question.py')
+2026-02-12 01:53:31,959 - INFO - Upload completed successfully: question.py
+2026-02-12 01:53:31,959 - INFO - Starting upload: lib/questionary/py.typed -> /home/kevin/test/lib/questionary/py.typed (0 bytes)
+2026-02-12 01:53:31,960 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/py.typed', 'wb')
+2026-02-12 01:53:31,961 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/py.typed', 'wb') -> 00000000
+2026-02-12 01:53:31,961 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,961 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/py.typed')
+2026-02-12 01:53:31,963 - INFO - Upload completed successfully: py.typed
+2026-02-12 01:53:31,963 - INFO - Starting upload: lib/questionary/prompt.py -> /home/kevin/test/lib/questionary/prompt.py (8481 bytes)
+2026-02-12 01:53:31,964 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompt.py', 'wb')
+2026-02-12 01:53:31,965 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompt.py', 'wb') -> 00000000
+2026-02-12 01:53:31,965 - DEBUG - Progress: 8481/8481 bytes
+2026-02-12 01:53:31,965 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,966 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompt.py')
+2026-02-12 01:53:31,968 - INFO - Upload completed successfully: prompt.py
+2026-02-12 01:53:31,968 - INFO - Starting upload: lib/questionary/form.py -> /home/kevin/test/lib/questionary/form.py (3426 bytes)
+2026-02-12 01:53:31,970 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/form.py', 'wb')
+2026-02-12 01:53:31,970 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/form.py', 'wb') -> 00000000
+2026-02-12 01:53:31,971 - DEBUG - Progress: 3426/3426 bytes
+2026-02-12 01:53:31,971 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,972 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/form.py')
+2026-02-12 01:53:31,973 - INFO - Upload completed successfully: form.py
+2026-02-12 01:53:31,973 - INFO - Starting upload: lib/questionary/constants.py -> /home/kevin/test/lib/questionary/constants.py (1946 bytes)
+2026-02-12 01:53:31,975 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/constants.py', 'wb')
+2026-02-12 01:53:31,975 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/constants.py', 'wb') -> 00000000
+2026-02-12 01:53:31,975 - DEBUG - Progress: 1946/1946 bytes
+2026-02-12 01:53:31,975 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,977 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/constants.py')
+2026-02-12 01:53:31,978 - INFO - Upload completed successfully: constants.py
+2026-02-12 01:53:31,978 - INFO - Starting upload: lib/questionary/__init__.py -> /home/kevin/test/lib/questionary/__init__.py (1630 bytes)
+2026-02-12 01:53:31,979 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__init__.py', 'wb')
+2026-02-12 01:53:31,980 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:31,980 - DEBUG - Progress: 1630/1630 bytes
+2026-02-12 01:53:31,980 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,981 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__init__.py')
+2026-02-12 01:53:31,983 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:31,983 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/questionary/__pycache__', 511)
+2026-02-12 01:53:31,984 - INFO - Created remote folder: /home/kevin/test/lib/questionary/__pycache__
+2026-02-12 01:53:31,984 - INFO - Starting upload: lib/questionary/__pycache__/version.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/version.cpython-314.pyc (183 bytes)
+2026-02-12 01:53:31,985 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/version.cpython-314.pyc', 'wb')
+2026-02-12 01:53:31,986 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/version.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:31,986 - DEBUG - Progress: 183/183 bytes
+2026-02-12 01:53:31,986 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,986 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/version.cpython-314.pyc')
+2026-02-12 01:53:31,988 - INFO - Upload completed successfully: version.cpython-314.pyc
+2026-02-12 01:53:31,988 - INFO - Starting upload: lib/questionary/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/utils.cpython-314.pyc (4960 bytes)
+2026-02-12 01:53:31,989 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:31,990 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:31,990 - DEBUG - Progress: 4960/4960 bytes
+2026-02-12 01:53:31,990 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,991 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:31,992 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:31,993 - INFO - Starting upload: lib/questionary/__pycache__/styles.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/styles.cpython-314.pyc (1096 bytes)
+2026-02-12 01:53:31,994 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/styles.cpython-314.pyc', 'wb')
+2026-02-12 01:53:31,995 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/styles.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:31,995 - DEBUG - Progress: 1096/1096 bytes
+2026-02-12 01:53:31,995 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:31,996 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/styles.cpython-314.pyc')
+2026-02-12 01:53:31,997 - INFO - Upload completed successfully: styles.cpython-314.pyc
+2026-02-12 01:53:31,997 - INFO - Starting upload: lib/questionary/__pycache__/question.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/question.cpython-314.pyc (6662 bytes)
+2026-02-12 01:53:31,999 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/question.cpython-314.pyc', 'wb')
+2026-02-12 01:53:31,999 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/question.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:31,999 - DEBUG - Progress: 6662/6662 bytes
+2026-02-12 01:53:31,999 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,001 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/question.cpython-314.pyc')
+2026-02-12 01:53:32,003 - INFO - Upload completed successfully: question.cpython-314.pyc
+2026-02-12 01:53:32,003 - INFO - Starting upload: lib/questionary/__pycache__/prompt.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/prompt.cpython-314.pyc (9757 bytes)
+2026-02-12 01:53:32,004 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/prompt.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,005 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/prompt.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,005 - DEBUG - Progress: 9757/9757 bytes
+2026-02-12 01:53:32,005 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,006 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/prompt.cpython-314.pyc')
+2026-02-12 01:53:32,008 - INFO - Upload completed successfully: prompt.cpython-314.pyc
+2026-02-12 01:53:32,008 - INFO - Starting upload: lib/questionary/__pycache__/form.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/form.cpython-314.pyc (6213 bytes)
+2026-02-12 01:53:32,009 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/form.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,010 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/form.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,010 - DEBUG - Progress: 6213/6213 bytes
+2026-02-12 01:53:32,010 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,011 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/form.cpython-314.pyc')
+2026-02-12 01:53:32,013 - INFO - Upload completed successfully: form.cpython-314.pyc
+2026-02-12 01:53:32,013 - INFO - Starting upload: lib/questionary/__pycache__/constants.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/constants.cpython-314.pyc (941 bytes)
+2026-02-12 01:53:32,014 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/constants.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,015 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/constants.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,015 - DEBUG - Progress: 941/941 bytes
+2026-02-12 01:53:32,015 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,015 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/constants.cpython-314.pyc')
+2026-02-12 01:53:32,017 - INFO - Upload completed successfully: constants.cpython-314.pyc
+2026-02-12 01:53:32,017 - INFO - Starting upload: lib/questionary/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/questionary/__pycache__/__init__.cpython-314.pyc (1547 bytes)
+2026-02-12 01:53:32,018 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,019 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,019 - DEBUG - Progress: 1547/1547 bytes
+2026-02-12 01:53:32,019 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,020 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,023 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,023 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/questionary/prompts', 511)
+2026-02-12 01:53:32,023 - INFO - Created remote folder: /home/kevin/test/lib/questionary/prompts
+2026-02-12 01:53:32,024 - INFO - Starting upload: lib/questionary/prompts/text.py -> /home/kevin/test/lib/questionary/prompts/text.py (3403 bytes)
+2026-02-12 01:53:32,026 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/text.py', 'wb')
+2026-02-12 01:53:32,026 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/text.py', 'wb') -> 00000000
+2026-02-12 01:53:32,026 - DEBUG - Progress: 3403/3403 bytes
+2026-02-12 01:53:32,026 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,027 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/text.py')
+2026-02-12 01:53:32,029 - INFO - Upload completed successfully: text.py
+2026-02-12 01:53:32,029 - INFO - Starting upload: lib/questionary/prompts/select.py -> /home/kevin/test/lib/questionary/prompts/select.py (10864 bytes)
+2026-02-12 01:53:32,030 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/select.py', 'wb')
+2026-02-12 01:53:32,031 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/select.py', 'wb') -> 00000000
+2026-02-12 01:53:32,031 - DEBUG - Progress: 10864/10864 bytes
+2026-02-12 01:53:32,031 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,032 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/select.py')
+2026-02-12 01:53:32,034 - INFO - Upload completed successfully: select.py
+2026-02-12 01:53:32,034 - INFO - Starting upload: lib/questionary/prompts/rawselect.py -> /home/kevin/test/lib/questionary/prompts/rawselect.py (2486 bytes)
+2026-02-12 01:53:32,035 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/rawselect.py', 'wb')
+2026-02-12 01:53:32,036 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/rawselect.py', 'wb') -> 00000000
+2026-02-12 01:53:32,036 - DEBUG - Progress: 2486/2486 bytes
+2026-02-12 01:53:32,036 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,037 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/rawselect.py')
+2026-02-12 01:53:32,039 - INFO - Upload completed successfully: rawselect.py
+2026-02-12 01:53:32,039 - INFO - Starting upload: lib/questionary/prompts/press_any_key_to_continue.py -> /home/kevin/test/lib/questionary/prompts/press_any_key_to_continue.py (1662 bytes)
+2026-02-12 01:53:32,041 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/press_any_key_to_continue.py', 'wb')
+2026-02-12 01:53:32,041 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/press_any_key_to_continue.py', 'wb') -> 00000000
+2026-02-12 01:53:32,041 - DEBUG - Progress: 1662/1662 bytes
+2026-02-12 01:53:32,041 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,042 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/press_any_key_to_continue.py')
+2026-02-12 01:53:32,044 - INFO - Upload completed successfully: press_any_key_to_continue.py
+2026-02-12 01:53:32,044 - INFO - Starting upload: lib/questionary/prompts/path.py -> /home/kevin/test/lib/questionary/prompts/path.py (9911 bytes)
+2026-02-12 01:53:32,045 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/path.py', 'wb')
+2026-02-12 01:53:32,046 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/path.py', 'wb') -> 00000000
+2026-02-12 01:53:32,046 - DEBUG - Progress: 9911/9911 bytes
+2026-02-12 01:53:32,046 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,048 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/path.py')
+2026-02-12 01:53:32,049 - INFO - Upload completed successfully: path.py
+2026-02-12 01:53:32,049 - INFO - Starting upload: lib/questionary/prompts/password.py -> /home/kevin/test/lib/questionary/prompts/password.py (1998 bytes)
+2026-02-12 01:53:32,051 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/password.py', 'wb')
+2026-02-12 01:53:32,051 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/password.py', 'wb') -> 00000000
+2026-02-12 01:53:32,052 - DEBUG - Progress: 1998/1998 bytes
+2026-02-12 01:53:32,052 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,053 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/password.py')
+2026-02-12 01:53:32,055 - INFO - Upload completed successfully: password.py
+2026-02-12 01:53:32,055 - INFO - Starting upload: lib/questionary/prompts/confirm.py -> /home/kevin/test/lib/questionary/prompts/confirm.py (4085 bytes)
+2026-02-12 01:53:32,056 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/confirm.py', 'wb')
+2026-02-12 01:53:32,057 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/confirm.py', 'wb') -> 00000000
+2026-02-12 01:53:32,057 - DEBUG - Progress: 4085/4085 bytes
+2026-02-12 01:53:32,057 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,058 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/confirm.py')
+2026-02-12 01:53:32,060 - INFO - Upload completed successfully: confirm.py
+2026-02-12 01:53:32,060 - INFO - Starting upload: lib/questionary/prompts/common.py -> /home/kevin/test/lib/questionary/prompts/common.py (21719 bytes)
+2026-02-12 01:53:32,061 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/common.py', 'wb')
+2026-02-12 01:53:32,062 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/common.py', 'wb') -> 00000000
+2026-02-12 01:53:32,062 - DEBUG - Progress: 21719/21719 bytes
+2026-02-12 01:53:32,062 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,065 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/common.py')
+2026-02-12 01:53:32,067 - INFO - Upload completed successfully: common.py
+2026-02-12 01:53:32,067 - INFO - Starting upload: lib/questionary/prompts/checkbox.py -> /home/kevin/test/lib/questionary/prompts/checkbox.py (11564 bytes)
+2026-02-12 01:53:32,068 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/checkbox.py', 'wb')
+2026-02-12 01:53:32,069 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/checkbox.py', 'wb') -> 00000000
+2026-02-12 01:53:32,069 - DEBUG - Progress: 11564/11564 bytes
+2026-02-12 01:53:32,069 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,071 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/checkbox.py')
+2026-02-12 01:53:32,073 - INFO - Upload completed successfully: checkbox.py
+2026-02-12 01:53:32,073 - INFO - Starting upload: lib/questionary/prompts/autocomplete.py -> /home/kevin/test/lib/questionary/prompts/autocomplete.py (7292 bytes)
+2026-02-12 01:53:32,074 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/autocomplete.py', 'wb')
+2026-02-12 01:53:32,075 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/autocomplete.py', 'wb') -> 00000000
+2026-02-12 01:53:32,075 - DEBUG - Progress: 7292/7292 bytes
+2026-02-12 01:53:32,075 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,077 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/autocomplete.py')
+2026-02-12 01:53:32,078 - INFO - Upload completed successfully: autocomplete.py
+2026-02-12 01:53:32,078 - INFO - Starting upload: lib/questionary/prompts/__init__.py -> /home/kevin/test/lib/questionary/prompts/__init__.py (940 bytes)
+2026-02-12 01:53:32,080 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__init__.py', 'wb')
+2026-02-12 01:53:32,080 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,081 - DEBUG - Progress: 940/940 bytes
+2026-02-12 01:53:32,081 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,081 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__init__.py')
+2026-02-12 01:53:32,083 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,083 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/questionary/prompts/__pycache__', 511)
+2026-02-12 01:53:32,084 - INFO - Created remote folder: /home/kevin/test/lib/questionary/prompts/__pycache__
+2026-02-12 01:53:32,084 - INFO - Starting upload: lib/questionary/prompts/__pycache__/text.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/text.cpython-314.pyc (4491 bytes)
+2026-02-12 01:53:32,085 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/text.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,086 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/text.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,086 - DEBUG - Progress: 4491/4491 bytes
+2026-02-12 01:53:32,086 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,087 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/text.cpython-314.pyc')
+2026-02-12 01:53:32,089 - INFO - Upload completed successfully: text.cpython-314.pyc
+2026-02-12 01:53:32,089 - INFO - Starting upload: lib/questionary/prompts/__pycache__/select.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/select.cpython-314.pyc (14111 bytes)
+2026-02-12 01:53:32,090 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/select.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,091 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/select.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,091 - DEBUG - Progress: 14111/14111 bytes
+2026-02-12 01:53:32,091 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,095 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/select.cpython-314.pyc')
+2026-02-12 01:53:32,097 - INFO - Upload completed successfully: select.cpython-314.pyc
+2026-02-12 01:53:32,097 - INFO - Starting upload: lib/questionary/prompts/__pycache__/rawselect.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/rawselect.cpython-314.pyc (3045 bytes)
+2026-02-12 01:53:32,098 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/rawselect.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,099 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/rawselect.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,099 - DEBUG - Progress: 3045/3045 bytes
+2026-02-12 01:53:32,099 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,100 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/rawselect.cpython-314.pyc')
+2026-02-12 01:53:32,102 - INFO - Upload completed successfully: rawselect.cpython-314.pyc
+2026-02-12 01:53:32,102 - INFO - Starting upload: lib/questionary/prompts/__pycache__/press_any_key_to_continue.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/press_any_key_to_continue.cpython-314.pyc (2774 bytes)
+2026-02-12 01:53:32,104 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/press_any_key_to_continue.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,104 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/press_any_key_to_continue.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,104 - DEBUG - Progress: 2774/2774 bytes
+2026-02-12 01:53:32,105 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,106 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/press_any_key_to_continue.cpython-314.pyc')
+2026-02-12 01:53:32,108 - INFO - Upload completed successfully: press_any_key_to_continue.cpython-314.pyc
+2026-02-12 01:53:32,108 - INFO - Starting upload: lib/questionary/prompts/__pycache__/path.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/path.cpython-314.pyc (12398 bytes)
+2026-02-12 01:53:32,109 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/path.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,110 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/path.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,110 - DEBUG - Progress: 12398/12398 bytes
+2026-02-12 01:53:32,110 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,112 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/path.cpython-314.pyc')
+2026-02-12 01:53:32,114 - INFO - Upload completed successfully: path.cpython-314.pyc
+2026-02-12 01:53:32,114 - INFO - Starting upload: lib/questionary/prompts/__pycache__/password.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/password.cpython-314.pyc (2475 bytes)
+2026-02-12 01:53:32,116 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/password.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,117 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/password.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,117 - DEBUG - Progress: 2475/2475 bytes
+2026-02-12 01:53:32,117 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,118 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/password.cpython-314.pyc')
+2026-02-12 01:53:32,120 - INFO - Upload completed successfully: password.cpython-314.pyc
+2026-02-12 01:53:32,120 - INFO - Starting upload: lib/questionary/prompts/__pycache__/confirm.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/confirm.cpython-314.pyc (6381 bytes)
+2026-02-12 01:53:32,121 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/confirm.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,122 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/confirm.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,122 - DEBUG - Progress: 6381/6381 bytes
+2026-02-12 01:53:32,122 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,123 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/confirm.cpython-314.pyc')
+2026-02-12 01:53:32,125 - INFO - Upload completed successfully: confirm.cpython-314.pyc
+2026-02-12 01:53:32,125 - INFO - Starting upload: lib/questionary/prompts/__pycache__/common.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/common.cpython-314.pyc (32807 bytes)
+2026-02-12 01:53:32,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/common.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/common.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,127 - DEBUG - Progress: 32768/32807 bytes
+2026-02-12 01:53:32,128 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,131 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/common.cpython-314.pyc')
+2026-02-12 01:53:32,133 - INFO - Upload completed successfully: common.cpython-314.pyc
+2026-02-12 01:53:32,133 - INFO - Starting upload: lib/questionary/prompts/__pycache__/checkbox.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/checkbox.cpython-314.pyc (15875 bytes)
+2026-02-12 01:53:32,134 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/checkbox.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,135 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/checkbox.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,135 - DEBUG - Progress: 15875/15875 bytes
+2026-02-12 01:53:32,135 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,139 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/checkbox.cpython-314.pyc')
+2026-02-12 01:53:32,141 - INFO - Upload completed successfully: checkbox.cpython-314.pyc
+2026-02-12 01:53:32,142 - INFO - Starting upload: lib/questionary/prompts/__pycache__/autocomplete.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/autocomplete.cpython-314.pyc (10778 bytes)
+2026-02-12 01:53:32,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/autocomplete.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/autocomplete.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,144 - DEBUG - Progress: 10778/10778 bytes
+2026-02-12 01:53:32,145 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,146 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/autocomplete.cpython-314.pyc')
+2026-02-12 01:53:32,148 - INFO - Upload completed successfully: autocomplete.cpython-314.pyc
+2026-02-12 01:53:32,148 - INFO - Starting upload: lib/questionary/prompts/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/questionary/prompts/__pycache__/__init__.cpython-314.pyc (1171 bytes)
+2026-02-12 01:53:32,150 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,151 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/questionary/prompts/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,151 - DEBUG - Progress: 1171/1171 bytes
+2026-02-12 01:53:32,151 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,152 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/questionary/prompts/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,154 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,154 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info', 511)
+2026-02-12 01:53:32,155 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info
+2026-02-12 01:53:32,155 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/RECORD -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/RECORD (23851 bytes)
+2026-02-12 01:53:32,157 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/RECORD', 'wb')
+2026-02-12 01:53:32,158 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:32,159 - DEBUG - Progress: 23851/23851 bytes
+2026-02-12 01:53:32,159 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,161 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/RECORD')
+2026-02-12 01:53:32,163 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:32,163 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/INSTALLER -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:32,164 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:32,165 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:32,165 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:32,165 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,166 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/INSTALLER')
+2026-02-12 01:53:32,167 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:32,167 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/top_level.txt -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt (15 bytes)
+2026-02-12 01:53:32,169 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt', 'wb')
+2026-02-12 01:53:32,169 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt', 'wb') -> 00000000
+2026-02-12 01:53:32,170 - DEBUG - Progress: 15/15 bytes
+2026-02-12 01:53:32,170 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,170 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/top_level.txt')
+2026-02-12 01:53:32,172 - INFO - Upload completed successfully: top_level.txt
+2026-02-12 01:53:32,172 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/WHEEL -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/WHEEL (91 bytes)
+2026-02-12 01:53:32,173 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:32,174 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:32,174 - DEBUG - Progress: 91/91 bytes
+2026-02-12 01:53:32,174 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,175 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/WHEEL')
+2026-02-12 01:53:32,176 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:32,176 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/METADATA -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/METADATA (6414 bytes)
+2026-02-12 01:53:32,177 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/METADATA', 'wb')
+2026-02-12 01:53:32,178 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:32,178 - DEBUG - Progress: 6414/6414 bytes
+2026-02-12 01:53:32,178 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,179 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/METADATA')
+2026-02-12 01:53:32,181 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:32,181 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses', 511)
+2026-02-12 01:53:32,182 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses
+2026-02-12 01:53:32,182 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE (1493 bytes)
+2026-02-12 01:53:32,184 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE', 'wb')
+2026-02-12 01:53:32,184 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:32,184 - DEBUG - Progress: 1493/1493 bytes
+2026-02-12 01:53:32,184 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,185 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/LICENSE')
+2026-02-12 01:53:32,187 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:32,187 - INFO - Starting upload: lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst -> /home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst (148 bytes)
+2026-02-12 01:53:32,188 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst', 'wb')
+2026-02-12 01:53:32,189 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst', 'wb') -> 00000000
+2026-02-12 01:53:32,189 - DEBUG - Progress: 148/148 bytes
+2026-02-12 01:53:32,189 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,190 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit-3.0.52.dist-info/licenses/AUTHORS.rst')
+2026-02-12 01:53:32,191 - INFO - Upload completed successfully: AUTHORS.rst
+2026-02-12 01:53:32,191 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit', 511)
+2026-02-12 01:53:32,192 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit
+2026-02-12 01:53:32,192 - INFO - Starting upload: lib/prompt_toolkit/win32_types.py -> /home/kevin/test/lib/prompt_toolkit/win32_types.py (5551 bytes)
+2026-02-12 01:53:32,193 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/win32_types.py', 'wb')
+2026-02-12 01:53:32,194 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/win32_types.py', 'wb') -> 00000000
+2026-02-12 01:53:32,194 - DEBUG - Progress: 5551/5551 bytes
+2026-02-12 01:53:32,194 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,195 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/win32_types.py')
+2026-02-12 01:53:32,197 - INFO - Upload completed successfully: win32_types.py
+2026-02-12 01:53:32,197 - INFO - Starting upload: lib/prompt_toolkit/validation.py -> /home/kevin/test/lib/prompt_toolkit/validation.py (5807 bytes)
+2026-02-12 01:53:32,198 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/validation.py', 'wb')
+2026-02-12 01:53:32,199 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/validation.py', 'wb') -> 00000000
+2026-02-12 01:53:32,199 - DEBUG - Progress: 5807/5807 bytes
+2026-02-12 01:53:32,199 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,200 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/validation.py')
+2026-02-12 01:53:32,202 - INFO - Upload completed successfully: validation.py
+2026-02-12 01:53:32,202 - INFO - Starting upload: lib/prompt_toolkit/utils.py -> /home/kevin/test/lib/prompt_toolkit/utils.py (8631 bytes)
+2026-02-12 01:53:32,203 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/utils.py', 'wb')
+2026-02-12 01:53:32,204 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:32,204 - DEBUG - Progress: 8631/8631 bytes
+2026-02-12 01:53:32,204 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,206 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/utils.py')
+2026-02-12 01:53:32,208 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:32,208 - INFO - Starting upload: lib/prompt_toolkit/token.py -> /home/kevin/test/lib/prompt_toolkit/token.py (121 bytes)
+2026-02-12 01:53:32,209 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/token.py', 'wb')
+2026-02-12 01:53:32,210 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/token.py', 'wb') -> 00000000
+2026-02-12 01:53:32,210 - DEBUG - Progress: 121/121 bytes
+2026-02-12 01:53:32,210 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,211 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/token.py')
+2026-02-12 01:53:32,212 - INFO - Upload completed successfully: token.py
+2026-02-12 01:53:32,212 - INFO - Starting upload: lib/prompt_toolkit/selection.py -> /home/kevin/test/lib/prompt_toolkit/selection.py (1274 bytes)
+2026-02-12 01:53:32,214 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/selection.py', 'wb')
+2026-02-12 01:53:32,214 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/selection.py', 'wb') -> 00000000
+2026-02-12 01:53:32,214 - DEBUG - Progress: 1274/1274 bytes
+2026-02-12 01:53:32,214 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,215 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/selection.py')
+2026-02-12 01:53:32,217 - INFO - Upload completed successfully: selection.py
+2026-02-12 01:53:32,217 - INFO - Starting upload: lib/prompt_toolkit/search.py -> /home/kevin/test/lib/prompt_toolkit/search.py (6951 bytes)
+2026-02-12 01:53:32,218 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/search.py', 'wb')
+2026-02-12 01:53:32,218 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/search.py', 'wb') -> 00000000
+2026-02-12 01:53:32,219 - DEBUG - Progress: 6951/6951 bytes
+2026-02-12 01:53:32,219 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,220 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/search.py')
+2026-02-12 01:53:32,222 - INFO - Upload completed successfully: search.py
+2026-02-12 01:53:32,222 - INFO - Starting upload: lib/prompt_toolkit/renderer.py -> /home/kevin/test/lib/prompt_toolkit/renderer.py (29398 bytes)
+2026-02-12 01:53:32,223 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/renderer.py', 'wb')
+2026-02-12 01:53:32,224 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/renderer.py', 'wb') -> 00000000
+2026-02-12 01:53:32,225 - DEBUG - Progress: 29398/29398 bytes
+2026-02-12 01:53:32,225 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,228 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/renderer.py')
+2026-02-12 01:53:32,229 - INFO - Upload completed successfully: renderer.py
+2026-02-12 01:53:32,230 - INFO - Starting upload: lib/prompt_toolkit/py.typed -> /home/kevin/test/lib/prompt_toolkit/py.typed (0 bytes)
+2026-02-12 01:53:32,231 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/py.typed', 'wb')
+2026-02-12 01:53:32,231 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/py.typed', 'wb') -> 00000000
+2026-02-12 01:53:32,232 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,232 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/py.typed')
+2026-02-12 01:53:32,233 - INFO - Upload completed successfully: py.typed
+2026-02-12 01:53:32,234 - INFO - Starting upload: lib/prompt_toolkit/patch_stdout.py -> /home/kevin/test/lib/prompt_toolkit/patch_stdout.py (9477 bytes)
+2026-02-12 01:53:32,235 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/patch_stdout.py', 'wb')
+2026-02-12 01:53:32,236 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/patch_stdout.py', 'wb') -> 00000000
+2026-02-12 01:53:32,236 - DEBUG - Progress: 9477/9477 bytes
+2026-02-12 01:53:32,236 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,238 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/patch_stdout.py')
+2026-02-12 01:53:32,239 - INFO - Upload completed successfully: patch_stdout.py
+2026-02-12 01:53:32,239 - INFO - Starting upload: lib/prompt_toolkit/mouse_events.py -> /home/kevin/test/lib/prompt_toolkit/mouse_events.py (2473 bytes)
+2026-02-12 01:53:32,241 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/mouse_events.py', 'wb')
+2026-02-12 01:53:32,241 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/mouse_events.py', 'wb') -> 00000000
+2026-02-12 01:53:32,241 - DEBUG - Progress: 2473/2473 bytes
+2026-02-12 01:53:32,241 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,243 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/mouse_events.py')
+2026-02-12 01:53:32,244 - INFO - Upload completed successfully: mouse_events.py
+2026-02-12 01:53:32,244 - INFO - Starting upload: lib/prompt_toolkit/log.py -> /home/kevin/test/lib/prompt_toolkit/log.py (153 bytes)
+2026-02-12 01:53:32,245 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/log.py', 'wb')
+2026-02-12 01:53:32,246 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/log.py', 'wb') -> 00000000
+2026-02-12 01:53:32,246 - DEBUG - Progress: 153/153 bytes
+2026-02-12 01:53:32,246 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,247 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/log.py')
+2026-02-12 01:53:32,248 - INFO - Upload completed successfully: log.py
+2026-02-12 01:53:32,248 - INFO - Starting upload: lib/prompt_toolkit/keys.py -> /home/kevin/test/lib/prompt_toolkit/keys.py (4916 bytes)
+2026-02-12 01:53:32,250 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/keys.py', 'wb')
+2026-02-12 01:53:32,250 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/keys.py', 'wb') -> 00000000
+2026-02-12 01:53:32,251 - DEBUG - Progress: 4916/4916 bytes
+2026-02-12 01:53:32,251 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,252 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/keys.py')
+2026-02-12 01:53:32,253 - INFO - Upload completed successfully: keys.py
+2026-02-12 01:53:32,254 - INFO - Starting upload: lib/prompt_toolkit/history.py -> /home/kevin/test/lib/prompt_toolkit/history.py (9441 bytes)
+2026-02-12 01:53:32,255 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/history.py', 'wb')
+2026-02-12 01:53:32,255 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/history.py', 'wb') -> 00000000
+2026-02-12 01:53:32,256 - DEBUG - Progress: 9441/9441 bytes
+2026-02-12 01:53:32,256 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,258 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/history.py')
+2026-02-12 01:53:32,259 - INFO - Upload completed successfully: history.py
+2026-02-12 01:53:32,259 - INFO - Starting upload: lib/prompt_toolkit/enums.py -> /home/kevin/test/lib/prompt_toolkit/enums.py (358 bytes)
+2026-02-12 01:53:32,261 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/enums.py', 'wb')
+2026-02-12 01:53:32,261 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/enums.py', 'wb') -> 00000000
+2026-02-12 01:53:32,261 - DEBUG - Progress: 358/358 bytes
+2026-02-12 01:53:32,262 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,262 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/enums.py')
+2026-02-12 01:53:32,264 - INFO - Upload completed successfully: enums.py
+2026-02-12 01:53:32,264 - INFO - Starting upload: lib/prompt_toolkit/document.py -> /home/kevin/test/lib/prompt_toolkit/document.py (40547 bytes)
+2026-02-12 01:53:32,266 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/document.py', 'wb')
+2026-02-12 01:53:32,266 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/document.py', 'wb') -> 00000000
+2026-02-12 01:53:32,266 - DEBUG - Progress: 32768/40547 bytes
+2026-02-12 01:53:32,267 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,271 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/document.py')
+2026-02-12 01:53:32,273 - INFO - Upload completed successfully: document.py
+2026-02-12 01:53:32,273 - INFO - Starting upload: lib/prompt_toolkit/data_structures.py -> /home/kevin/test/lib/prompt_toolkit/data_structures.py (212 bytes)
+2026-02-12 01:53:32,274 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/data_structures.py', 'wb')
+2026-02-12 01:53:32,275 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/data_structures.py', 'wb') -> 00000000
+2026-02-12 01:53:32,275 - DEBUG - Progress: 212/212 bytes
+2026-02-12 01:53:32,275 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,276 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/data_structures.py')
+2026-02-12 01:53:32,277 - INFO - Upload completed successfully: data_structures.py
+2026-02-12 01:53:32,277 - INFO - Starting upload: lib/prompt_toolkit/cursor_shapes.py -> /home/kevin/test/lib/prompt_toolkit/cursor_shapes.py (3721 bytes)
+2026-02-12 01:53:32,279 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/cursor_shapes.py', 'wb')
+2026-02-12 01:53:32,279 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/cursor_shapes.py', 'wb') -> 00000000
+2026-02-12 01:53:32,279 - DEBUG - Progress: 3721/3721 bytes
+2026-02-12 01:53:32,280 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,281 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/cursor_shapes.py')
+2026-02-12 01:53:32,282 - INFO - Upload completed successfully: cursor_shapes.py
+2026-02-12 01:53:32,282 - INFO - Starting upload: lib/prompt_toolkit/cache.py -> /home/kevin/test/lib/prompt_toolkit/cache.py (3823 bytes)
+2026-02-12 01:53:32,283 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/cache.py', 'wb')
+2026-02-12 01:53:32,284 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/cache.py', 'wb') -> 00000000
+2026-02-12 01:53:32,284 - DEBUG - Progress: 3823/3823 bytes
+2026-02-12 01:53:32,284 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,285 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/cache.py')
+2026-02-12 01:53:32,287 - INFO - Upload completed successfully: cache.py
+2026-02-12 01:53:32,287 - INFO - Starting upload: lib/prompt_toolkit/buffer.py -> /home/kevin/test/lib/prompt_toolkit/buffer.py (74513 bytes)
+2026-02-12 01:53:32,289 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/buffer.py', 'wb')
+2026-02-12 01:53:32,290 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/buffer.py', 'wb') -> 00000000
+2026-02-12 01:53:32,291 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,301 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/buffer.py')
+2026-02-12 01:53:32,304 - INFO - Upload completed successfully: buffer.py
+2026-02-12 01:53:32,304 - INFO - Starting upload: lib/prompt_toolkit/auto_suggest.py -> /home/kevin/test/lib/prompt_toolkit/auto_suggest.py (5798 bytes)
+2026-02-12 01:53:32,306 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/auto_suggest.py', 'wb')
+2026-02-12 01:53:32,306 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/auto_suggest.py', 'wb') -> 00000000
+2026-02-12 01:53:32,307 - DEBUG - Progress: 5798/5798 bytes
+2026-02-12 01:53:32,307 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,308 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/auto_suggest.py')
+2026-02-12 01:53:32,310 - INFO - Upload completed successfully: auto_suggest.py
+2026-02-12 01:53:32,311 - INFO - Starting upload: lib/prompt_toolkit/__init__.py -> /home/kevin/test/lib/prompt_toolkit/__init__.py (1376 bytes)
+2026-02-12 01:53:32,312 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__init__.py', 'wb')
+2026-02-12 01:53:32,313 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,313 - DEBUG - Progress: 1376/1376 bytes
+2026-02-12 01:53:32,313 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,314 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__init__.py')
+2026-02-12 01:53:32,316 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,317 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/__pycache__', 511)
+2026-02-12 01:53:32,317 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/__pycache__
+2026-02-12 01:53:32,317 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc (8004 bytes)
+2026-02-12 01:53:32,319 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,320 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,321 - DEBUG - Progress: 8004/8004 bytes
+2026-02-12 01:53:32,321 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,322 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/win32_types.cpython-314.pyc')
+2026-02-12 01:53:32,324 - INFO - Upload completed successfully: win32_types.cpython-314.pyc
+2026-02-12 01:53:32,325 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc (11639 bytes)
+2026-02-12 01:53:32,326 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,327 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,327 - DEBUG - Progress: 11639/11639 bytes
+2026-02-12 01:53:32,327 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,329 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/validation.cpython-314.pyc')
+2026-02-12 01:53:32,331 - INFO - Upload completed successfully: validation.cpython-314.pyc
+2026-02-12 01:53:32,332 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc (14612 bytes)
+2026-02-12 01:53:32,333 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,334 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,334 - DEBUG - Progress: 14612/14612 bytes
+2026-02-12 01:53:32,334 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,336 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:32,338 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:32,338 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/token.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc (299 bytes)
+2026-02-12 01:53:32,339 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,340 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,340 - DEBUG - Progress: 299/299 bytes
+2026-02-12 01:53:32,340 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,341 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/token.cpython-314.pyc')
+2026-02-12 01:53:32,342 - INFO - Upload completed successfully: token.cpython-314.pyc
+2026-02-12 01:53:32,342 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc (2576 bytes)
+2026-02-12 01:53:32,344 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,344 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,344 - DEBUG - Progress: 2576/2576 bytes
+2026-02-12 01:53:32,344 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,346 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/selection.cpython-314.pyc')
+2026-02-12 01:53:32,347 - INFO - Upload completed successfully: selection.cpython-314.pyc
+2026-02-12 01:53:32,347 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/search.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc (8905 bytes)
+2026-02-12 01:53:32,348 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,349 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,349 - DEBUG - Progress: 8905/8905 bytes
+2026-02-12 01:53:32,349 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,351 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/search.cpython-314.pyc')
+2026-02-12 01:53:32,352 - INFO - Upload completed successfully: search.cpython-314.pyc
+2026-02-12 01:53:32,352 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc (33951 bytes)
+2026-02-12 01:53:32,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,355 - DEBUG - Progress: 32768/33951 bytes
+2026-02-12 01:53:32,355 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,359 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/renderer.cpython-314.pyc')
+2026-02-12 01:53:32,362 - INFO - Upload completed successfully: renderer.cpython-314.pyc
+2026-02-12 01:53:32,362 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc (14544 bytes)
+2026-02-12 01:53:32,366 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,367 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,368 - DEBUG - Progress: 14544/14544 bytes
+2026-02-12 01:53:32,368 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,370 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/patch_stdout.cpython-314.pyc')
+2026-02-12 01:53:32,373 - INFO - Upload completed successfully: patch_stdout.cpython-314.pyc
+2026-02-12 01:53:32,374 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc (3275 bytes)
+2026-02-12 01:53:32,377 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,378 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,378 - DEBUG - Progress: 3275/3275 bytes
+2026-02-12 01:53:32,379 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,380 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/mouse_events.cpython-314.pyc')
+2026-02-12 01:53:32,383 - INFO - Upload completed successfully: mouse_events.cpython-314.pyc
+2026-02-12 01:53:32,383 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/log.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc (384 bytes)
+2026-02-12 01:53:32,386 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,387 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,387 - DEBUG - Progress: 384/384 bytes
+2026-02-12 01:53:32,387 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,388 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/log.cpython-314.pyc')
+2026-02-12 01:53:32,389 - INFO - Upload completed successfully: log.cpython-314.pyc
+2026-02-12 01:53:32,389 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc (5409 bytes)
+2026-02-12 01:53:32,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,391 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,391 - DEBUG - Progress: 5409/5409 bytes
+2026-02-12 01:53:32,391 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,392 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/keys.cpython-314.pyc')
+2026-02-12 01:53:32,394 - INFO - Upload completed successfully: keys.cpython-314.pyc
+2026-02-12 01:53:32,394 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/history.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc (16925 bytes)
+2026-02-12 01:53:32,395 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,396 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,396 - DEBUG - Progress: 16925/16925 bytes
+2026-02-12 01:53:32,396 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,400 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/history.cpython-314.pyc')
+2026-02-12 01:53:32,401 - INFO - Upload completed successfully: history.cpython-314.pyc
+2026-02-12 01:53:32,402 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc (606 bytes)
+2026-02-12 01:53:32,403 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,403 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,404 - DEBUG - Progress: 606/606 bytes
+2026-02-12 01:53:32,404 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,404 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/enums.cpython-314.pyc')
+2026-02-12 01:53:32,406 - INFO - Upload completed successfully: enums.cpython-314.pyc
+2026-02-12 01:53:32,406 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/document.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc (57592 bytes)
+2026-02-12 01:53:32,407 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,408 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,408 - DEBUG - Progress: 32768/57592 bytes
+2026-02-12 01:53:32,408 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,416 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/document.cpython-314.pyc')
+2026-02-12 01:53:32,418 - INFO - Upload completed successfully: document.cpython-314.pyc
+2026-02-12 01:53:32,418 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc (781 bytes)
+2026-02-12 01:53:32,420 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,420 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,420 - DEBUG - Progress: 781/781 bytes
+2026-02-12 01:53:32,420 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,421 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/data_structures.cpython-314.pyc')
+2026-02-12 01:53:32,423 - INFO - Upload completed successfully: data_structures.cpython-314.pyc
+2026-02-12 01:53:32,423 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc (5921 bytes)
+2026-02-12 01:53:32,424 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,425 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,425 - DEBUG - Progress: 5921/5921 bytes
+2026-02-12 01:53:32,425 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,426 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/cursor_shapes.cpython-314.pyc')
+2026-02-12 01:53:32,428 - INFO - Upload completed successfully: cursor_shapes.cpython-314.pyc
+2026-02-12 01:53:32,428 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc (6758 bytes)
+2026-02-12 01:53:32,429 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,430 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,430 - DEBUG - Progress: 6758/6758 bytes
+2026-02-12 01:53:32,430 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,431 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/cache.cpython-314.pyc')
+2026-02-12 01:53:32,432 - INFO - Upload completed successfully: cache.cpython-314.pyc
+2026-02-12 01:53:32,432 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc (91689 bytes)
+2026-02-12 01:53:32,434 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,435 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,435 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,446 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/buffer.cpython-314.pyc')
+2026-02-12 01:53:32,449 - INFO - Upload completed successfully: buffer.cpython-314.pyc
+2026-02-12 01:53:32,449 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc (10748 bytes)
+2026-02-12 01:53:32,451 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,451 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,451 - DEBUG - Progress: 10748/10748 bytes
+2026-02-12 01:53:32,452 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,453 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/auto_suggest.cpython-314.pyc')
+2026-02-12 01:53:32,455 - INFO - Upload completed successfully: auto_suggest.cpython-314.pyc
+2026-02-12 01:53:32,455 - INFO - Starting upload: lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc (1885 bytes)
+2026-02-12 01:53:32,457 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,458 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,458 - DEBUG - Progress: 1885/1885 bytes
+2026-02-12 01:53:32,458 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,459 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,462 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,462 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/widgets', 511)
+2026-02-12 01:53:32,462 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/widgets
+2026-02-12 01:53:32,463 - INFO - Starting upload: lib/prompt_toolkit/widgets/toolbars.py -> /home/kevin/test/lib/prompt_toolkit/widgets/toolbars.py (12178 bytes)
+2026-02-12 01:53:32,467 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/toolbars.py', 'wb')
+2026-02-12 01:53:32,468 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/toolbars.py', 'wb') -> 00000000
+2026-02-12 01:53:32,468 - DEBUG - Progress: 12178/12178 bytes
+2026-02-12 01:53:32,468 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,470 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/toolbars.py')
+2026-02-12 01:53:32,472 - INFO - Upload completed successfully: toolbars.py
+2026-02-12 01:53:32,473 - INFO - Starting upload: lib/prompt_toolkit/widgets/menus.py -> /home/kevin/test/lib/prompt_toolkit/widgets/menus.py (13419 bytes)
+2026-02-12 01:53:32,475 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/menus.py', 'wb')
+2026-02-12 01:53:32,475 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/menus.py', 'wb') -> 00000000
+2026-02-12 01:53:32,476 - DEBUG - Progress: 13419/13419 bytes
+2026-02-12 01:53:32,476 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,478 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/menus.py')
+2026-02-12 01:53:32,480 - INFO - Upload completed successfully: menus.py
+2026-02-12 01:53:32,480 - INFO - Starting upload: lib/prompt_toolkit/widgets/dialogs.py -> /home/kevin/test/lib/prompt_toolkit/widgets/dialogs.py (3380 bytes)
+2026-02-12 01:53:32,482 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/dialogs.py', 'wb')
+2026-02-12 01:53:32,483 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/dialogs.py', 'wb') -> 00000000
+2026-02-12 01:53:32,483 - DEBUG - Progress: 3380/3380 bytes
+2026-02-12 01:53:32,483 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,484 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/dialogs.py')
+2026-02-12 01:53:32,487 - INFO - Upload completed successfully: dialogs.py
+2026-02-12 01:53:32,487 - INFO - Starting upload: lib/prompt_toolkit/widgets/base.py -> /home/kevin/test/lib/prompt_toolkit/widgets/base.py (35546 bytes)
+2026-02-12 01:53:32,489 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/base.py', 'wb')
+2026-02-12 01:53:32,489 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/base.py', 'wb') -> 00000000
+2026-02-12 01:53:32,490 - DEBUG - Progress: 32768/35546 bytes
+2026-02-12 01:53:32,490 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,494 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/base.py')
+2026-02-12 01:53:32,496 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:32,497 - INFO - Starting upload: lib/prompt_toolkit/widgets/__init__.py -> /home/kevin/test/lib/prompt_toolkit/widgets/__init__.py (1218 bytes)
+2026-02-12 01:53:32,498 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__init__.py', 'wb')
+2026-02-12 01:53:32,499 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,500 - DEBUG - Progress: 1218/1218 bytes
+2026-02-12 01:53:32,500 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,501 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/__init__.py')
+2026-02-12 01:53:32,503 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,503 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__', 511)
+2026-02-12 01:53:32,504 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/widgets/__pycache__
+2026-02-12 01:53:32,504 - INFO - Starting upload: lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc (20584 bytes)
+2026-02-12 01:53:32,505 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,506 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,506 - DEBUG - Progress: 20584/20584 bytes
+2026-02-12 01:53:32,506 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,509 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/toolbars.cpython-314.pyc')
+2026-02-12 01:53:32,511 - INFO - Upload completed successfully: toolbars.cpython-314.pyc
+2026-02-12 01:53:32,511 - INFO - Starting upload: lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc (22376 bytes)
+2026-02-12 01:53:32,512 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,513 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,513 - DEBUG - Progress: 22376/22376 bytes
+2026-02-12 01:53:32,513 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,517 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/menus.cpython-314.pyc')
+2026-02-12 01:53:32,518 - INFO - Upload completed successfully: menus.cpython-314.pyc
+2026-02-12 01:53:32,518 - INFO - Starting upload: lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc (4340 bytes)
+2026-02-12 01:53:32,520 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,520 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,520 - DEBUG - Progress: 4340/4340 bytes
+2026-02-12 01:53:32,520 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,522 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/dialogs.cpython-314.pyc')
+2026-02-12 01:53:32,523 - INFO - Upload completed successfully: dialogs.cpython-314.pyc
+2026-02-12 01:53:32,524 - INFO - Starting upload: lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc (50745 bytes)
+2026-02-12 01:53:32,525 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,526 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,526 - DEBUG - Progress: 32768/50745 bytes
+2026-02-12 01:53:32,526 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,532 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:32,534 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:32,534 - INFO - Starting upload: lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc (1228 bytes)
+2026-02-12 01:53:32,536 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,537 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,537 - DEBUG - Progress: 1228/1228 bytes
+2026-02-12 01:53:32,537 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,538 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/widgets/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,540 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,540 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/styles', 511)
+2026-02-12 01:53:32,541 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/styles
+2026-02-12 01:53:32,541 - INFO - Starting upload: lib/prompt_toolkit/styles/style_transformation.py -> /home/kevin/test/lib/prompt_toolkit/styles/style_transformation.py (12427 bytes)
+2026-02-12 01:53:32,543 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/style_transformation.py', 'wb')
+2026-02-12 01:53:32,543 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/style_transformation.py', 'wb') -> 00000000
+2026-02-12 01:53:32,544 - DEBUG - Progress: 12427/12427 bytes
+2026-02-12 01:53:32,544 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,546 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/style_transformation.py')
+2026-02-12 01:53:32,548 - INFO - Upload completed successfully: style_transformation.py
+2026-02-12 01:53:32,548 - INFO - Starting upload: lib/prompt_toolkit/styles/style.py -> /home/kevin/test/lib/prompt_toolkit/styles/style.py (13263 bytes)
+2026-02-12 01:53:32,549 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/style.py', 'wb')
+2026-02-12 01:53:32,550 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/style.py', 'wb') -> 00000000
+2026-02-12 01:53:32,550 - DEBUG - Progress: 13263/13263 bytes
+2026-02-12 01:53:32,550 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,552 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/style.py')
+2026-02-12 01:53:32,554 - INFO - Upload completed successfully: style.py
+2026-02-12 01:53:32,554 - INFO - Starting upload: lib/prompt_toolkit/styles/pygments.py -> /home/kevin/test/lib/prompt_toolkit/styles/pygments.py (1974 bytes)
+2026-02-12 01:53:32,555 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/pygments.py', 'wb')
+2026-02-12 01:53:32,556 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/pygments.py', 'wb') -> 00000000
+2026-02-12 01:53:32,556 - DEBUG - Progress: 1974/1974 bytes
+2026-02-12 01:53:32,556 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,557 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/pygments.py')
+2026-02-12 01:53:32,559 - INFO - Upload completed successfully: pygments.py
+2026-02-12 01:53:32,559 - INFO - Starting upload: lib/prompt_toolkit/styles/named_colors.py -> /home/kevin/test/lib/prompt_toolkit/styles/named_colors.py (4367 bytes)
+2026-02-12 01:53:32,560 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/named_colors.py', 'wb')
+2026-02-12 01:53:32,561 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/named_colors.py', 'wb') -> 00000000
+2026-02-12 01:53:32,561 - DEBUG - Progress: 4367/4367 bytes
+2026-02-12 01:53:32,561 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,562 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/named_colors.py')
+2026-02-12 01:53:32,564 - INFO - Upload completed successfully: named_colors.py
+2026-02-12 01:53:32,564 - INFO - Starting upload: lib/prompt_toolkit/styles/defaults.py -> /home/kevin/test/lib/prompt_toolkit/styles/defaults.py (8699 bytes)
+2026-02-12 01:53:32,565 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/defaults.py', 'wb')
+2026-02-12 01:53:32,566 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/defaults.py', 'wb') -> 00000000
+2026-02-12 01:53:32,567 - DEBUG - Progress: 8699/8699 bytes
+2026-02-12 01:53:32,567 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,568 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/defaults.py')
+2026-02-12 01:53:32,570 - INFO - Upload completed successfully: defaults.py
+2026-02-12 01:53:32,570 - INFO - Starting upload: lib/prompt_toolkit/styles/base.py -> /home/kevin/test/lib/prompt_toolkit/styles/base.py (5064 bytes)
+2026-02-12 01:53:32,572 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/base.py', 'wb')
+2026-02-12 01:53:32,572 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/base.py', 'wb') -> 00000000
+2026-02-12 01:53:32,572 - DEBUG - Progress: 5064/5064 bytes
+2026-02-12 01:53:32,573 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,574 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/base.py')
+2026-02-12 01:53:32,576 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:32,576 - INFO - Starting upload: lib/prompt_toolkit/styles/__init__.py -> /home/kevin/test/lib/prompt_toolkit/styles/__init__.py (1640 bytes)
+2026-02-12 01:53:32,577 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__init__.py', 'wb')
+2026-02-12 01:53:32,578 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,578 - DEBUG - Progress: 1640/1640 bytes
+2026-02-12 01:53:32,578 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,579 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__init__.py')
+2026-02-12 01:53:32,581 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,581 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__', 511)
+2026-02-12 01:53:32,581 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/styles/__pycache__
+2026-02-12 01:53:32,581 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc (20229 bytes)
+2026-02-12 01:53:32,583 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,583 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,584 - DEBUG - Progress: 20229/20229 bytes
+2026-02-12 01:53:32,584 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,588 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style_transformation.cpython-314.pyc')
+2026-02-12 01:53:32,590 - INFO - Upload completed successfully: style_transformation.cpython-314.pyc
+2026-02-12 01:53:32,590 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc (18603 bytes)
+2026-02-12 01:53:32,591 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,592 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,592 - DEBUG - Progress: 18603/18603 bytes
+2026-02-12 01:53:32,592 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,596 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/style.cpython-314.pyc')
+2026-02-12 01:53:32,598 - INFO - Upload completed successfully: style.cpython-314.pyc
+2026-02-12 01:53:32,598 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc (2859 bytes)
+2026-02-12 01:53:32,602 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,602 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,603 - DEBUG - Progress: 2859/2859 bytes
+2026-02-12 01:53:32,603 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,604 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/pygments.cpython-314.pyc')
+2026-02-12 01:53:32,608 - INFO - Upload completed successfully: pygments.cpython-314.pyc
+2026-02-12 01:53:32,608 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc (6395 bytes)
+2026-02-12 01:53:32,612 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,613 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,613 - DEBUG - Progress: 6395/6395 bytes
+2026-02-12 01:53:32,613 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,615 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/named_colors.cpython-314.pyc')
+2026-02-12 01:53:32,618 - INFO - Upload completed successfully: named_colors.cpython-314.pyc
+2026-02-12 01:53:32,618 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc (8286 bytes)
+2026-02-12 01:53:32,622 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,623 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,624 - DEBUG - Progress: 8286/8286 bytes
+2026-02-12 01:53:32,624 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,625 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/defaults.cpython-314.pyc')
+2026-02-12 01:53:32,627 - INFO - Upload completed successfully: defaults.cpython-314.pyc
+2026-02-12 01:53:32,627 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc (7097 bytes)
+2026-02-12 01:53:32,629 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,629 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,629 - DEBUG - Progress: 7097/7097 bytes
+2026-02-12 01:53:32,629 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,631 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:32,632 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:32,632 - INFO - Starting upload: lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc (1363 bytes)
+2026-02-12 01:53:32,634 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,634 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,635 - DEBUG - Progress: 1363/1363 bytes
+2026-02-12 01:53:32,635 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,635 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/styles/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,637 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,637 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/shortcuts', 511)
+2026-02-12 01:53:32,637 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/shortcuts
+2026-02-12 01:53:32,637 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/utils.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/utils.py (6950 bytes)
+2026-02-12 01:53:32,639 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/utils.py', 'wb')
+2026-02-12 01:53:32,639 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:32,640 - DEBUG - Progress: 6950/6950 bytes
+2026-02-12 01:53:32,640 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,641 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/utils.py')
+2026-02-12 01:53:32,643 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:32,643 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/prompt.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/prompt.py (60747 bytes)
+2026-02-12 01:53:32,644 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/prompt.py', 'wb')
+2026-02-12 01:53:32,645 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/prompt.py', 'wb') -> 00000000
+2026-02-12 01:53:32,645 - DEBUG - Progress: 32768/60747 bytes
+2026-02-12 01:53:32,645 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,655 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/prompt.py')
+2026-02-12 01:53:32,657 - INFO - Upload completed successfully: prompt.py
+2026-02-12 01:53:32,658 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/dialogs.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/dialogs.py (9007 bytes)
+2026-02-12 01:53:32,660 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/dialogs.py', 'wb')
+2026-02-12 01:53:32,660 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/dialogs.py', 'wb') -> 00000000
+2026-02-12 01:53:32,661 - DEBUG - Progress: 9007/9007 bytes
+2026-02-12 01:53:32,661 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,663 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/dialogs.py')
+2026-02-12 01:53:32,665 - INFO - Upload completed successfully: dialogs.py
+2026-02-12 01:53:32,665 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/choice_input.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/choice_input.py (10523 bytes)
+2026-02-12 01:53:32,667 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/choice_input.py', 'wb')
+2026-02-12 01:53:32,668 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/choice_input.py', 'wb') -> 00000000
+2026-02-12 01:53:32,669 - DEBUG - Progress: 10523/10523 bytes
+2026-02-12 01:53:32,669 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,670 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/choice_input.py')
+2026-02-12 01:53:32,673 - INFO - Upload completed successfully: choice_input.py
+2026-02-12 01:53:32,673 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/__init__.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/__init__.py (1020 bytes)
+2026-02-12 01:53:32,675 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__init__.py', 'wb')
+2026-02-12 01:53:32,676 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,676 - DEBUG - Progress: 1020/1020 bytes
+2026-02-12 01:53:32,676 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,677 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__init__.py')
+2026-02-12 01:53:32,679 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,679 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__', 511)
+2026-02-12 01:53:32,679 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__
+2026-02-12 01:53:32,679 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc (9360 bytes)
+2026-02-12 01:53:32,681 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,682 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,682 - DEBUG - Progress: 9360/9360 bytes
+2026-02-12 01:53:32,682 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,684 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:32,686 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:32,686 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc (66126 bytes)
+2026-02-12 01:53:32,688 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,689 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,689 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,697 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/prompt.cpython-314.pyc')
+2026-02-12 01:53:32,699 - INFO - Upload completed successfully: prompt.cpython-314.pyc
+2026-02-12 01:53:32,699 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc (14603 bytes)
+2026-02-12 01:53:32,700 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,701 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,701 - DEBUG - Progress: 14603/14603 bytes
+2026-02-12 01:53:32,701 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,705 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/dialogs.cpython-314.pyc')
+2026-02-12 01:53:32,707 - INFO - Upload completed successfully: dialogs.cpython-314.pyc
+2026-02-12 01:53:32,707 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc (13803 bytes)
+2026-02-12 01:53:32,709 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,710 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,710 - DEBUG - Progress: 13803/13803 bytes
+2026-02-12 01:53:32,710 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,712 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/choice_input.cpython-314.pyc')
+2026-02-12 01:53:32,714 - INFO - Upload completed successfully: choice_input.cpython-314.pyc
+2026-02-12 01:53:32,714 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc (960 bytes)
+2026-02-12 01:53:32,716 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,717 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,717 - DEBUG - Progress: 960/960 bytes
+2026-02-12 01:53:32,717 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,718 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,720 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,720 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar', 511)
+2026-02-12 01:53:32,721 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar
+2026-02-12 01:53:32,721 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/progress_bar/formatters.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py (11739 bytes)
+2026-02-12 01:53:32,723 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py', 'wb')
+2026-02-12 01:53:32,723 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py', 'wb') -> 00000000
+2026-02-12 01:53:32,724 - DEBUG - Progress: 11739/11739 bytes
+2026-02-12 01:53:32,724 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,726 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/formatters.py')
+2026-02-12 01:53:32,728 - INFO - Upload completed successfully: formatters.py
+2026-02-12 01:53:32,728 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/progress_bar/base.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/base.py (14402 bytes)
+2026-02-12 01:53:32,730 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/base.py', 'wb')
+2026-02-12 01:53:32,731 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/base.py', 'wb') -> 00000000
+2026-02-12 01:53:32,732 - DEBUG - Progress: 14402/14402 bytes
+2026-02-12 01:53:32,732 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,734 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/base.py')
+2026-02-12 01:53:32,736 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:32,736 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/progress_bar/__init__.py -> /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py (540 bytes)
+2026-02-12 01:53:32,738 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py', 'wb')
+2026-02-12 01:53:32,739 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,739 - DEBUG - Progress: 540/540 bytes
+2026-02-12 01:53:32,739 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,740 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__init__.py')
+2026-02-12 01:53:32,741 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,741 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__', 511)
+2026-02-12 01:53:32,742 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__
+2026-02-12 01:53:32,742 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc (21811 bytes)
+2026-02-12 01:53:32,743 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,744 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,744 - DEBUG - Progress: 21811/21811 bytes
+2026-02-12 01:53:32,744 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,747 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/formatters.cpython-314.pyc')
+2026-02-12 01:53:32,748 - INFO - Upload completed successfully: formatters.cpython-314.pyc
+2026-02-12 01:53:32,749 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc (22643 bytes)
+2026-02-12 01:53:32,750 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,751 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,751 - DEBUG - Progress: 22643/22643 bytes
+2026-02-12 01:53:32,751 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,754 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:32,756 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:32,756 - INFO - Starting upload: lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc (654 bytes)
+2026-02-12 01:53:32,757 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,758 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,758 - DEBUG - Progress: 654/654 bytes
+2026-02-12 01:53:32,758 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,759 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/shortcuts/progress_bar/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,760 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,760 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/output', 511)
+2026-02-12 01:53:32,761 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/output
+2026-02-12 01:53:32,761 - INFO - Starting upload: lib/prompt_toolkit/output/windows10.py -> /home/kevin/test/lib/prompt_toolkit/output/windows10.py (4362 bytes)
+2026-02-12 01:53:32,762 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/windows10.py', 'wb')
+2026-02-12 01:53:32,763 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/windows10.py', 'wb') -> 00000000
+2026-02-12 01:53:32,763 - DEBUG - Progress: 4362/4362 bytes
+2026-02-12 01:53:32,763 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,765 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/windows10.py')
+2026-02-12 01:53:32,766 - INFO - Upload completed successfully: windows10.py
+2026-02-12 01:53:32,766 - INFO - Starting upload: lib/prompt_toolkit/output/win32.py -> /home/kevin/test/lib/prompt_toolkit/output/win32.py (22639 bytes)
+2026-02-12 01:53:32,768 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/win32.py', 'wb')
+2026-02-12 01:53:32,768 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/win32.py', 'wb') -> 00000000
+2026-02-12 01:53:32,769 - DEBUG - Progress: 22639/22639 bytes
+2026-02-12 01:53:32,769 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,772 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/win32.py')
+2026-02-12 01:53:32,773 - INFO - Upload completed successfully: win32.py
+2026-02-12 01:53:32,774 - INFO - Starting upload: lib/prompt_toolkit/output/vt100.py -> /home/kevin/test/lib/prompt_toolkit/output/vt100.py (23474 bytes)
+2026-02-12 01:53:32,775 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/vt100.py', 'wb')
+2026-02-12 01:53:32,776 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/vt100.py', 'wb') -> 00000000
+2026-02-12 01:53:32,776 - DEBUG - Progress: 23474/23474 bytes
+2026-02-12 01:53:32,776 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,779 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/vt100.py')
+2026-02-12 01:53:32,781 - INFO - Upload completed successfully: vt100.py
+2026-02-12 01:53:32,781 - INFO - Starting upload: lib/prompt_toolkit/output/plain_text.py -> /home/kevin/test/lib/prompt_toolkit/output/plain_text.py (3296 bytes)
+2026-02-12 01:53:32,783 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/plain_text.py', 'wb')
+2026-02-12 01:53:32,784 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/plain_text.py', 'wb') -> 00000000
+2026-02-12 01:53:32,784 - DEBUG - Progress: 3296/3296 bytes
+2026-02-12 01:53:32,784 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,785 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/plain_text.py')
+2026-02-12 01:53:32,787 - INFO - Upload completed successfully: plain_text.py
+2026-02-12 01:53:32,787 - INFO - Starting upload: lib/prompt_toolkit/output/flush_stdout.py -> /home/kevin/test/lib/prompt_toolkit/output/flush_stdout.py (3236 bytes)
+2026-02-12 01:53:32,789 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/flush_stdout.py', 'wb')
+2026-02-12 01:53:32,790 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/flush_stdout.py', 'wb') -> 00000000
+2026-02-12 01:53:32,790 - DEBUG - Progress: 3236/3236 bytes
+2026-02-12 01:53:32,790 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,791 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/flush_stdout.py')
+2026-02-12 01:53:32,794 - INFO - Upload completed successfully: flush_stdout.py
+2026-02-12 01:53:32,795 - INFO - Starting upload: lib/prompt_toolkit/output/defaults.py -> /home/kevin/test/lib/prompt_toolkit/output/defaults.py (3689 bytes)
+2026-02-12 01:53:32,797 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/defaults.py', 'wb')
+2026-02-12 01:53:32,797 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/defaults.py', 'wb') -> 00000000
+2026-02-12 01:53:32,798 - DEBUG - Progress: 3689/3689 bytes
+2026-02-12 01:53:32,798 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,799 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/defaults.py')
+2026-02-12 01:53:32,801 - INFO - Upload completed successfully: defaults.py
+2026-02-12 01:53:32,801 - INFO - Starting upload: lib/prompt_toolkit/output/conemu.py -> /home/kevin/test/lib/prompt_toolkit/output/conemu.py (1865 bytes)
+2026-02-12 01:53:32,803 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/conemu.py', 'wb')
+2026-02-12 01:53:32,804 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/conemu.py', 'wb') -> 00000000
+2026-02-12 01:53:32,804 - DEBUG - Progress: 1865/1865 bytes
+2026-02-12 01:53:32,804 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,805 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/conemu.py')
+2026-02-12 01:53:32,807 - INFO - Upload completed successfully: conemu.py
+2026-02-12 01:53:32,807 - INFO - Starting upload: lib/prompt_toolkit/output/color_depth.py -> /home/kevin/test/lib/prompt_toolkit/output/color_depth.py (1569 bytes)
+2026-02-12 01:53:32,809 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/color_depth.py', 'wb')
+2026-02-12 01:53:32,810 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/color_depth.py', 'wb') -> 00000000
+2026-02-12 01:53:32,810 - DEBUG - Progress: 1569/1569 bytes
+2026-02-12 01:53:32,810 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,811 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/color_depth.py')
+2026-02-12 01:53:32,814 - INFO - Upload completed successfully: color_depth.py
+2026-02-12 01:53:32,814 - INFO - Starting upload: lib/prompt_toolkit/output/base.py -> /home/kevin/test/lib/prompt_toolkit/output/base.py (8348 bytes)
+2026-02-12 01:53:32,816 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/base.py', 'wb')
+2026-02-12 01:53:32,817 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/base.py', 'wb') -> 00000000
+2026-02-12 01:53:32,827 - DEBUG - Progress: 8348/8348 bytes
+2026-02-12 01:53:32,827 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,829 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/base.py')
+2026-02-12 01:53:32,831 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:32,832 - INFO - Starting upload: lib/prompt_toolkit/output/__init__.py -> /home/kevin/test/lib/prompt_toolkit/output/__init__.py (280 bytes)
+2026-02-12 01:53:32,834 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__init__.py', 'wb')
+2026-02-12 01:53:32,834 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,835 - DEBUG - Progress: 280/280 bytes
+2026-02-12 01:53:32,835 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,835 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__init__.py')
+2026-02-12 01:53:32,837 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,838 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__', 511)
+2026-02-12 01:53:32,838 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/output/__pycache__
+2026-02-12 01:53:32,839 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc (5848 bytes)
+2026-02-12 01:53:32,841 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,841 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,842 - DEBUG - Progress: 5848/5848 bytes
+2026-02-12 01:53:32,842 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,843 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/windows10.cpython-314.pyc')
+2026-02-12 01:53:32,846 - INFO - Upload completed successfully: windows10.cpython-314.pyc
+2026-02-12 01:53:32,846 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc (36227 bytes)
+2026-02-12 01:53:32,848 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,849 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,850 - DEBUG - Progress: 32768/36227 bytes
+2026-02-12 01:53:32,850 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,854 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/win32.cpython-314.pyc')
+2026-02-12 01:53:32,856 - INFO - Upload completed successfully: win32.cpython-314.pyc
+2026-02-12 01:53:32,856 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc (34951 bytes)
+2026-02-12 01:53:32,858 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,859 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,859 - DEBUG - Progress: 32768/34951 bytes
+2026-02-12 01:53:32,859 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,864 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/vt100.cpython-314.pyc')
+2026-02-12 01:53:32,866 - INFO - Upload completed successfully: vt100.cpython-314.pyc
+2026-02-12 01:53:32,866 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc (12406 bytes)
+2026-02-12 01:53:32,868 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,869 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,869 - DEBUG - Progress: 12406/12406 bytes
+2026-02-12 01:53:32,869 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,872 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/plain_text.cpython-314.pyc')
+2026-02-12 01:53:32,874 - INFO - Upload completed successfully: plain_text.cpython-314.pyc
+2026-02-12 01:53:32,874 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc (2796 bytes)
+2026-02-12 01:53:32,885 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,886 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,887 - DEBUG - Progress: 2796/2796 bytes
+2026-02-12 01:53:32,887 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,888 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/flush_stdout.cpython-314.pyc')
+2026-02-12 01:53:32,890 - INFO - Upload completed successfully: flush_stdout.cpython-314.pyc
+2026-02-12 01:53:32,890 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc (3160 bytes)
+2026-02-12 01:53:32,892 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,893 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,893 - DEBUG - Progress: 3160/3160 bytes
+2026-02-12 01:53:32,893 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,895 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/defaults.cpython-314.pyc')
+2026-02-12 01:53:32,896 - INFO - Upload completed successfully: defaults.cpython-314.pyc
+2026-02-12 01:53:32,897 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc (3127 bytes)
+2026-02-12 01:53:32,898 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,899 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,899 - DEBUG - Progress: 3127/3127 bytes
+2026-02-12 01:53:32,899 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,900 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/conemu.cpython-314.pyc')
+2026-02-12 01:53:32,902 - INFO - Upload completed successfully: conemu.cpython-314.pyc
+2026-02-12 01:53:32,902 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc (2357 bytes)
+2026-02-12 01:53:32,904 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,904 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,905 - DEBUG - Progress: 2357/2357 bytes
+2026-02-12 01:53:32,905 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,906 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/color_depth.cpython-314.pyc')
+2026-02-12 01:53:32,907 - INFO - Upload completed successfully: color_depth.cpython-314.pyc
+2026-02-12 01:53:32,907 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc (23669 bytes)
+2026-02-12 01:53:32,909 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,910 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,910 - DEBUG - Progress: 23669/23669 bytes
+2026-02-12 01:53:32,910 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,913 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:32,915 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:32,915 - INFO - Starting upload: lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc (415 bytes)
+2026-02-12 01:53:32,916 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,917 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,917 - DEBUG - Progress: 415/415 bytes
+2026-02-12 01:53:32,917 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,918 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/output/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,919 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,919 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/lexers', 511)
+2026-02-12 01:53:32,920 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/lexers
+2026-02-12 01:53:32,920 - INFO - Starting upload: lib/prompt_toolkit/lexers/pygments.py -> /home/kevin/test/lib/prompt_toolkit/lexers/pygments.py (11922 bytes)
+2026-02-12 01:53:32,921 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/pygments.py', 'wb')
+2026-02-12 01:53:32,922 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/pygments.py', 'wb') -> 00000000
+2026-02-12 01:53:32,922 - DEBUG - Progress: 11922/11922 bytes
+2026-02-12 01:53:32,922 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,925 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/lexers/pygments.py')
+2026-02-12 01:53:32,926 - INFO - Upload completed successfully: pygments.py
+2026-02-12 01:53:32,926 - INFO - Starting upload: lib/prompt_toolkit/lexers/base.py -> /home/kevin/test/lib/prompt_toolkit/lexers/base.py (2350 bytes)
+2026-02-12 01:53:32,928 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/base.py', 'wb')
+2026-02-12 01:53:32,929 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/base.py', 'wb') -> 00000000
+2026-02-12 01:53:32,929 - DEBUG - Progress: 2350/2350 bytes
+2026-02-12 01:53:32,929 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,930 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/lexers/base.py')
+2026-02-12 01:53:32,931 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:32,931 - INFO - Starting upload: lib/prompt_toolkit/lexers/__init__.py -> /home/kevin/test/lib/prompt_toolkit/lexers/__init__.py (409 bytes)
+2026-02-12 01:53:32,933 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__init__.py', 'wb')
+2026-02-12 01:53:32,933 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:32,934 - DEBUG - Progress: 409/409 bytes
+2026-02-12 01:53:32,934 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,934 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/lexers/__init__.py')
+2026-02-12 01:53:32,936 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:32,936 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__', 511)
+2026-02-12 01:53:32,936 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/lexers/__pycache__
+2026-02-12 01:53:32,936 - INFO - Starting upload: lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc (14000 bytes)
+2026-02-12 01:53:32,938 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,938 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,939 - DEBUG - Progress: 14000/14000 bytes
+2026-02-12 01:53:32,939 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,941 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/pygments.cpython-314.pyc')
+2026-02-12 01:53:32,943 - INFO - Upload completed successfully: pygments.cpython-314.pyc
+2026-02-12 01:53:32,943 - INFO - Starting upload: lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc (5108 bytes)
+2026-02-12 01:53:32,944 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,945 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,945 - DEBUG - Progress: 5108/5108 bytes
+2026-02-12 01:53:32,945 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,946 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:32,948 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:32,948 - INFO - Starting upload: lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc (561 bytes)
+2026-02-12 01:53:32,949 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:32,950 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:32,950 - DEBUG - Progress: 561/561 bytes
+2026-02-12 01:53:32,950 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,951 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/lexers/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:32,952 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:32,953 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/layout', 511)
+2026-02-12 01:53:32,953 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/layout
+2026-02-12 01:53:32,953 - INFO - Starting upload: lib/prompt_toolkit/layout/utils.py -> /home/kevin/test/lib/prompt_toolkit/layout/utils.py (2371 bytes)
+2026-02-12 01:53:32,955 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/utils.py', 'wb')
+2026-02-12 01:53:32,955 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:32,955 - DEBUG - Progress: 2371/2371 bytes
+2026-02-12 01:53:32,955 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,957 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/utils.py')
+2026-02-12 01:53:32,958 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:32,958 - INFO - Starting upload: lib/prompt_toolkit/layout/scrollable_pane.py -> /home/kevin/test/lib/prompt_toolkit/layout/scrollable_pane.py (19264 bytes)
+2026-02-12 01:53:32,960 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/scrollable_pane.py', 'wb')
+2026-02-12 01:53:32,960 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/scrollable_pane.py', 'wb') -> 00000000
+2026-02-12 01:53:32,961 - DEBUG - Progress: 19264/19264 bytes
+2026-02-12 01:53:32,961 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,963 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/scrollable_pane.py')
+2026-02-12 01:53:32,965 - INFO - Upload completed successfully: scrollable_pane.py
+2026-02-12 01:53:32,965 - INFO - Starting upload: lib/prompt_toolkit/layout/screen.py -> /home/kevin/test/lib/prompt_toolkit/layout/screen.py (10113 bytes)
+2026-02-12 01:53:32,967 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/screen.py', 'wb')
+2026-02-12 01:53:32,967 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/screen.py', 'wb') -> 00000000
+2026-02-12 01:53:32,973 - DEBUG - Progress: 10113/10113 bytes
+2026-02-12 01:53:32,973 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,975 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/screen.py')
+2026-02-12 01:53:32,976 - INFO - Upload completed successfully: screen.py
+2026-02-12 01:53:32,977 - INFO - Starting upload: lib/prompt_toolkit/layout/processors.py -> /home/kevin/test/lib/prompt_toolkit/layout/processors.py (34296 bytes)
+2026-02-12 01:53:32,978 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/processors.py', 'wb')
+2026-02-12 01:53:32,979 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/processors.py', 'wb') -> 00000000
+2026-02-12 01:53:32,979 - DEBUG - Progress: 32768/34296 bytes
+2026-02-12 01:53:32,979 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,983 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/processors.py')
+2026-02-12 01:53:32,985 - INFO - Upload completed successfully: processors.py
+2026-02-12 01:53:32,986 - INFO - Starting upload: lib/prompt_toolkit/layout/mouse_handlers.py -> /home/kevin/test/lib/prompt_toolkit/layout/mouse_handlers.py (1589 bytes)
+2026-02-12 01:53:32,988 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/mouse_handlers.py', 'wb')
+2026-02-12 01:53:32,988 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/mouse_handlers.py', 'wb') -> 00000000
+2026-02-12 01:53:32,988 - DEBUG - Progress: 1589/1589 bytes
+2026-02-12 01:53:32,988 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,989 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/mouse_handlers.py')
+2026-02-12 01:53:32,992 - INFO - Upload completed successfully: mouse_handlers.py
+2026-02-12 01:53:32,992 - INFO - Starting upload: lib/prompt_toolkit/layout/menus.py -> /home/kevin/test/lib/prompt_toolkit/layout/menus.py (27195 bytes)
+2026-02-12 01:53:32,994 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/menus.py', 'wb')
+2026-02-12 01:53:32,995 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/menus.py', 'wb') -> 00000000
+2026-02-12 01:53:32,995 - DEBUG - Progress: 27195/27195 bytes
+2026-02-12 01:53:32,995 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:32,999 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/menus.py')
+2026-02-12 01:53:33,000 - INFO - Upload completed successfully: menus.py
+2026-02-12 01:53:33,000 - INFO - Starting upload: lib/prompt_toolkit/layout/margins.py -> /home/kevin/test/lib/prompt_toolkit/layout/margins.py (10375 bytes)
+2026-02-12 01:53:33,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/margins.py', 'wb')
+2026-02-12 01:53:33,003 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/margins.py', 'wb') -> 00000000
+2026-02-12 01:53:33,004 - DEBUG - Progress: 10375/10375 bytes
+2026-02-12 01:53:33,004 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,005 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/margins.py')
+2026-02-12 01:53:33,008 - INFO - Upload completed successfully: margins.py
+2026-02-12 01:53:33,008 - INFO - Starting upload: lib/prompt_toolkit/layout/layout.py -> /home/kevin/test/lib/prompt_toolkit/layout/layout.py (13960 bytes)
+2026-02-12 01:53:33,010 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/layout.py', 'wb')
+2026-02-12 01:53:33,011 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/layout.py', 'wb') -> 00000000
+2026-02-12 01:53:33,011 - DEBUG - Progress: 13960/13960 bytes
+2026-02-12 01:53:33,011 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,013 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/layout.py')
+2026-02-12 01:53:33,015 - INFO - Upload completed successfully: layout.py
+2026-02-12 01:53:33,016 - INFO - Starting upload: lib/prompt_toolkit/layout/dummy.py -> /home/kevin/test/lib/prompt_toolkit/layout/dummy.py (1047 bytes)
+2026-02-12 01:53:33,017 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/dummy.py', 'wb')
+2026-02-12 01:53:33,018 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/dummy.py', 'wb') -> 00000000
+2026-02-12 01:53:33,018 - DEBUG - Progress: 1047/1047 bytes
+2026-02-12 01:53:33,018 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,019 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/dummy.py')
+2026-02-12 01:53:33,022 - INFO - Upload completed successfully: dummy.py
+2026-02-12 01:53:33,022 - INFO - Starting upload: lib/prompt_toolkit/layout/dimension.py -> /home/kevin/test/lib/prompt_toolkit/layout/dimension.py (6948 bytes)
+2026-02-12 01:53:33,023 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/dimension.py', 'wb')
+2026-02-12 01:53:33,024 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/dimension.py', 'wb') -> 00000000
+2026-02-12 01:53:33,024 - DEBUG - Progress: 6948/6948 bytes
+2026-02-12 01:53:33,024 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,026 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/dimension.py')
+2026-02-12 01:53:33,027 - INFO - Upload completed successfully: dimension.py
+2026-02-12 01:53:33,027 - INFO - Starting upload: lib/prompt_toolkit/layout/controls.py -> /home/kevin/test/lib/prompt_toolkit/layout/controls.py (35993 bytes)
+2026-02-12 01:53:33,029 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/controls.py', 'wb')
+2026-02-12 01:53:33,029 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/controls.py', 'wb') -> 00000000
+2026-02-12 01:53:33,030 - DEBUG - Progress: 32768/35993 bytes
+2026-02-12 01:53:33,030 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,034 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/controls.py')
+2026-02-12 01:53:33,036 - INFO - Upload completed successfully: controls.py
+2026-02-12 01:53:33,036 - INFO - Starting upload: lib/prompt_toolkit/layout/containers.py -> /home/kevin/test/lib/prompt_toolkit/layout/containers.py (100179 bytes)
+2026-02-12 01:53:33,037 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/containers.py', 'wb')
+2026-02-12 01:53:33,038 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/containers.py', 'wb') -> 00000000
+2026-02-12 01:53:33,039 - DEBUG - Progress: 32768/100179 bytes
+2026-02-12 01:53:33,039 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,050 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/containers.py')
+2026-02-12 01:53:33,053 - INFO - Upload completed successfully: containers.py
+2026-02-12 01:53:33,053 - INFO - Starting upload: lib/prompt_toolkit/layout/__init__.py -> /home/kevin/test/lib/prompt_toolkit/layout/__init__.py (3603 bytes)
+2026-02-12 01:53:33,055 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__init__.py', 'wb')
+2026-02-12 01:53:33,056 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,056 - DEBUG - Progress: 3603/3603 bytes
+2026-02-12 01:53:33,056 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,057 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__init__.py')
+2026-02-12 01:53:33,059 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,060 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__', 511)
+2026-02-12 01:53:33,060 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/layout/__pycache__
+2026-02-12 01:53:33,060 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc (4560 bytes)
+2026-02-12 01:53:33,063 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,063 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,064 - DEBUG - Progress: 4560/4560 bytes
+2026-02-12 01:53:33,064 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,065 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:33,067 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:33,067 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc (22432 bytes)
+2026-02-12 01:53:33,069 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,070 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,070 - DEBUG - Progress: 22432/22432 bytes
+2026-02-12 01:53:33,070 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,073 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/scrollable_pane.cpython-314.pyc')
+2026-02-12 01:53:33,076 - INFO - Upload completed successfully: scrollable_pane.cpython-314.pyc
+2026-02-12 01:53:33,076 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc (14228 bytes)
+2026-02-12 01:53:33,078 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,078 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,079 - DEBUG - Progress: 14228/14228 bytes
+2026-02-12 01:53:33,079 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,082 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/screen.cpython-314.pyc')
+2026-02-12 01:53:33,085 - INFO - Upload completed successfully: screen.cpython-314.pyc
+2026-02-12 01:53:33,085 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc (47148 bytes)
+2026-02-12 01:53:33,086 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,087 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,088 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,093 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/processors.cpython-314.pyc')
+2026-02-12 01:53:33,095 - INFO - Upload completed successfully: processors.cpython-314.pyc
+2026-02-12 01:53:33,096 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc (2773 bytes)
+2026-02-12 01:53:33,097 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,098 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,098 - DEBUG - Progress: 2773/2773 bytes
+2026-02-12 01:53:33,098 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,099 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/mouse_handlers.cpython-314.pyc')
+2026-02-12 01:53:33,102 - INFO - Upload completed successfully: mouse_handlers.cpython-314.pyc
+2026-02-12 01:53:33,102 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc (37272 bytes)
+2026-02-12 01:53:33,104 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,104 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,105 - DEBUG - Progress: 32768/37272 bytes
+2026-02-12 01:53:33,105 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,109 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/menus.cpython-314.pyc')
+2026-02-12 01:53:33,111 - INFO - Upload completed successfully: menus.cpython-314.pyc
+2026-02-12 01:53:33,111 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc (13851 bytes)
+2026-02-12 01:53:33,113 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,114 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,114 - DEBUG - Progress: 13851/13851 bytes
+2026-02-12 01:53:33,114 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,118 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/margins.cpython-314.pyc')
+2026-02-12 01:53:33,120 - INFO - Upload completed successfully: margins.cpython-314.pyc
+2026-02-12 01:53:33,121 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc (19987 bytes)
+2026-02-12 01:53:33,123 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,123 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,124 - DEBUG - Progress: 19987/19987 bytes
+2026-02-12 01:53:33,124 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,126 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/layout.cpython-314.pyc')
+2026-02-12 01:53:33,129 - INFO - Upload completed successfully: layout.cpython-314.pyc
+2026-02-12 01:53:33,129 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc (2003 bytes)
+2026-02-12 01:53:33,131 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,131 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,132 - DEBUG - Progress: 2003/2003 bytes
+2026-02-12 01:53:33,132 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,133 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dummy.cpython-314.pyc')
+2026-02-12 01:53:33,135 - INFO - Upload completed successfully: dummy.cpython-314.pyc
+2026-02-12 01:53:33,135 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc (9500 bytes)
+2026-02-12 01:53:33,137 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,138 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,138 - DEBUG - Progress: 9500/9500 bytes
+2026-02-12 01:53:33,138 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,140 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/dimension.cpython-314.pyc')
+2026-02-12 01:53:33,142 - INFO - Upload completed successfully: dimension.cpython-314.pyc
+2026-02-12 01:53:33,142 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc (44150 bytes)
+2026-02-12 01:53:33,145 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,145 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,146 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,151 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/controls.cpython-314.pyc')
+2026-02-12 01:53:33,153 - INFO - Upload completed successfully: controls.cpython-314.pyc
+2026-02-12 01:53:33,153 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc (119326 bytes)
+2026-02-12 01:53:33,155 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,156 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,157 - DEBUG - Progress: 32768/119326 bytes
+2026-02-12 01:53:33,157 - DEBUG - Progress: 65536/119326 bytes
+2026-02-12 01:53:33,157 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,172 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/containers.cpython-314.pyc')
+2026-02-12 01:53:33,175 - INFO - Upload completed successfully: containers.cpython-314.pyc
+2026-02-12 01:53:33,175 - INFO - Starting upload: lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc (3956 bytes)
+2026-02-12 01:53:33,177 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,178 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,178 - DEBUG - Progress: 3956/3956 bytes
+2026-02-12 01:53:33,178 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,179 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/layout/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,184 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,184 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/key_binding', 511)
+2026-02-12 01:53:33,185 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/key_binding
+2026-02-12 01:53:33,185 - INFO - Starting upload: lib/prompt_toolkit/key_binding/vi_state.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/vi_state.py (3337 bytes)
+2026-02-12 01:53:33,187 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/vi_state.py', 'wb')
+2026-02-12 01:53:33,188 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/vi_state.py', 'wb') -> 00000000
+2026-02-12 01:53:33,189 - DEBUG - Progress: 3337/3337 bytes
+2026-02-12 01:53:33,189 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,190 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/vi_state.py')
+2026-02-12 01:53:33,193 - INFO - Upload completed successfully: vi_state.py
+2026-02-12 01:53:33,193 - INFO - Starting upload: lib/prompt_toolkit/key_binding/key_processor.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/key_processor.py (17555 bytes)
+2026-02-12 01:53:33,196 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/key_processor.py', 'wb')
+2026-02-12 01:53:33,197 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/key_processor.py', 'wb') -> 00000000
+2026-02-12 01:53:33,200 - DEBUG - Progress: 17555/17555 bytes
+2026-02-12 01:53:33,200 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,202 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/key_processor.py')
+2026-02-12 01:53:33,206 - INFO - Upload completed successfully: key_processor.py
+2026-02-12 01:53:33,206 - INFO - Starting upload: lib/prompt_toolkit/key_binding/key_bindings.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/key_bindings.py (20933 bytes)
+2026-02-12 01:53:33,209 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/key_bindings.py', 'wb')
+2026-02-12 01:53:33,209 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/key_bindings.py', 'wb') -> 00000000
+2026-02-12 01:53:33,210 - DEBUG - Progress: 20933/20933 bytes
+2026-02-12 01:53:33,210 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,213 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/key_bindings.py')
+2026-02-12 01:53:33,215 - INFO - Upload completed successfully: key_bindings.py
+2026-02-12 01:53:33,215 - INFO - Starting upload: lib/prompt_toolkit/key_binding/emacs_state.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/emacs_state.py (884 bytes)
+2026-02-12 01:53:33,217 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/emacs_state.py', 'wb')
+2026-02-12 01:53:33,218 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/emacs_state.py', 'wb') -> 00000000
+2026-02-12 01:53:33,218 - DEBUG - Progress: 884/884 bytes
+2026-02-12 01:53:33,218 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,219 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/emacs_state.py')
+2026-02-12 01:53:33,221 - INFO - Upload completed successfully: emacs_state.py
+2026-02-12 01:53:33,221 - INFO - Starting upload: lib/prompt_toolkit/key_binding/digraphs.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/digraphs.py (32785 bytes)
+2026-02-12 01:53:33,223 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/digraphs.py', 'wb')
+2026-02-12 01:53:33,224 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/digraphs.py', 'wb') -> 00000000
+2026-02-12 01:53:33,225 - DEBUG - Progress: 32768/32785 bytes
+2026-02-12 01:53:33,225 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,228 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/digraphs.py')
+2026-02-12 01:53:33,230 - INFO - Upload completed successfully: digraphs.py
+2026-02-12 01:53:33,230 - INFO - Starting upload: lib/prompt_toolkit/key_binding/defaults.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/defaults.py (1975 bytes)
+2026-02-12 01:53:33,232 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/defaults.py', 'wb')
+2026-02-12 01:53:33,232 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/defaults.py', 'wb') -> 00000000
+2026-02-12 01:53:33,232 - DEBUG - Progress: 1975/1975 bytes
+2026-02-12 01:53:33,232 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,234 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/defaults.py')
+2026-02-12 01:53:33,236 - INFO - Upload completed successfully: defaults.py
+2026-02-12 01:53:33,236 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__init__.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/__init__.py (447 bytes)
+2026-02-12 01:53:33,237 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__init__.py', 'wb')
+2026-02-12 01:53:33,238 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,238 - DEBUG - Progress: 447/447 bytes
+2026-02-12 01:53:33,238 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,238 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__init__.py')
+2026-02-12 01:53:33,240 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,240 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__', 511)
+2026-02-12 01:53:33,240 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__
+2026-02-12 01:53:33,241 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc (4110 bytes)
+2026-02-12 01:53:33,242 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,243 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,243 - DEBUG - Progress: 4110/4110 bytes
+2026-02-12 01:53:33,243 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,244 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/vi_state.cpython-314.pyc')
+2026-02-12 01:53:33,246 - INFO - Upload completed successfully: vi_state.cpython-314.pyc
+2026-02-12 01:53:33,246 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc (26179 bytes)
+2026-02-12 01:53:33,248 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,249 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,249 - DEBUG - Progress: 26179/26179 bytes
+2026-02-12 01:53:33,249 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,252 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_processor.cpython-314.pyc')
+2026-02-12 01:53:33,254 - INFO - Upload completed successfully: key_processor.cpython-314.pyc
+2026-02-12 01:53:33,254 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc (30675 bytes)
+2026-02-12 01:53:33,255 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,256 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,256 - DEBUG - Progress: 30675/30675 bytes
+2026-02-12 01:53:33,256 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,259 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/key_bindings.cpython-314.pyc')
+2026-02-12 01:53:33,261 - INFO - Upload completed successfully: key_bindings.cpython-314.pyc
+2026-02-12 01:53:33,261 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc (2276 bytes)
+2026-02-12 01:53:33,263 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,264 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,264 - DEBUG - Progress: 2276/2276 bytes
+2026-02-12 01:53:33,264 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,266 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/emacs_state.cpython-314.pyc')
+2026-02-12 01:53:33,268 - INFO - Upload completed successfully: emacs_state.cpython-314.pyc
+2026-02-12 01:53:33,268 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc (55458 bytes)
+2026-02-12 01:53:33,270 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,271 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,271 - DEBUG - Progress: 32768/55458 bytes
+2026-02-12 01:53:33,271 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,278 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/digraphs.cpython-314.pyc')
+2026-02-12 01:53:33,280 - INFO - Upload completed successfully: digraphs.cpython-314.pyc
+2026-02-12 01:53:33,280 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc (1864 bytes)
+2026-02-12 01:53:33,281 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,282 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,282 - DEBUG - Progress: 1864/1864 bytes
+2026-02-12 01:53:33,282 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,283 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/defaults.cpython-314.pyc')
+2026-02-12 01:53:33,285 - INFO - Upload completed successfully: defaults.cpython-314.pyc
+2026-02-12 01:53:33,286 - INFO - Starting upload: lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc (529 bytes)
+2026-02-12 01:53:33,288 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,288 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,289 - DEBUG - Progress: 529/529 bytes
+2026-02-12 01:53:33,289 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,289 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,291 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,291 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings', 511)
+2026-02-12 01:53:33,292 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/key_binding/bindings
+2026-02-12 01:53:33,292 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/vi.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/vi.py (75602 bytes)
+2026-02-12 01:53:33,294 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/vi.py', 'wb')
+2026-02-12 01:53:33,295 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/vi.py', 'wb') -> 00000000
+2026-02-12 01:53:33,296 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,304 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/vi.py')
+2026-02-12 01:53:33,306 - INFO - Upload completed successfully: vi.py
+2026-02-12 01:53:33,306 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/search.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/search.py (2632 bytes)
+2026-02-12 01:53:33,308 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/search.py', 'wb')
+2026-02-12 01:53:33,309 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/search.py', 'wb') -> 00000000
+2026-02-12 01:53:33,309 - DEBUG - Progress: 2632/2632 bytes
+2026-02-12 01:53:33,309 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,311 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/search.py')
+2026-02-12 01:53:33,313 - INFO - Upload completed successfully: search.py
+2026-02-12 01:53:33,313 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/scroll.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/scroll.py (5613 bytes)
+2026-02-12 01:53:33,315 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/scroll.py', 'wb')
+2026-02-12 01:53:33,316 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/scroll.py', 'wb') -> 00000000
+2026-02-12 01:53:33,317 - DEBUG - Progress: 5613/5613 bytes
+2026-02-12 01:53:33,317 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,318 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/scroll.py')
+2026-02-12 01:53:33,320 - INFO - Upload completed successfully: scroll.py
+2026-02-12 01:53:33,320 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/page_navigation.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/page_navigation.py (2392 bytes)
+2026-02-12 01:53:33,322 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/page_navigation.py', 'wb')
+2026-02-12 01:53:33,323 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/page_navigation.py', 'wb') -> 00000000
+2026-02-12 01:53:33,323 - DEBUG - Progress: 2392/2392 bytes
+2026-02-12 01:53:33,323 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,324 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/page_navigation.py')
+2026-02-12 01:53:33,326 - INFO - Upload completed successfully: page_navigation.py
+2026-02-12 01:53:33,327 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/open_in_editor.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py (1356 bytes)
+2026-02-12 01:53:33,329 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py', 'wb')
+2026-02-12 01:53:33,329 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py', 'wb') -> 00000000
+2026-02-12 01:53:33,330 - DEBUG - Progress: 1356/1356 bytes
+2026-02-12 01:53:33,330 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,331 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/open_in_editor.py')
+2026-02-12 01:53:33,333 - INFO - Upload completed successfully: open_in_editor.py
+2026-02-12 01:53:33,333 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/named_commands.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/named_commands.py (18407 bytes)
+2026-02-12 01:53:33,335 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/named_commands.py', 'wb')
+2026-02-12 01:53:33,336 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/named_commands.py', 'wb') -> 00000000
+2026-02-12 01:53:33,336 - DEBUG - Progress: 18407/18407 bytes
+2026-02-12 01:53:33,336 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,339 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/named_commands.py')
+2026-02-12 01:53:33,341 - INFO - Upload completed successfully: named_commands.py
+2026-02-12 01:53:33,341 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/mouse.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/mouse.py (18586 bytes)
+2026-02-12 01:53:33,343 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/mouse.py', 'wb')
+2026-02-12 01:53:33,344 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/mouse.py', 'wb') -> 00000000
+2026-02-12 01:53:33,344 - DEBUG - Progress: 18586/18586 bytes
+2026-02-12 01:53:33,344 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,347 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/mouse.py')
+2026-02-12 01:53:33,349 - INFO - Upload completed successfully: mouse.py
+2026-02-12 01:53:33,349 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/focus.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/focus.py (507 bytes)
+2026-02-12 01:53:33,351 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/focus.py', 'wb')
+2026-02-12 01:53:33,352 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/focus.py', 'wb') -> 00000000
+2026-02-12 01:53:33,352 - DEBUG - Progress: 507/507 bytes
+2026-02-12 01:53:33,352 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,353 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/focus.py')
+2026-02-12 01:53:33,355 - INFO - Upload completed successfully: focus.py
+2026-02-12 01:53:33,355 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/emacs.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/emacs.py (19634 bytes)
+2026-02-12 01:53:33,357 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/emacs.py', 'wb')
+2026-02-12 01:53:33,358 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/emacs.py', 'wb') -> 00000000
+2026-02-12 01:53:33,358 - DEBUG - Progress: 19634/19634 bytes
+2026-02-12 01:53:33,358 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,361 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/emacs.py')
+2026-02-12 01:53:33,363 - INFO - Upload completed successfully: emacs.py
+2026-02-12 01:53:33,363 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/cpr.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/cpr.py (786 bytes)
+2026-02-12 01:53:33,365 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/cpr.py', 'wb')
+2026-02-12 01:53:33,365 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/cpr.py', 'wb') -> 00000000
+2026-02-12 01:53:33,366 - DEBUG - Progress: 786/786 bytes
+2026-02-12 01:53:33,366 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,366 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/cpr.py')
+2026-02-12 01:53:33,368 - INFO - Upload completed successfully: cpr.py
+2026-02-12 01:53:33,368 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/completion.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/completion.py (6903 bytes)
+2026-02-12 01:53:33,370 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/completion.py', 'wb')
+2026-02-12 01:53:33,371 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/completion.py', 'wb') -> 00000000
+2026-02-12 01:53:33,372 - DEBUG - Progress: 6903/6903 bytes
+2026-02-12 01:53:33,372 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,373 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/completion.py')
+2026-02-12 01:53:33,375 - INFO - Upload completed successfully: completion.py
+2026-02-12 01:53:33,375 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/basic.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/basic.py (7229 bytes)
+2026-02-12 01:53:33,377 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/basic.py', 'wb')
+2026-02-12 01:53:33,378 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/basic.py', 'wb') -> 00000000
+2026-02-12 01:53:33,378 - DEBUG - Progress: 7229/7229 bytes
+2026-02-12 01:53:33,378 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,380 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/basic.py')
+2026-02-12 01:53:33,382 - INFO - Upload completed successfully: basic.py
+2026-02-12 01:53:33,382 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/auto_suggest.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py (1855 bytes)
+2026-02-12 01:53:33,383 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py', 'wb')
+2026-02-12 01:53:33,384 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py', 'wb') -> 00000000
+2026-02-12 01:53:33,384 - DEBUG - Progress: 1855/1855 bytes
+2026-02-12 01:53:33,384 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,385 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/auto_suggest.py')
+2026-02-12 01:53:33,387 - INFO - Upload completed successfully: auto_suggest.py
+2026-02-12 01:53:33,387 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__init__.py -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__init__.py (0 bytes)
+2026-02-12 01:53:33,388 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__init__.py', 'wb')
+2026-02-12 01:53:33,389 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,389 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,389 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__init__.py')
+2026-02-12 01:53:33,391 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,391 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__', 511)
+2026-02-12 01:53:33,391 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__
+2026-02-12 01:53:33,391 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc (126150 bytes)
+2026-02-12 01:53:33,393 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,394 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,394 - DEBUG - Progress: 32768/126150 bytes
+2026-02-12 01:53:33,394 - DEBUG - Progress: 65536/126150 bytes
+2026-02-12 01:53:33,394 - DEBUG - Progress: 98304/126150 bytes
+2026-02-12 01:53:33,395 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,409 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/vi.cpython-314.pyc')
+2026-02-12 01:53:33,411 - INFO - Upload completed successfully: vi.cpython-314.pyc
+2026-02-12 01:53:33,411 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc (5118 bytes)
+2026-02-12 01:53:33,413 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,413 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,413 - DEBUG - Progress: 5118/5118 bytes
+2026-02-12 01:53:33,414 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,415 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/search.cpython-314.pyc')
+2026-02-12 01:53:33,417 - INFO - Upload completed successfully: search.cpython-314.pyc
+2026-02-12 01:53:33,417 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc (8532 bytes)
+2026-02-12 01:53:33,419 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,420 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,420 - DEBUG - Progress: 8532/8532 bytes
+2026-02-12 01:53:33,420 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,421 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/scroll.cpython-314.pyc')
+2026-02-12 01:53:33,424 - INFO - Upload completed successfully: scroll.cpython-314.pyc
+2026-02-12 01:53:33,424 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc (3062 bytes)
+2026-02-12 01:53:33,426 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,426 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,426 - DEBUG - Progress: 3062/3062 bytes
+2026-02-12 01:53:33,427 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,428 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/page_navigation.cpython-314.pyc')
+2026-02-12 01:53:33,430 - INFO - Upload completed successfully: page_navigation.cpython-314.pyc
+2026-02-12 01:53:33,430 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc (2154 bytes)
+2026-02-12 01:53:33,432 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,433 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,433 - DEBUG - Progress: 2154/2154 bytes
+2026-02-12 01:53:33,433 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,434 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/open_in_editor.cpython-314.pyc')
+2026-02-12 01:53:33,436 - INFO - Upload completed successfully: open_in_editor.cpython-314.pyc
+2026-02-12 01:53:33,437 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc (36371 bytes)
+2026-02-12 01:53:33,439 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,439 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,440 - DEBUG - Progress: 32768/36371 bytes
+2026-02-12 01:53:33,440 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,445 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/named_commands.cpython-314.pyc')
+2026-02-12 01:53:33,448 - INFO - Upload completed successfully: named_commands.cpython-314.pyc
+2026-02-12 01:53:33,448 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc (13064 bytes)
+2026-02-12 01:53:33,450 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,450 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,450 - DEBUG - Progress: 13064/13064 bytes
+2026-02-12 01:53:33,450 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,454 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/mouse.cpython-314.pyc')
+2026-02-12 01:53:33,456 - INFO - Upload completed successfully: mouse.cpython-314.pyc
+2026-02-12 01:53:33,456 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc (1262 bytes)
+2026-02-12 01:53:33,458 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,458 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,458 - DEBUG - Progress: 1262/1262 bytes
+2026-02-12 01:53:33,458 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,459 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/focus.cpython-314.pyc')
+2026-02-12 01:53:33,461 - INFO - Upload completed successfully: focus.cpython-314.pyc
+2026-02-12 01:53:33,461 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc (32709 bytes)
+2026-02-12 01:53:33,463 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,464 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,464 - DEBUG - Progress: 32709/32709 bytes
+2026-02-12 01:53:33,464 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,468 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/emacs.cpython-314.pyc')
+2026-02-12 01:53:33,469 - INFO - Upload completed successfully: emacs.cpython-314.pyc
+2026-02-12 01:53:33,469 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc (1719 bytes)
+2026-02-12 01:53:33,471 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,472 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,472 - DEBUG - Progress: 1719/1719 bytes
+2026-02-12 01:53:33,472 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,473 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/cpr.cpython-314.pyc')
+2026-02-12 01:53:33,475 - INFO - Upload completed successfully: cpr.cpython-314.pyc
+2026-02-12 01:53:33,475 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc (10991 bytes)
+2026-02-12 01:53:33,477 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,478 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,478 - DEBUG - Progress: 10991/10991 bytes
+2026-02-12 01:53:33,478 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,480 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/completion.cpython-314.pyc')
+2026-02-12 01:53:33,482 - INFO - Upload completed successfully: completion.cpython-314.pyc
+2026-02-12 01:53:33,482 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc (13234 bytes)
+2026-02-12 01:53:33,484 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,485 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,485 - DEBUG - Progress: 13234/13234 bytes
+2026-02-12 01:53:33,485 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,487 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/basic.cpython-314.pyc')
+2026-02-12 01:53:33,488 - INFO - Upload completed successfully: basic.cpython-314.pyc
+2026-02-12 01:53:33,488 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc (3794 bytes)
+2026-02-12 01:53:33,490 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,490 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,490 - DEBUG - Progress: 3794/3794 bytes
+2026-02-12 01:53:33,490 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,492 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/auto_suggest.cpython-314.pyc')
+2026-02-12 01:53:33,493 - INFO - Upload completed successfully: auto_suggest.cpython-314.pyc
+2026-02-12 01:53:33,494 - INFO - Starting upload: lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc (182 bytes)
+2026-02-12 01:53:33,495 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,496 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,496 - DEBUG - Progress: 182/182 bytes
+2026-02-12 01:53:33,496 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,496 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/key_binding/bindings/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,498 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,498 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/input', 511)
+2026-02-12 01:53:33,499 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/input
+2026-02-12 01:53:33,499 - INFO - Starting upload: lib/prompt_toolkit/input/win32_pipe.py -> /home/kevin/test/lib/prompt_toolkit/input/win32_pipe.py (4700 bytes)
+2026-02-12 01:53:33,500 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/win32_pipe.py', 'wb')
+2026-02-12 01:53:33,501 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/win32_pipe.py', 'wb') -> 00000000
+2026-02-12 01:53:33,503 - DEBUG - Progress: 4700/4700 bytes
+2026-02-12 01:53:33,503 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,504 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/win32_pipe.py')
+2026-02-12 01:53:33,505 - INFO - Upload completed successfully: win32_pipe.py
+2026-02-12 01:53:33,505 - INFO - Starting upload: lib/prompt_toolkit/input/win32.py -> /home/kevin/test/lib/prompt_toolkit/input/win32.py (31410 bytes)
+2026-02-12 01:53:33,507 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/win32.py', 'wb')
+2026-02-12 01:53:33,507 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/win32.py', 'wb') -> 00000000
+2026-02-12 01:53:33,508 - DEBUG - Progress: 31410/31410 bytes
+2026-02-12 01:53:33,508 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,512 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/win32.py')
+2026-02-12 01:53:33,514 - INFO - Upload completed successfully: win32.py
+2026-02-12 01:53:33,514 - INFO - Starting upload: lib/prompt_toolkit/input/vt100_parser.py -> /home/kevin/test/lib/prompt_toolkit/input/vt100_parser.py (8407 bytes)
+2026-02-12 01:53:33,516 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/vt100_parser.py', 'wb')
+2026-02-12 01:53:33,516 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/vt100_parser.py', 'wb') -> 00000000
+2026-02-12 01:53:33,517 - DEBUG - Progress: 8407/8407 bytes
+2026-02-12 01:53:33,517 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,519 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/vt100_parser.py')
+2026-02-12 01:53:33,520 - INFO - Upload completed successfully: vt100_parser.py
+2026-02-12 01:53:33,520 - INFO - Starting upload: lib/prompt_toolkit/input/vt100.py -> /home/kevin/test/lib/prompt_toolkit/input/vt100.py (10514 bytes)
+2026-02-12 01:53:33,522 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/vt100.py', 'wb')
+2026-02-12 01:53:33,522 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/vt100.py', 'wb') -> 00000000
+2026-02-12 01:53:33,523 - DEBUG - Progress: 10514/10514 bytes
+2026-02-12 01:53:33,523 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,525 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/vt100.py')
+2026-02-12 01:53:33,526 - INFO - Upload completed successfully: vt100.py
+2026-02-12 01:53:33,526 - INFO - Starting upload: lib/prompt_toolkit/input/typeahead.py -> /home/kevin/test/lib/prompt_toolkit/input/typeahead.py (2545 bytes)
+2026-02-12 01:53:33,528 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/typeahead.py', 'wb')
+2026-02-12 01:53:33,528 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/typeahead.py', 'wb') -> 00000000
+2026-02-12 01:53:33,529 - DEBUG - Progress: 2545/2545 bytes
+2026-02-12 01:53:33,529 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,530 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/typeahead.py')
+2026-02-12 01:53:33,532 - INFO - Upload completed successfully: typeahead.py
+2026-02-12 01:53:33,532 - INFO - Starting upload: lib/prompt_toolkit/input/posix_utils.py -> /home/kevin/test/lib/prompt_toolkit/input/posix_utils.py (3973 bytes)
+2026-02-12 01:53:33,534 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/posix_utils.py', 'wb')
+2026-02-12 01:53:33,534 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/posix_utils.py', 'wb') -> 00000000
+2026-02-12 01:53:33,534 - DEBUG - Progress: 3973/3973 bytes
+2026-02-12 01:53:33,534 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,536 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/posix_utils.py')
+2026-02-12 01:53:33,537 - INFO - Upload completed successfully: posix_utils.py
+2026-02-12 01:53:33,537 - INFO - Starting upload: lib/prompt_toolkit/input/posix_pipe.py -> /home/kevin/test/lib/prompt_toolkit/input/posix_pipe.py (3158 bytes)
+2026-02-12 01:53:33,538 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/posix_pipe.py', 'wb')
+2026-02-12 01:53:33,539 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/posix_pipe.py', 'wb') -> 00000000
+2026-02-12 01:53:33,539 - DEBUG - Progress: 3158/3158 bytes
+2026-02-12 01:53:33,539 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,540 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/posix_pipe.py')
+2026-02-12 01:53:33,542 - INFO - Upload completed successfully: posix_pipe.py
+2026-02-12 01:53:33,542 - INFO - Starting upload: lib/prompt_toolkit/input/defaults.py -> /home/kevin/test/lib/prompt_toolkit/input/defaults.py (2500 bytes)
+2026-02-12 01:53:33,544 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/defaults.py', 'wb')
+2026-02-12 01:53:33,545 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/defaults.py', 'wb') -> 00000000
+2026-02-12 01:53:33,545 - DEBUG - Progress: 2500/2500 bytes
+2026-02-12 01:53:33,545 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,546 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/defaults.py')
+2026-02-12 01:53:33,548 - INFO - Upload completed successfully: defaults.py
+2026-02-12 01:53:33,548 - INFO - Starting upload: lib/prompt_toolkit/input/base.py -> /home/kevin/test/lib/prompt_toolkit/input/base.py (4030 bytes)
+2026-02-12 01:53:33,549 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/base.py', 'wb')
+2026-02-12 01:53:33,550 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/base.py', 'wb') -> 00000000
+2026-02-12 01:53:33,550 - DEBUG - Progress: 4030/4030 bytes
+2026-02-12 01:53:33,550 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,551 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/base.py')
+2026-02-12 01:53:33,553 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:33,553 - INFO - Starting upload: lib/prompt_toolkit/input/ansi_escape_sequences.py -> /home/kevin/test/lib/prompt_toolkit/input/ansi_escape_sequences.py (13663 bytes)
+2026-02-12 01:53:33,554 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/ansi_escape_sequences.py', 'wb')
+2026-02-12 01:53:33,555 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/ansi_escape_sequences.py', 'wb') -> 00000000
+2026-02-12 01:53:33,555 - DEBUG - Progress: 13663/13663 bytes
+2026-02-12 01:53:33,555 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,559 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/ansi_escape_sequences.py')
+2026-02-12 01:53:33,561 - INFO - Upload completed successfully: ansi_escape_sequences.py
+2026-02-12 01:53:33,561 - INFO - Starting upload: lib/prompt_toolkit/input/__init__.py -> /home/kevin/test/lib/prompt_toolkit/input/__init__.py (273 bytes)
+2026-02-12 01:53:33,562 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__init__.py', 'wb')
+2026-02-12 01:53:33,563 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,563 - DEBUG - Progress: 273/273 bytes
+2026-02-12 01:53:33,563 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,564 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__init__.py')
+2026-02-12 01:53:33,565 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,565 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__', 511)
+2026-02-12 01:53:33,566 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/input/__pycache__
+2026-02-12 01:53:33,566 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc (8755 bytes)
+2026-02-12 01:53:33,567 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,568 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,568 - DEBUG - Progress: 8755/8755 bytes
+2026-02-12 01:53:33,568 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,570 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32_pipe.cpython-314.pyc')
+2026-02-12 01:53:33,572 - INFO - Upload completed successfully: win32_pipe.cpython-314.pyc
+2026-02-12 01:53:33,572 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc (41651 bytes)
+2026-02-12 01:53:33,574 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,574 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,575 - DEBUG - Progress: 32768/41651 bytes
+2026-02-12 01:53:33,575 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,580 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/win32.cpython-314.pyc')
+2026-02-12 01:53:33,582 - INFO - Upload completed successfully: win32.cpython-314.pyc
+2026-02-12 01:53:33,582 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc (10696 bytes)
+2026-02-12 01:53:33,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,585 - DEBUG - Progress: 10696/10696 bytes
+2026-02-12 01:53:33,585 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,587 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100_parser.cpython-314.pyc')
+2026-02-12 01:53:33,589 - INFO - Upload completed successfully: vt100_parser.cpython-314.pyc
+2026-02-12 01:53:33,589 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc (15196 bytes)
+2026-02-12 01:53:33,591 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,592 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,592 - DEBUG - Progress: 15196/15196 bytes
+2026-02-12 01:53:33,592 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,594 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/vt100.cpython-314.pyc')
+2026-02-12 01:53:33,596 - INFO - Upload completed successfully: vt100.cpython-314.pyc
+2026-02-12 01:53:33,596 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc (3558 bytes)
+2026-02-12 01:53:33,598 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,599 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,599 - DEBUG - Progress: 3558/3558 bytes
+2026-02-12 01:53:33,599 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,600 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/typeahead.cpython-314.pyc')
+2026-02-12 01:53:33,602 - INFO - Upload completed successfully: typeahead.cpython-314.pyc
+2026-02-12 01:53:33,602 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc (3366 bytes)
+2026-02-12 01:53:33,604 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,605 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,605 - DEBUG - Progress: 3366/3366 bytes
+2026-02-12 01:53:33,605 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,607 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_utils.cpython-314.pyc')
+2026-02-12 01:53:33,609 - INFO - Upload completed successfully: posix_utils.cpython-314.pyc
+2026-02-12 01:53:33,609 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc (7768 bytes)
+2026-02-12 01:53:33,611 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,612 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,612 - DEBUG - Progress: 7768/7768 bytes
+2026-02-12 01:53:33,612 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,614 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/posix_pipe.cpython-314.pyc')
+2026-02-12 01:53:33,616 - INFO - Upload completed successfully: posix_pipe.cpython-314.pyc
+2026-02-12 01:53:33,616 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc (2923 bytes)
+2026-02-12 01:53:33,617 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,618 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,618 - DEBUG - Progress: 2923/2923 bytes
+2026-02-12 01:53:33,618 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,619 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/defaults.cpython-314.pyc')
+2026-02-12 01:53:33,621 - INFO - Upload completed successfully: defaults.cpython-314.pyc
+2026-02-12 01:53:33,621 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc (8716 bytes)
+2026-02-12 01:53:33,623 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,624 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,624 - DEBUG - Progress: 8716/8716 bytes
+2026-02-12 01:53:33,624 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,626 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:33,628 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:33,628 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc (17919 bytes)
+2026-02-12 01:53:33,629 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,630 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,630 - DEBUG - Progress: 17919/17919 bytes
+2026-02-12 01:53:33,630 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,632 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/ansi_escape_sequences.cpython-314.pyc')
+2026-02-12 01:53:33,635 - INFO - Upload completed successfully: ansi_escape_sequences.cpython-314.pyc
+2026-02-12 01:53:33,635 - INFO - Starting upload: lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc (420 bytes)
+2026-02-12 01:53:33,638 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,639 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,640 - DEBUG - Progress: 420/420 bytes
+2026-02-12 01:53:33,640 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,641 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/input/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,643 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,643 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/formatted_text', 511)
+2026-02-12 01:53:33,644 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/formatted_text
+2026-02-12 01:53:33,644 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/utils.py -> /home/kevin/test/lib/prompt_toolkit/formatted_text/utils.py (3044 bytes)
+2026-02-12 01:53:33,646 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/utils.py', 'wb')
+2026-02-12 01:53:33,647 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:33,647 - DEBUG - Progress: 3044/3044 bytes
+2026-02-12 01:53:33,647 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,648 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/utils.py')
+2026-02-12 01:53:33,650 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:33,650 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/pygments.py -> /home/kevin/test/lib/prompt_toolkit/formatted_text/pygments.py (780 bytes)
+2026-02-12 01:53:33,652 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/pygments.py', 'wb')
+2026-02-12 01:53:33,652 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/pygments.py', 'wb') -> 00000000
+2026-02-12 01:53:33,652 - DEBUG - Progress: 780/780 bytes
+2026-02-12 01:53:33,652 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,653 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/pygments.py')
+2026-02-12 01:53:33,655 - INFO - Upload completed successfully: pygments.py
+2026-02-12 01:53:33,655 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/html.py -> /home/kevin/test/lib/prompt_toolkit/formatted_text/html.py (4374 bytes)
+2026-02-12 01:53:33,656 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/html.py', 'wb')
+2026-02-12 01:53:33,657 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/html.py', 'wb') -> 00000000
+2026-02-12 01:53:33,658 - DEBUG - Progress: 4374/4374 bytes
+2026-02-12 01:53:33,658 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,659 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/html.py')
+2026-02-12 01:53:33,661 - INFO - Upload completed successfully: html.py
+2026-02-12 01:53:33,661 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/base.py -> /home/kevin/test/lib/prompt_toolkit/formatted_text/base.py (5162 bytes)
+2026-02-12 01:53:33,662 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/base.py', 'wb')
+2026-02-12 01:53:33,663 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/base.py', 'wb') -> 00000000
+2026-02-12 01:53:33,663 - DEBUG - Progress: 5162/5162 bytes
+2026-02-12 01:53:33,664 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,665 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/base.py')
+2026-02-12 01:53:33,667 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:33,667 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/ansi.py -> /home/kevin/test/lib/prompt_toolkit/formatted_text/ansi.py (9824 bytes)
+2026-02-12 01:53:33,669 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/ansi.py', 'wb')
+2026-02-12 01:53:33,669 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/ansi.py', 'wb') -> 00000000
+2026-02-12 01:53:33,670 - DEBUG - Progress: 9824/9824 bytes
+2026-02-12 01:53:33,670 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,672 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/ansi.py')
+2026-02-12 01:53:33,674 - INFO - Upload completed successfully: ansi.py
+2026-02-12 01:53:33,674 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__init__.py -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__init__.py (1509 bytes)
+2026-02-12 01:53:33,675 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__init__.py', 'wb')
+2026-02-12 01:53:33,676 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,676 - DEBUG - Progress: 1509/1509 bytes
+2026-02-12 01:53:33,676 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,677 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__init__.py')
+2026-02-12 01:53:33,679 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,679 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__', 511)
+2026-02-12 01:53:33,680 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__
+2026-02-12 01:53:33,680 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc (4696 bytes)
+2026-02-12 01:53:33,682 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,683 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,683 - DEBUG - Progress: 4696/4696 bytes
+2026-02-12 01:53:33,683 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,685 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:33,686 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:33,687 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc (1749 bytes)
+2026-02-12 01:53:33,689 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,689 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,690 - DEBUG - Progress: 1749/1749 bytes
+2026-02-12 01:53:33,690 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,691 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/pygments.cpython-314.pyc')
+2026-02-12 01:53:33,693 - INFO - Upload completed successfully: pygments.cpython-314.pyc
+2026-02-12 01:53:33,693 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc (7488 bytes)
+2026-02-12 01:53:33,694 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,695 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,695 - DEBUG - Progress: 7488/7488 bytes
+2026-02-12 01:53:33,695 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,698 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/html.cpython-314.pyc')
+2026-02-12 01:53:33,699 - INFO - Upload completed successfully: html.cpython-314.pyc
+2026-02-12 01:53:33,700 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc (8152 bytes)
+2026-02-12 01:53:33,701 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,702 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,702 - DEBUG - Progress: 8152/8152 bytes
+2026-02-12 01:53:33,702 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,704 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:33,706 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:33,706 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc (11612 bytes)
+2026-02-12 01:53:33,708 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,708 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,708 - DEBUG - Progress: 11612/11612 bytes
+2026-02-12 01:53:33,709 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,710 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/ansi.cpython-314.pyc')
+2026-02-12 01:53:33,712 - INFO - Upload completed successfully: ansi.cpython-314.pyc
+2026-02-12 01:53:33,712 - INFO - Starting upload: lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc (1489 bytes)
+2026-02-12 01:53:33,714 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,714 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,714 - DEBUG - Progress: 1489/1489 bytes
+2026-02-12 01:53:33,715 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,715 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/formatted_text/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,717 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,717 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/filters', 511)
+2026-02-12 01:53:33,718 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/filters
+2026-02-12 01:53:33,718 - INFO - Starting upload: lib/prompt_toolkit/filters/utils.py -> /home/kevin/test/lib/prompt_toolkit/filters/utils.py (859 bytes)
+2026-02-12 01:53:33,720 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/utils.py', 'wb')
+2026-02-12 01:53:33,720 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:33,721 - DEBUG - Progress: 859/859 bytes
+2026-02-12 01:53:33,721 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,721 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/utils.py')
+2026-02-12 01:53:33,723 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:33,723 - INFO - Starting upload: lib/prompt_toolkit/filters/cli.py -> /home/kevin/test/lib/prompt_toolkit/filters/cli.py (1867 bytes)
+2026-02-12 01:53:33,724 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/cli.py', 'wb')
+2026-02-12 01:53:33,725 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/cli.py', 'wb') -> 00000000
+2026-02-12 01:53:33,725 - DEBUG - Progress: 1867/1867 bytes
+2026-02-12 01:53:33,725 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,726 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/cli.py')
+2026-02-12 01:53:33,728 - INFO - Upload completed successfully: cli.py
+2026-02-12 01:53:33,728 - INFO - Starting upload: lib/prompt_toolkit/filters/base.py -> /home/kevin/test/lib/prompt_toolkit/filters/base.py (6855 bytes)
+2026-02-12 01:53:33,730 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/base.py', 'wb')
+2026-02-12 01:53:33,730 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/base.py', 'wb') -> 00000000
+2026-02-12 01:53:33,731 - DEBUG - Progress: 6855/6855 bytes
+2026-02-12 01:53:33,731 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,733 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/base.py')
+2026-02-12 01:53:33,734 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:33,734 - INFO - Starting upload: lib/prompt_toolkit/filters/app.py -> /home/kevin/test/lib/prompt_toolkit/filters/app.py (10374 bytes)
+2026-02-12 01:53:33,736 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/app.py', 'wb')
+2026-02-12 01:53:33,736 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/app.py', 'wb') -> 00000000
+2026-02-12 01:53:33,737 - DEBUG - Progress: 10374/10374 bytes
+2026-02-12 01:53:33,737 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,739 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/app.py')
+2026-02-12 01:53:33,740 - INFO - Upload completed successfully: app.py
+2026-02-12 01:53:33,740 - INFO - Starting upload: lib/prompt_toolkit/filters/__init__.py -> /home/kevin/test/lib/prompt_toolkit/filters/__init__.py (1990 bytes)
+2026-02-12 01:53:33,742 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__init__.py', 'wb')
+2026-02-12 01:53:33,742 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,743 - DEBUG - Progress: 1990/1990 bytes
+2026-02-12 01:53:33,743 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,744 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/__init__.py')
+2026-02-12 01:53:33,746 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,746 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__', 511)
+2026-02-12 01:53:33,747 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/filters/__pycache__
+2026-02-12 01:53:33,747 - INFO - Starting upload: lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc (1652 bytes)
+2026-02-12 01:53:33,748 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,749 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,749 - DEBUG - Progress: 1652/1652 bytes
+2026-02-12 01:53:33,749 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,750 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:33,751 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:33,752 - INFO - Starting upload: lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc (3655 bytes)
+2026-02-12 01:53:33,755 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,756 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,756 - DEBUG - Progress: 3655/3655 bytes
+2026-02-12 01:53:33,756 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,757 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/cli.cpython-314.pyc')
+2026-02-12 01:53:33,760 - INFO - Upload completed successfully: cli.cpython-314.pyc
+2026-02-12 01:53:33,760 - INFO - Starting upload: lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc (14996 bytes)
+2026-02-12 01:53:33,761 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,762 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,762 - DEBUG - Progress: 14996/14996 bytes
+2026-02-12 01:53:33,762 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,766 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:33,768 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:33,768 - INFO - Starting upload: lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc (21184 bytes)
+2026-02-12 01:53:33,770 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,770 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,770 - DEBUG - Progress: 21184/21184 bytes
+2026-02-12 01:53:33,771 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,774 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/app.cpython-314.pyc')
+2026-02-12 01:53:33,775 - INFO - Upload completed successfully: app.cpython-314.pyc
+2026-02-12 01:53:33,775 - INFO - Starting upload: lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc (2520 bytes)
+2026-02-12 01:53:33,778 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,779 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,779 - DEBUG - Progress: 2520/2520 bytes
+2026-02-12 01:53:33,780 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,781 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/filters/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,784 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,784 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/eventloop', 511)
+2026-02-12 01:53:33,784 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/eventloop
+2026-02-12 01:53:33,785 - INFO - Starting upload: lib/prompt_toolkit/eventloop/win32.py -> /home/kevin/test/lib/prompt_toolkit/eventloop/win32.py (2286 bytes)
+2026-02-12 01:53:33,788 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/win32.py', 'wb')
+2026-02-12 01:53:33,789 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/win32.py', 'wb') -> 00000000
+2026-02-12 01:53:33,789 - DEBUG - Progress: 2286/2286 bytes
+2026-02-12 01:53:33,790 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,791 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/win32.py')
+2026-02-12 01:53:33,795 - INFO - Upload completed successfully: win32.py
+2026-02-12 01:53:33,795 - INFO - Starting upload: lib/prompt_toolkit/eventloop/utils.py -> /home/kevin/test/lib/prompt_toolkit/eventloop/utils.py (3200 bytes)
+2026-02-12 01:53:33,798 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/utils.py', 'wb')
+2026-02-12 01:53:33,799 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:33,799 - DEBUG - Progress: 3200/3200 bytes
+2026-02-12 01:53:33,799 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,800 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/utils.py')
+2026-02-12 01:53:33,802 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:33,802 - INFO - Starting upload: lib/prompt_toolkit/eventloop/inputhook.py -> /home/kevin/test/lib/prompt_toolkit/eventloop/inputhook.py (6130 bytes)
+2026-02-12 01:53:33,803 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/inputhook.py', 'wb')
+2026-02-12 01:53:33,804 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/inputhook.py', 'wb') -> 00000000
+2026-02-12 01:53:33,804 - DEBUG - Progress: 6130/6130 bytes
+2026-02-12 01:53:33,805 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,806 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/inputhook.py')
+2026-02-12 01:53:33,807 - INFO - Upload completed successfully: inputhook.py
+2026-02-12 01:53:33,808 - INFO - Starting upload: lib/prompt_toolkit/eventloop/async_generator.py -> /home/kevin/test/lib/prompt_toolkit/eventloop/async_generator.py (3933 bytes)
+2026-02-12 01:53:33,809 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/async_generator.py', 'wb')
+2026-02-12 01:53:33,810 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/async_generator.py', 'wb') -> 00000000
+2026-02-12 01:53:33,810 - DEBUG - Progress: 3933/3933 bytes
+2026-02-12 01:53:33,810 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,811 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/async_generator.py')
+2026-02-12 01:53:33,813 - INFO - Upload completed successfully: async_generator.py
+2026-02-12 01:53:33,813 - INFO - Starting upload: lib/prompt_toolkit/eventloop/__init__.py -> /home/kevin/test/lib/prompt_toolkit/eventloop/__init__.py (730 bytes)
+2026-02-12 01:53:33,814 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__init__.py', 'wb')
+2026-02-12 01:53:33,815 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,815 - DEBUG - Progress: 730/730 bytes
+2026-02-12 01:53:33,815 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,816 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__init__.py')
+2026-02-12 01:53:33,817 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,817 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__', 511)
+2026-02-12 01:53:33,818 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__
+2026-02-12 01:53:33,818 - INFO - Starting upload: lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc (2945 bytes)
+2026-02-12 01:53:33,820 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,820 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,820 - DEBUG - Progress: 2945/2945 bytes
+2026-02-12 01:53:33,820 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,822 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/win32.cpython-314.pyc')
+2026-02-12 01:53:33,824 - INFO - Upload completed successfully: win32.cpython-314.pyc
+2026-02-12 01:53:33,824 - INFO - Starting upload: lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc (4178 bytes)
+2026-02-12 01:53:33,825 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,826 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,826 - DEBUG - Progress: 4178/4178 bytes
+2026-02-12 01:53:33,826 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,827 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:33,829 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:33,829 - INFO - Starting upload: lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc (9443 bytes)
+2026-02-12 01:53:33,830 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,831 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,831 - DEBUG - Progress: 9443/9443 bytes
+2026-02-12 01:53:33,831 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,833 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/inputhook.cpython-314.pyc')
+2026-02-12 01:53:33,835 - INFO - Upload completed successfully: inputhook.cpython-314.pyc
+2026-02-12 01:53:33,835 - INFO - Starting upload: lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc (4847 bytes)
+2026-02-12 01:53:33,837 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,837 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,837 - DEBUG - Progress: 4847/4847 bytes
+2026-02-12 01:53:33,838 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,839 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/async_generator.cpython-314.pyc')
+2026-02-12 01:53:33,841 - INFO - Upload completed successfully: async_generator.cpython-314.pyc
+2026-02-12 01:53:33,841 - INFO - Starting upload: lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc (702 bytes)
+2026-02-12 01:53:33,842 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,843 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,843 - DEBUG - Progress: 702/702 bytes
+2026-02-12 01:53:33,843 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,844 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/eventloop/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,845 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,845 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib', 511)
+2026-02-12 01:53:33,846 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib
+2026-02-12 01:53:33,846 - INFO - Starting upload: lib/prompt_toolkit/contrib/__init__.py -> /home/kevin/test/lib/prompt_toolkit/contrib/__init__.py (0 bytes)
+2026-02-12 01:53:33,848 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/__init__.py', 'wb')
+2026-02-12 01:53:33,848 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,849 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,849 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/__init__.py')
+2026-02-12 01:53:33,850 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,851 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/__pycache__', 511)
+2026-02-12 01:53:33,851 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/__pycache__
+2026-02-12 01:53:33,851 - INFO - Starting upload: lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc (169 bytes)
+2026-02-12 01:53:33,853 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,853 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,853 - DEBUG - Progress: 169/169 bytes
+2026-02-12 01:53:33,853 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,854 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,855 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,855 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet', 511)
+2026-02-12 01:53:33,856 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/telnet
+2026-02-12 01:53:33,856 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/server.py -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/server.py (13461 bytes)
+2026-02-12 01:53:33,858 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/server.py', 'wb')
+2026-02-12 01:53:33,858 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/server.py', 'wb') -> 00000000
+2026-02-12 01:53:33,859 - DEBUG - Progress: 13461/13461 bytes
+2026-02-12 01:53:33,859 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,861 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/server.py')
+2026-02-12 01:53:33,863 - INFO - Upload completed successfully: server.py
+2026-02-12 01:53:33,863 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/protocol.py -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/protocol.py (5584 bytes)
+2026-02-12 01:53:33,864 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/protocol.py', 'wb')
+2026-02-12 01:53:33,865 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/protocol.py', 'wb') -> 00000000
+2026-02-12 01:53:33,865 - DEBUG - Progress: 5584/5584 bytes
+2026-02-12 01:53:33,865 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,866 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/protocol.py')
+2026-02-12 01:53:33,868 - INFO - Upload completed successfully: protocol.py
+2026-02-12 01:53:33,868 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/log.py -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/log.py (167 bytes)
+2026-02-12 01:53:33,870 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/log.py', 'wb')
+2026-02-12 01:53:33,870 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/log.py', 'wb') -> 00000000
+2026-02-12 01:53:33,871 - DEBUG - Progress: 167/167 bytes
+2026-02-12 01:53:33,871 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,871 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/log.py')
+2026-02-12 01:53:33,873 - INFO - Upload completed successfully: log.py
+2026-02-12 01:53:33,873 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/__init__.py -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/__init__.py (104 bytes)
+2026-02-12 01:53:33,876 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__init__.py', 'wb')
+2026-02-12 01:53:33,877 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,877 - DEBUG - Progress: 104/104 bytes
+2026-02-12 01:53:33,877 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,878 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__init__.py')
+2026-02-12 01:53:33,880 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,880 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__', 511)
+2026-02-12 01:53:33,881 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__
+2026-02-12 01:53:33,881 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc (25007 bytes)
+2026-02-12 01:53:33,883 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,883 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,884 - DEBUG - Progress: 25007/25007 bytes
+2026-02-12 01:53:33,884 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,887 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/server.cpython-314.pyc')
+2026-02-12 01:53:33,889 - INFO - Upload completed successfully: server.cpython-314.pyc
+2026-02-12 01:53:33,889 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc (9403 bytes)
+2026-02-12 01:53:33,891 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,892 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,892 - DEBUG - Progress: 9403/9403 bytes
+2026-02-12 01:53:33,892 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,894 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/protocol.cpython-314.pyc')
+2026-02-12 01:53:33,896 - INFO - Upload completed successfully: protocol.cpython-314.pyc
+2026-02-12 01:53:33,896 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc (411 bytes)
+2026-02-12 01:53:33,899 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,899 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,900 - DEBUG - Progress: 411/411 bytes
+2026-02-12 01:53:33,900 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,901 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/log.cpython-314.pyc')
+2026-02-12 01:53:33,903 - INFO - Upload completed successfully: log.cpython-314.pyc
+2026-02-12 01:53:33,903 - INFO - Starting upload: lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc (301 bytes)
+2026-02-12 01:53:33,905 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,906 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,906 - DEBUG - Progress: 301/301 bytes
+2026-02-12 01:53:33,906 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,906 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/telnet/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,908 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,908 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh', 511)
+2026-02-12 01:53:33,909 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/ssh
+2026-02-12 01:53:33,909 - INFO - Starting upload: lib/prompt_toolkit/contrib/ssh/server.py -> /home/kevin/test/lib/prompt_toolkit/contrib/ssh/server.py (6130 bytes)
+2026-02-12 01:53:33,910 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/server.py', 'wb')
+2026-02-12 01:53:33,911 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/server.py', 'wb') -> 00000000
+2026-02-12 01:53:33,911 - DEBUG - Progress: 6130/6130 bytes
+2026-02-12 01:53:33,911 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,912 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/server.py')
+2026-02-12 01:53:33,914 - INFO - Upload completed successfully: server.py
+2026-02-12 01:53:33,914 - INFO - Starting upload: lib/prompt_toolkit/contrib/ssh/__init__.py -> /home/kevin/test/lib/prompt_toolkit/contrib/ssh/__init__.py (180 bytes)
+2026-02-12 01:53:33,916 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__init__.py', 'wb')
+2026-02-12 01:53:33,916 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,917 - DEBUG - Progress: 180/180 bytes
+2026-02-12 01:53:33,917 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,917 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__init__.py')
+2026-02-12 01:53:33,919 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,919 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__', 511)
+2026-02-12 01:53:33,919 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__
+2026-02-12 01:53:33,919 - INFO - Starting upload: lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc (11405 bytes)
+2026-02-12 01:53:33,921 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,921 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,922 - DEBUG - Progress: 11405/11405 bytes
+2026-02-12 01:53:33,922 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,924 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/server.cpython-314.pyc')
+2026-02-12 01:53:33,926 - INFO - Upload completed successfully: server.cpython-314.pyc
+2026-02-12 01:53:33,926 - INFO - Starting upload: lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc (352 bytes)
+2026-02-12 01:53:33,928 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,928 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,928 - DEBUG - Progress: 352/352 bytes
+2026-02-12 01:53:33,929 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,929 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/ssh/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:33,931 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:33,931 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages', 511)
+2026-02-12 01:53:33,932 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages
+2026-02-12 01:53:33,932 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/validation.py -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/validation.py (2059 bytes)
+2026-02-12 01:53:33,933 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/validation.py', 'wb')
+2026-02-12 01:53:33,934 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/validation.py', 'wb') -> 00000000
+2026-02-12 01:53:33,934 - DEBUG - Progress: 2059/2059 bytes
+2026-02-12 01:53:33,934 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,935 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/validation.py')
+2026-02-12 01:53:33,937 - INFO - Upload completed successfully: validation.py
+2026-02-12 01:53:33,937 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/regex_parser.py -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py (7732 bytes)
+2026-02-12 01:53:33,938 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py', 'wb')
+2026-02-12 01:53:33,939 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py', 'wb') -> 00000000
+2026-02-12 01:53:33,939 - DEBUG - Progress: 7732/7732 bytes
+2026-02-12 01:53:33,939 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,941 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/regex_parser.py')
+2026-02-12 01:53:33,943 - INFO - Upload completed successfully: regex_parser.py
+2026-02-12 01:53:33,943 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/lexer.py -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/lexer.py (3415 bytes)
+2026-02-12 01:53:33,944 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/lexer.py', 'wb')
+2026-02-12 01:53:33,945 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/lexer.py', 'wb') -> 00000000
+2026-02-12 01:53:33,945 - DEBUG - Progress: 3415/3415 bytes
+2026-02-12 01:53:33,945 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,946 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/lexer.py')
+2026-02-12 01:53:33,948 - INFO - Upload completed successfully: lexer.py
+2026-02-12 01:53:33,948 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/completion.py -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/completion.py (3468 bytes)
+2026-02-12 01:53:33,949 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/completion.py', 'wb')
+2026-02-12 01:53:33,950 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/completion.py', 'wb') -> 00000000
+2026-02-12 01:53:33,950 - DEBUG - Progress: 3468/3468 bytes
+2026-02-12 01:53:33,950 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,951 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/completion.py')
+2026-02-12 01:53:33,953 - INFO - Upload completed successfully: completion.py
+2026-02-12 01:53:33,953 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/compiler.py -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/compiler.py (21948 bytes)
+2026-02-12 01:53:33,955 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/compiler.py', 'wb')
+2026-02-12 01:53:33,956 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/compiler.py', 'wb') -> 00000000
+2026-02-12 01:53:33,956 - DEBUG - Progress: 21948/21948 bytes
+2026-02-12 01:53:33,956 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,959 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/compiler.py')
+2026-02-12 01:53:33,961 - INFO - Upload completed successfully: compiler.py
+2026-02-12 01:53:33,961 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__init__.py -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__init__.py (3279 bytes)
+2026-02-12 01:53:33,962 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__init__.py', 'wb')
+2026-02-12 01:53:33,963 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:33,963 - DEBUG - Progress: 3279/3279 bytes
+2026-02-12 01:53:33,963 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,964 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__init__.py')
+2026-02-12 01:53:33,966 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:33,966 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__', 511)
+2026-02-12 01:53:33,967 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__
+2026-02-12 01:53:33,967 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc (2917 bytes)
+2026-02-12 01:53:33,969 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,969 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,970 - DEBUG - Progress: 2917/2917 bytes
+2026-02-12 01:53:33,970 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,971 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/validation.cpython-314.pyc')
+2026-02-12 01:53:33,973 - INFO - Upload completed successfully: validation.cpython-314.pyc
+2026-02-12 01:53:33,973 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc (14535 bytes)
+2026-02-12 01:53:33,974 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,975 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,975 - DEBUG - Progress: 14535/14535 bytes
+2026-02-12 01:53:33,976 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,979 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/regex_parser.cpython-314.pyc')
+2026-02-12 01:53:33,981 - INFO - Upload completed successfully: regex_parser.cpython-314.pyc
+2026-02-12 01:53:33,981 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc (5079 bytes)
+2026-02-12 01:53:33,983 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,983 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,984 - DEBUG - Progress: 5079/5079 bytes
+2026-02-12 01:53:33,984 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,985 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/lexer.cpython-314.pyc')
+2026-02-12 01:53:33,987 - INFO - Upload completed successfully: lexer.cpython-314.pyc
+2026-02-12 01:53:33,987 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc (5197 bytes)
+2026-02-12 01:53:33,988 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,989 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,989 - DEBUG - Progress: 5197/5197 bytes
+2026-02-12 01:53:33,989 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,990 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/completion.cpython-314.pyc')
+2026-02-12 01:53:33,992 - INFO - Upload completed successfully: completion.cpython-314.pyc
+2026-02-12 01:53:33,992 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc (30836 bytes)
+2026-02-12 01:53:33,993 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc', 'wb')
+2026-02-12 01:53:33,994 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:33,995 - DEBUG - Progress: 30836/30836 bytes
+2026-02-12 01:53:33,995 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:33,998 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/compiler.cpython-314.pyc')
+2026-02-12 01:53:34,000 - INFO - Upload completed successfully: compiler.cpython-314.pyc
+2026-02-12 01:53:34,000 - INFO - Starting upload: lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc (3507 bytes)
+2026-02-12 01:53:34,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,003 - DEBUG - Progress: 3507/3507 bytes
+2026-02-12 01:53:34,003 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,004 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/regular_languages/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:34,006 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:34,006 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers', 511)
+2026-02-12 01:53:34,006 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/completers
+2026-02-12 01:53:34,006 - INFO - Starting upload: lib/prompt_toolkit/contrib/completers/system.py -> /home/kevin/test/lib/prompt_toolkit/contrib/completers/system.py (2057 bytes)
+2026-02-12 01:53:34,008 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/system.py', 'wb')
+2026-02-12 01:53:34,009 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/system.py', 'wb') -> 00000000
+2026-02-12 01:53:34,009 - DEBUG - Progress: 2057/2057 bytes
+2026-02-12 01:53:34,009 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,010 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/system.py')
+2026-02-12 01:53:34,011 - INFO - Upload completed successfully: system.py
+2026-02-12 01:53:34,011 - INFO - Starting upload: lib/prompt_toolkit/contrib/completers/__init__.py -> /home/kevin/test/lib/prompt_toolkit/contrib/completers/__init__.py (103 bytes)
+2026-02-12 01:53:34,012 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__init__.py', 'wb')
+2026-02-12 01:53:34,013 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:34,013 - DEBUG - Progress: 103/103 bytes
+2026-02-12 01:53:34,013 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,014 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__init__.py')
+2026-02-12 01:53:34,015 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:34,015 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__', 511)
+2026-02-12 01:53:34,016 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__
+2026-02-12 01:53:34,016 - INFO - Starting upload: lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc (2760 bytes)
+2026-02-12 01:53:34,017 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,018 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,018 - DEBUG - Progress: 2760/2760 bytes
+2026-02-12 01:53:34,018 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,019 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/system.cpython-314.pyc')
+2026-02-12 01:53:34,021 - INFO - Upload completed successfully: system.cpython-314.pyc
+2026-02-12 01:53:34,021 - INFO - Starting upload: lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc (304 bytes)
+2026-02-12 01:53:34,023 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,024 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,024 - DEBUG - Progress: 304/304 bytes
+2026-02-12 01:53:34,024 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,025 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/contrib/completers/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:34,026 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:34,026 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/completion', 511)
+2026-02-12 01:53:34,027 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/completion
+2026-02-12 01:53:34,027 - INFO - Starting upload: lib/prompt_toolkit/completion/word_completer.py -> /home/kevin/test/lib/prompt_toolkit/completion/word_completer.py (3435 bytes)
+2026-02-12 01:53:34,028 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/word_completer.py', 'wb')
+2026-02-12 01:53:34,029 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/word_completer.py', 'wb') -> 00000000
+2026-02-12 01:53:34,029 - DEBUG - Progress: 3435/3435 bytes
+2026-02-12 01:53:34,029 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,030 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/word_completer.py')
+2026-02-12 01:53:34,032 - INFO - Upload completed successfully: word_completer.py
+2026-02-12 01:53:34,032 - INFO - Starting upload: lib/prompt_toolkit/completion/nested.py -> /home/kevin/test/lib/prompt_toolkit/completion/nested.py (3844 bytes)
+2026-02-12 01:53:34,033 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/nested.py', 'wb')
+2026-02-12 01:53:34,034 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/nested.py', 'wb') -> 00000000
+2026-02-12 01:53:34,034 - DEBUG - Progress: 3844/3844 bytes
+2026-02-12 01:53:34,034 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,036 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/nested.py')
+2026-02-12 01:53:34,038 - INFO - Upload completed successfully: nested.py
+2026-02-12 01:53:34,038 - INFO - Starting upload: lib/prompt_toolkit/completion/fuzzy_completer.py -> /home/kevin/test/lib/prompt_toolkit/completion/fuzzy_completer.py (7738 bytes)
+2026-02-12 01:53:34,039 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/fuzzy_completer.py', 'wb')
+2026-02-12 01:53:34,040 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/fuzzy_completer.py', 'wb') -> 00000000
+2026-02-12 01:53:34,040 - DEBUG - Progress: 7738/7738 bytes
+2026-02-12 01:53:34,040 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,042 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/fuzzy_completer.py')
+2026-02-12 01:53:34,044 - INFO - Upload completed successfully: fuzzy_completer.py
+2026-02-12 01:53:34,044 - INFO - Starting upload: lib/prompt_toolkit/completion/filesystem.py -> /home/kevin/test/lib/prompt_toolkit/completion/filesystem.py (3949 bytes)
+2026-02-12 01:53:34,045 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/filesystem.py', 'wb')
+2026-02-12 01:53:34,046 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/filesystem.py', 'wb') -> 00000000
+2026-02-12 01:53:34,046 - DEBUG - Progress: 3949/3949 bytes
+2026-02-12 01:53:34,046 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,048 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/filesystem.py')
+2026-02-12 01:53:34,049 - INFO - Upload completed successfully: filesystem.py
+2026-02-12 01:53:34,050 - INFO - Starting upload: lib/prompt_toolkit/completion/deduplicate.py -> /home/kevin/test/lib/prompt_toolkit/completion/deduplicate.py (1436 bytes)
+2026-02-12 01:53:34,051 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/deduplicate.py', 'wb')
+2026-02-12 01:53:34,052 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/deduplicate.py', 'wb') -> 00000000
+2026-02-12 01:53:34,052 - DEBUG - Progress: 1436/1436 bytes
+2026-02-12 01:53:34,052 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,053 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/deduplicate.py')
+2026-02-12 01:53:34,054 - INFO - Upload completed successfully: deduplicate.py
+2026-02-12 01:53:34,054 - INFO - Starting upload: lib/prompt_toolkit/completion/base.py -> /home/kevin/test/lib/prompt_toolkit/completion/base.py (16103 bytes)
+2026-02-12 01:53:34,055 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/base.py', 'wb')
+2026-02-12 01:53:34,056 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/base.py', 'wb') -> 00000000
+2026-02-12 01:53:34,056 - DEBUG - Progress: 16103/16103 bytes
+2026-02-12 01:53:34,056 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,059 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/base.py')
+2026-02-12 01:53:34,060 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:34,060 - INFO - Starting upload: lib/prompt_toolkit/completion/__init__.py -> /home/kevin/test/lib/prompt_toolkit/completion/__init__.py (992 bytes)
+2026-02-12 01:53:34,062 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__init__.py', 'wb')
+2026-02-12 01:53:34,063 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:34,063 - DEBUG - Progress: 992/992 bytes
+2026-02-12 01:53:34,063 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,064 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__init__.py')
+2026-02-12 01:53:34,066 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:34,066 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__', 511)
+2026-02-12 01:53:34,066 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/completion/__pycache__
+2026-02-12 01:53:34,066 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc (4813 bytes)
+2026-02-12 01:53:34,068 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,068 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,069 - DEBUG - Progress: 4813/4813 bytes
+2026-02-12 01:53:34,069 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,070 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/word_completer.cpython-314.pyc')
+2026-02-12 01:53:34,071 - INFO - Upload completed successfully: word_completer.cpython-314.pyc
+2026-02-12 01:53:34,071 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc (5325 bytes)
+2026-02-12 01:53:34,074 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,075 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,075 - DEBUG - Progress: 5325/5325 bytes
+2026-02-12 01:53:34,075 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,076 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/nested.cpython-314.pyc')
+2026-02-12 01:53:34,078 - INFO - Upload completed successfully: nested.cpython-314.pyc
+2026-02-12 01:53:34,078 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc (10991 bytes)
+2026-02-12 01:53:34,081 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,082 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,082 - DEBUG - Progress: 10991/10991 bytes
+2026-02-12 01:53:34,082 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,084 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/fuzzy_completer.cpython-314.pyc')
+2026-02-12 01:53:34,086 - INFO - Upload completed successfully: fuzzy_completer.cpython-314.pyc
+2026-02-12 01:53:34,086 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc (5951 bytes)
+2026-02-12 01:53:34,089 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,089 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,089 - DEBUG - Progress: 5951/5951 bytes
+2026-02-12 01:53:34,089 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,091 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/filesystem.cpython-314.pyc')
+2026-02-12 01:53:34,093 - INFO - Upload completed successfully: filesystem.cpython-314.pyc
+2026-02-12 01:53:34,093 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc (2295 bytes)
+2026-02-12 01:53:34,095 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,096 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,096 - DEBUG - Progress: 2295/2295 bytes
+2026-02-12 01:53:34,096 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,097 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/deduplicate.cpython-314.pyc')
+2026-02-12 01:53:34,099 - INFO - Upload completed successfully: deduplicate.cpython-314.pyc
+2026-02-12 01:53:34,099 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc (23460 bytes)
+2026-02-12 01:53:34,101 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,101 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,102 - DEBUG - Progress: 23460/23460 bytes
+2026-02-12 01:53:34,102 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,105 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:34,106 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:34,106 - INFO - Starting upload: lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc (920 bytes)
+2026-02-12 01:53:34,108 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,108 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,108 - DEBUG - Progress: 920/920 bytes
+2026-02-12 01:53:34,108 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,109 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/completion/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:34,111 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:34,111 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/clipboard', 511)
+2026-02-12 01:53:34,111 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/clipboard
+2026-02-12 01:53:34,111 - INFO - Starting upload: lib/prompt_toolkit/clipboard/pyperclip.py -> /home/kevin/test/lib/prompt_toolkit/clipboard/pyperclip.py (1160 bytes)
+2026-02-12 01:53:34,113 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/pyperclip.py', 'wb')
+2026-02-12 01:53:34,113 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/pyperclip.py', 'wb') -> 00000000
+2026-02-12 01:53:34,114 - DEBUG - Progress: 1160/1160 bytes
+2026-02-12 01:53:34,114 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,114 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/pyperclip.py')
+2026-02-12 01:53:34,116 - INFO - Upload completed successfully: pyperclip.py
+2026-02-12 01:53:34,116 - INFO - Starting upload: lib/prompt_toolkit/clipboard/in_memory.py -> /home/kevin/test/lib/prompt_toolkit/clipboard/in_memory.py (1060 bytes)
+2026-02-12 01:53:34,118 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/in_memory.py', 'wb')
+2026-02-12 01:53:34,118 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/in_memory.py', 'wb') -> 00000000
+2026-02-12 01:53:34,118 - DEBUG - Progress: 1060/1060 bytes
+2026-02-12 01:53:34,119 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,119 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/in_memory.py')
+2026-02-12 01:53:34,121 - INFO - Upload completed successfully: in_memory.py
+2026-02-12 01:53:34,121 - INFO - Starting upload: lib/prompt_toolkit/clipboard/base.py -> /home/kevin/test/lib/prompt_toolkit/clipboard/base.py (2515 bytes)
+2026-02-12 01:53:34,122 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/base.py', 'wb')
+2026-02-12 01:53:34,123 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/base.py', 'wb') -> 00000000
+2026-02-12 01:53:34,123 - DEBUG - Progress: 2515/2515 bytes
+2026-02-12 01:53:34,123 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,124 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/base.py')
+2026-02-12 01:53:34,126 - INFO - Upload completed successfully: base.py
+2026-02-12 01:53:34,126 - INFO - Starting upload: lib/prompt_toolkit/clipboard/__init__.py -> /home/kevin/test/lib/prompt_toolkit/clipboard/__init__.py (439 bytes)
+2026-02-12 01:53:34,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__init__.py', 'wb')
+2026-02-12 01:53:34,128 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:34,128 - DEBUG - Progress: 439/439 bytes
+2026-02-12 01:53:34,128 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,129 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__init__.py')
+2026-02-12 01:53:34,130 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:34,130 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__', 511)
+2026-02-12 01:53:34,131 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__
+2026-02-12 01:53:34,131 - INFO - Starting upload: lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc (2225 bytes)
+2026-02-12 01:53:34,133 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,133 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,133 - DEBUG - Progress: 2225/2225 bytes
+2026-02-12 01:53:34,133 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,135 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/pyperclip.cpython-314.pyc')
+2026-02-12 01:53:34,136 - INFO - Upload completed successfully: pyperclip.cpython-314.pyc
+2026-02-12 01:53:34,136 - INFO - Starting upload: lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc (2761 bytes)
+2026-02-12 01:53:34,137 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,138 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,138 - DEBUG - Progress: 2761/2761 bytes
+2026-02-12 01:53:34,138 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,140 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/in_memory.cpython-314.pyc')
+2026-02-12 01:53:34,142 - INFO - Upload completed successfully: in_memory.cpython-314.pyc
+2026-02-12 01:53:34,142 - INFO - Starting upload: lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc (6896 bytes)
+2026-02-12 01:53:34,143 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,144 - DEBUG - Progress: 6896/6896 bytes
+2026-02-12 01:53:34,144 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,145 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/base.cpython-314.pyc')
+2026-02-12 01:53:34,147 - INFO - Upload completed successfully: base.cpython-314.pyc
+2026-02-12 01:53:34,147 - INFO - Starting upload: lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc (441 bytes)
+2026-02-12 01:53:34,149 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,149 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,149 - DEBUG - Progress: 441/441 bytes
+2026-02-12 01:53:34,149 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,150 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/clipboard/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:34,151 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:34,152 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/application', 511)
+2026-02-12 01:53:34,152 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/application
+2026-02-12 01:53:34,152 - INFO - Starting upload: lib/prompt_toolkit/application/run_in_terminal.py -> /home/kevin/test/lib/prompt_toolkit/application/run_in_terminal.py (3767 bytes)
+2026-02-12 01:53:34,154 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/run_in_terminal.py', 'wb')
+2026-02-12 01:53:34,155 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/run_in_terminal.py', 'wb') -> 00000000
+2026-02-12 01:53:34,155 - DEBUG - Progress: 3767/3767 bytes
+2026-02-12 01:53:34,155 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,156 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/run_in_terminal.py')
+2026-02-12 01:53:34,157 - INFO - Upload completed successfully: run_in_terminal.py
+2026-02-12 01:53:34,158 - INFO - Starting upload: lib/prompt_toolkit/application/dummy.py -> /home/kevin/test/lib/prompt_toolkit/application/dummy.py (1619 bytes)
+2026-02-12 01:53:34,159 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/dummy.py', 'wb')
+2026-02-12 01:53:34,159 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/dummy.py', 'wb') -> 00000000
+2026-02-12 01:53:34,160 - DEBUG - Progress: 1619/1619 bytes
+2026-02-12 01:53:34,160 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,160 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/dummy.py')
+2026-02-12 01:53:34,162 - INFO - Upload completed successfully: dummy.py
+2026-02-12 01:53:34,162 - INFO - Starting upload: lib/prompt_toolkit/application/current.py -> /home/kevin/test/lib/prompt_toolkit/application/current.py (6209 bytes)
+2026-02-12 01:53:34,163 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/current.py', 'wb')
+2026-02-12 01:53:34,164 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/current.py', 'wb') -> 00000000
+2026-02-12 01:53:34,166 - DEBUG - Progress: 6209/6209 bytes
+2026-02-12 01:53:34,166 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,167 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/current.py')
+2026-02-12 01:53:34,169 - INFO - Upload completed successfully: current.py
+2026-02-12 01:53:34,169 - INFO - Starting upload: lib/prompt_toolkit/application/application.py -> /home/kevin/test/lib/prompt_toolkit/application/application.py (63242 bytes)
+2026-02-12 01:53:34,171 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/application.py', 'wb')
+2026-02-12 01:53:34,171 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/application.py', 'wb') -> 00000000
+2026-02-12 01:53:34,172 - DEBUG - Progress: 32768/63242 bytes
+2026-02-12 01:53:34,172 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,180 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/application.py')
+2026-02-12 01:53:34,183 - INFO - Upload completed successfully: application.py
+2026-02-12 01:53:34,183 - INFO - Starting upload: lib/prompt_toolkit/application/__init__.py -> /home/kevin/test/lib/prompt_toolkit/application/__init__.py (657 bytes)
+2026-02-12 01:53:34,185 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__init__.py', 'wb')
+2026-02-12 01:53:34,186 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:34,186 - DEBUG - Progress: 657/657 bytes
+2026-02-12 01:53:34,186 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,187 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/__init__.py')
+2026-02-12 01:53:34,189 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:34,189 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__', 511)
+2026-02-12 01:53:34,190 - INFO - Created remote folder: /home/kevin/test/lib/prompt_toolkit/application/__pycache__
+2026-02-12 01:53:34,190 - INFO - Starting upload: lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc (5689 bytes)
+2026-02-12 01:53:34,192 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,192 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,193 - DEBUG - Progress: 5689/5689 bytes
+2026-02-12 01:53:34,193 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,194 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/run_in_terminal.cpython-314.pyc')
+2026-02-12 01:53:34,196 - INFO - Upload completed successfully: run_in_terminal.cpython-314.pyc
+2026-02-12 01:53:34,196 - INFO - Starting upload: lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc (3398 bytes)
+2026-02-12 01:53:34,198 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,199 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,199 - DEBUG - Progress: 3398/3398 bytes
+2026-02-12 01:53:34,199 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,200 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/dummy.cpython-314.pyc')
+2026-02-12 01:53:34,202 - INFO - Upload completed successfully: dummy.cpython-314.pyc
+2026-02-12 01:53:34,202 - INFO - Starting upload: lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc (8355 bytes)
+2026-02-12 01:53:34,204 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,205 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,205 - DEBUG - Progress: 8355/8355 bytes
+2026-02-12 01:53:34,205 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,207 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/current.cpython-314.pyc')
+2026-02-12 01:53:34,209 - INFO - Upload completed successfully: current.cpython-314.pyc
+2026-02-12 01:53:34,209 - INFO - Starting upload: lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc (78749 bytes)
+2026-02-12 01:53:34,211 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,212 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,212 - DEBUG - Progress: 65536/78749 bytes
+2026-02-12 01:53:34,212 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,222 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/application.cpython-314.pyc')
+2026-02-12 01:53:34,224 - INFO - Upload completed successfully: application.cpython-314.pyc
+2026-02-12 01:53:34,224 - INFO - Starting upload: lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc (671 bytes)
+2026-02-12 01:53:34,226 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,227 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,227 - DEBUG - Progress: 671/671 bytes
+2026-02-12 01:53:34,227 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,228 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/prompt_toolkit/application/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:34,229 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:34,229 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info', 511)
+2026-02-12 01:53:34,230 - INFO - Created remote folder: /home/kevin/test/lib/wcwidth-0.6.0.dist-info
+2026-02-12 01:53:34,230 - INFO - Starting upload: lib/wcwidth-0.6.0.dist-info/RECORD -> /home/kevin/test/lib/wcwidth-0.6.0.dist-info/RECORD (2429 bytes)
+2026-02-12 01:53:34,232 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/RECORD', 'wb')
+2026-02-12 01:53:34,233 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:34,233 - DEBUG - Progress: 2429/2429 bytes
+2026-02-12 01:53:34,233 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,234 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/RECORD')
+2026-02-12 01:53:34,236 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:34,236 - INFO - Starting upload: lib/wcwidth-0.6.0.dist-info/INSTALLER -> /home/kevin/test/lib/wcwidth-0.6.0.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:34,238 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:34,239 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:34,239 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:34,239 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,240 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/INSTALLER')
+2026-02-12 01:53:34,242 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:34,242 - INFO - Starting upload: lib/wcwidth-0.6.0.dist-info/WHEEL -> /home/kevin/test/lib/wcwidth-0.6.0.dist-info/WHEEL (87 bytes)
+2026-02-12 01:53:34,244 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:34,244 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:34,244 - DEBUG - Progress: 87/87 bytes
+2026-02-12 01:53:34,244 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,245 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/WHEEL')
+2026-02-12 01:53:34,247 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:34,247 - INFO - Starting upload: lib/wcwidth-0.6.0.dist-info/METADATA -> /home/kevin/test/lib/wcwidth-0.6.0.dist-info/METADATA (30525 bytes)
+2026-02-12 01:53:34,249 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/METADATA', 'wb')
+2026-02-12 01:53:34,250 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:34,250 - DEBUG - Progress: 30525/30525 bytes
+2026-02-12 01:53:34,251 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,254 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/METADATA')
+2026-02-12 01:53:34,256 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:34,256 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/licenses', 511)
+2026-02-12 01:53:34,256 - INFO - Created remote folder: /home/kevin/test/lib/wcwidth-0.6.0.dist-info/licenses
+2026-02-12 01:53:34,256 - INFO - Starting upload: lib/wcwidth-0.6.0.dist-info/licenses/LICENSE -> /home/kevin/test/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE (1322 bytes)
+2026-02-12 01:53:34,258 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE', 'wb')
+2026-02-12 01:53:34,258 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:34,258 - DEBUG - Progress: 1322/1322 bytes
+2026-02-12 01:53:34,259 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,259 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth-0.6.0.dist-info/licenses/LICENSE')
+2026-02-12 01:53:34,261 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:34,261 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/wcwidth', 511)
+2026-02-12 01:53:34,262 - INFO - Created remote folder: /home/kevin/test/lib/wcwidth
+2026-02-12 01:53:34,262 - INFO - Starting upload: lib/wcwidth/wcwidth.py -> /home/kevin/test/lib/wcwidth/wcwidth.py (41127 bytes)
+2026-02-12 01:53:34,263 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/wcwidth.py', 'wb')
+2026-02-12 01:53:34,264 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/wcwidth.py', 'wb') -> 00000000
+2026-02-12 01:53:34,265 - DEBUG - Progress: 32768/41127 bytes
+2026-02-12 01:53:34,265 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,270 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/wcwidth.py')
+2026-02-12 01:53:34,271 - INFO - Upload completed successfully: wcwidth.py
+2026-02-12 01:53:34,272 - INFO - Starting upload: lib/wcwidth/unicode_versions.py -> /home/kevin/test/lib/wcwidth/unicode_versions.py (541 bytes)
+2026-02-12 01:53:34,273 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/unicode_versions.py', 'wb')
+2026-02-12 01:53:34,274 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/unicode_versions.py', 'wb') -> 00000000
+2026-02-12 01:53:34,274 - DEBUG - Progress: 541/541 bytes
+2026-02-12 01:53:34,274 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,275 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/unicode_versions.py')
+2026-02-12 01:53:34,276 - INFO - Upload completed successfully: unicode_versions.py
+2026-02-12 01:53:34,276 - INFO - Starting upload: lib/wcwidth/textwrap.py -> /home/kevin/test/lib/wcwidth/textwrap.py (28997 bytes)
+2026-02-12 01:53:34,278 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/textwrap.py', 'wb')
+2026-02-12 01:53:34,278 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/textwrap.py', 'wb') -> 00000000
+2026-02-12 01:53:34,279 - DEBUG - Progress: 28997/28997 bytes
+2026-02-12 01:53:34,279 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,283 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/textwrap.py')
+2026-02-12 01:53:34,284 - INFO - Upload completed successfully: textwrap.py
+2026-02-12 01:53:34,285 - INFO - Starting upload: lib/wcwidth/table_zero.py -> /home/kevin/test/lib/wcwidth/table_zero.py (25642 bytes)
+2026-02-12 01:53:34,286 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_zero.py', 'wb')
+2026-02-12 01:53:34,287 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_zero.py', 'wb') -> 00000000
+2026-02-12 01:53:34,287 - DEBUG - Progress: 25642/25642 bytes
+2026-02-12 01:53:34,287 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,291 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/table_zero.py')
+2026-02-12 01:53:34,293 - INFO - Upload completed successfully: table_zero.py
+2026-02-12 01:53:34,293 - INFO - Starting upload: lib/wcwidth/table_wide.py -> /home/kevin/test/lib/wcwidth/table_wide.py (8974 bytes)
+2026-02-12 01:53:34,295 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_wide.py', 'wb')
+2026-02-12 01:53:34,295 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_wide.py', 'wb') -> 00000000
+2026-02-12 01:53:34,296 - DEBUG - Progress: 8974/8974 bytes
+2026-02-12 01:53:34,296 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,297 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/table_wide.py')
+2026-02-12 01:53:34,299 - INFO - Upload completed successfully: table_wide.py
+2026-02-12 01:53:34,299 - INFO - Starting upload: lib/wcwidth/table_vs16.py -> /home/kevin/test/lib/wcwidth/table_vs16.py (6890 bytes)
+2026-02-12 01:53:34,301 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_vs16.py', 'wb')
+2026-02-12 01:53:34,302 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_vs16.py', 'wb') -> 00000000
+2026-02-12 01:53:34,302 - DEBUG - Progress: 6890/6890 bytes
+2026-02-12 01:53:34,302 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,304 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/table_vs16.py')
+2026-02-12 01:53:34,306 - INFO - Upload completed successfully: table_vs16.py
+2026-02-12 01:53:34,306 - INFO - Starting upload: lib/wcwidth/table_mc.py -> /home/kevin/test/lib/wcwidth/table_mc.py (13993 bytes)
+2026-02-12 01:53:34,308 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_mc.py', 'wb')
+2026-02-12 01:53:34,308 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_mc.py', 'wb') -> 00000000
+2026-02-12 01:53:34,309 - DEBUG - Progress: 13993/13993 bytes
+2026-02-12 01:53:34,309 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,311 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/table_mc.py')
+2026-02-12 01:53:34,313 - INFO - Upload completed successfully: table_mc.py
+2026-02-12 01:53:34,314 - INFO - Starting upload: lib/wcwidth/table_grapheme.py -> /home/kevin/test/lib/wcwidth/table_grapheme.py (142899 bytes)
+2026-02-12 01:53:34,316 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_grapheme.py', 'wb')
+2026-02-12 01:53:34,316 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_grapheme.py', 'wb') -> 00000000
+2026-02-12 01:53:34,318 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,334 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/table_grapheme.py')
+2026-02-12 01:53:34,337 - INFO - Upload completed successfully: table_grapheme.py
+2026-02-12 01:53:34,337 - INFO - Starting upload: lib/wcwidth/table_ambiguous.py -> /home/kevin/test/lib/wcwidth/table_ambiguous.py (11900 bytes)
+2026-02-12 01:53:34,339 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_ambiguous.py', 'wb')
+2026-02-12 01:53:34,339 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/table_ambiguous.py', 'wb') -> 00000000
+2026-02-12 01:53:34,340 - DEBUG - Progress: 11900/11900 bytes
+2026-02-12 01:53:34,340 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,342 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/table_ambiguous.py')
+2026-02-12 01:53:34,344 - INFO - Upload completed successfully: table_ambiguous.py
+2026-02-12 01:53:34,344 - INFO - Starting upload: lib/wcwidth/sgr_state.py -> /home/kevin/test/lib/wcwidth/sgr_state.py (12429 bytes)
+2026-02-12 01:53:34,346 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/sgr_state.py', 'wb')
+2026-02-12 01:53:34,347 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/sgr_state.py', 'wb') -> 00000000
+2026-02-12 01:53:34,348 - DEBUG - Progress: 12429/12429 bytes
+2026-02-12 01:53:34,348 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,350 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/sgr_state.py')
+2026-02-12 01:53:34,352 - INFO - Upload completed successfully: sgr_state.py
+2026-02-12 01:53:34,352 - INFO - Starting upload: lib/wcwidth/py.typed -> /home/kevin/test/lib/wcwidth/py.typed (0 bytes)
+2026-02-12 01:53:34,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/py.typed', 'wb')
+2026-02-12 01:53:34,355 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/py.typed', 'wb') -> 00000000
+2026-02-12 01:53:34,355 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,356 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/py.typed')
+2026-02-12 01:53:34,357 - INFO - Upload completed successfully: py.typed
+2026-02-12 01:53:34,358 - INFO - Starting upload: lib/wcwidth/grapheme.py -> /home/kevin/test/lib/wcwidth/grapheme.py (13382 bytes)
+2026-02-12 01:53:34,359 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/grapheme.py', 'wb')
+2026-02-12 01:53:34,360 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/grapheme.py', 'wb') -> 00000000
+2026-02-12 01:53:34,361 - DEBUG - Progress: 13382/13382 bytes
+2026-02-12 01:53:34,361 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,364 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/grapheme.py')
+2026-02-12 01:53:34,366 - INFO - Upload completed successfully: grapheme.py
+2026-02-12 01:53:34,367 - INFO - Starting upload: lib/wcwidth/escape_sequences.py -> /home/kevin/test/lib/wcwidth/escape_sequences.py (3097 bytes)
+2026-02-12 01:53:34,369 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/escape_sequences.py', 'wb')
+2026-02-12 01:53:34,370 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/escape_sequences.py', 'wb') -> 00000000
+2026-02-12 01:53:34,370 - DEBUG - Progress: 3097/3097 bytes
+2026-02-12 01:53:34,370 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,371 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/escape_sequences.py')
+2026-02-12 01:53:34,373 - INFO - Upload completed successfully: escape_sequences.py
+2026-02-12 01:53:34,373 - INFO - Starting upload: lib/wcwidth/control_codes.py -> /home/kevin/test/lib/wcwidth/control_codes.py (1579 bytes)
+2026-02-12 01:53:34,375 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/control_codes.py', 'wb')
+2026-02-12 01:53:34,376 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/control_codes.py', 'wb') -> 00000000
+2026-02-12 01:53:34,376 - DEBUG - Progress: 1579/1579 bytes
+2026-02-12 01:53:34,376 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,377 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/control_codes.py')
+2026-02-12 01:53:34,379 - INFO - Upload completed successfully: control_codes.py
+2026-02-12 01:53:34,379 - INFO - Starting upload: lib/wcwidth/bisearch.py -> /home/kevin/test/lib/wcwidth/bisearch.py (812 bytes)
+2026-02-12 01:53:34,382 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/bisearch.py', 'wb')
+2026-02-12 01:53:34,383 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/bisearch.py', 'wb') -> 00000000
+2026-02-12 01:53:34,383 - DEBUG - Progress: 812/812 bytes
+2026-02-12 01:53:34,383 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,384 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/bisearch.py')
+2026-02-12 01:53:34,386 - INFO - Upload completed successfully: bisearch.py
+2026-02-12 01:53:34,386 - INFO - Starting upload: lib/wcwidth/__init__.py -> /home/kevin/test/lib/wcwidth/__init__.py (1793 bytes)
+2026-02-12 01:53:34,388 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__init__.py', 'wb')
+2026-02-12 01:53:34,388 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:34,389 - DEBUG - Progress: 1793/1793 bytes
+2026-02-12 01:53:34,389 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,390 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__init__.py')
+2026-02-12 01:53:34,392 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:34,392 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/wcwidth/__pycache__', 511)
+2026-02-12 01:53:34,393 - INFO - Created remote folder: /home/kevin/test/lib/wcwidth/__pycache__
+2026-02-12 01:53:34,393 - INFO - Starting upload: lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc (34466 bytes)
+2026-02-12 01:53:34,395 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,396 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,396 - DEBUG - Progress: 32768/34466 bytes
+2026-02-12 01:53:34,396 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,401 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/wcwidth.cpython-314.pyc')
+2026-02-12 01:53:34,403 - INFO - Upload completed successfully: wcwidth.cpython-314.pyc
+2026-02-12 01:53:34,403 - INFO - Starting upload: lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc (914 bytes)
+2026-02-12 01:53:34,405 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,406 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,406 - DEBUG - Progress: 914/914 bytes
+2026-02-12 01:53:34,406 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,407 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/unicode_versions.cpython-314.pyc')
+2026-02-12 01:53:34,410 - INFO - Upload completed successfully: unicode_versions.cpython-314.pyc
+2026-02-12 01:53:34,411 - INFO - Starting upload: lib/wcwidth/__pycache__/textwrap.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc (27431 bytes)
+2026-02-12 01:53:34,414 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,415 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,415 - DEBUG - Progress: 27431/27431 bytes
+2026-02-12 01:53:34,415 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,419 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/textwrap.cpython-314.pyc')
+2026-02-12 01:53:34,423 - INFO - Upload completed successfully: textwrap.cpython-314.pyc
+2026-02-12 01:53:34,423 - INFO - Starting upload: lib/wcwidth/__pycache__/table_zero.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc (4419 bytes)
+2026-02-12 01:53:34,425 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,426 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,426 - DEBUG - Progress: 4419/4419 bytes
+2026-02-12 01:53:34,426 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,427 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/table_zero.cpython-314.pyc')
+2026-02-12 01:53:34,429 - INFO - Upload completed successfully: table_zero.cpython-314.pyc
+2026-02-12 01:53:34,430 - INFO - Starting upload: lib/wcwidth/__pycache__/table_wide.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc (1880 bytes)
+2026-02-12 01:53:34,432 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,432 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,432 - DEBUG - Progress: 1880/1880 bytes
+2026-02-12 01:53:34,432 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,433 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/table_wide.cpython-314.pyc')
+2026-02-12 01:53:34,436 - INFO - Upload completed successfully: table_wide.cpython-314.pyc
+2026-02-12 01:53:34,436 - INFO - Starting upload: lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc (1746 bytes)
+2026-02-12 01:53:34,438 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,438 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,438 - DEBUG - Progress: 1746/1746 bytes
+2026-02-12 01:53:34,438 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,439 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/table_vs16.cpython-314.pyc')
+2026-02-12 01:53:34,441 - INFO - Upload completed successfully: table_vs16.cpython-314.pyc
+2026-02-12 01:53:34,441 - INFO - Starting upload: lib/wcwidth/__pycache__/table_mc.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc (2688 bytes)
+2026-02-12 01:53:34,443 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,443 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,443 - DEBUG - Progress: 2688/2688 bytes
+2026-02-12 01:53:34,443 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,445 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/table_mc.cpython-314.pyc')
+2026-02-12 01:53:34,446 - INFO - Upload completed successfully: table_mc.cpython-314.pyc
+2026-02-12 01:53:34,447 - INFO - Starting upload: lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc (24115 bytes)
+2026-02-12 01:53:34,449 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,450 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,450 - DEBUG - Progress: 24115/24115 bytes
+2026-02-12 01:53:34,450 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,454 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/table_grapheme.cpython-314.pyc')
+2026-02-12 01:53:34,455 - INFO - Upload completed successfully: table_grapheme.cpython-314.pyc
+2026-02-12 01:53:34,455 - INFO - Starting upload: lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc (2508 bytes)
+2026-02-12 01:53:34,457 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,457 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,457 - DEBUG - Progress: 2508/2508 bytes
+2026-02-12 01:53:34,457 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,459 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/table_ambiguous.cpython-314.pyc')
+2026-02-12 01:53:34,460 - INFO - Upload completed successfully: table_ambiguous.cpython-314.pyc
+2026-02-12 01:53:34,460 - INFO - Starting upload: lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc (16357 bytes)
+2026-02-12 01:53:34,462 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,462 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,463 - DEBUG - Progress: 16357/16357 bytes
+2026-02-12 01:53:34,463 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,465 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/sgr_state.cpython-314.pyc')
+2026-02-12 01:53:34,467 - INFO - Upload completed successfully: sgr_state.cpython-314.pyc
+2026-02-12 01:53:34,467 - INFO - Starting upload: lib/wcwidth/__pycache__/grapheme.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc (15474 bytes)
+2026-02-12 01:53:34,468 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,469 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,469 - DEBUG - Progress: 15474/15474 bytes
+2026-02-12 01:53:34,469 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,471 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/grapheme.cpython-314.pyc')
+2026-02-12 01:53:34,473 - INFO - Upload completed successfully: grapheme.cpython-314.pyc
+2026-02-12 01:53:34,473 - INFO - Starting upload: lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc (1605 bytes)
+2026-02-12 01:53:34,474 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,475 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,475 - DEBUG - Progress: 1605/1605 bytes
+2026-02-12 01:53:34,475 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,476 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/escape_sequences.cpython-314.pyc')
+2026-02-12 01:53:34,478 - INFO - Upload completed successfully: escape_sequences.cpython-314.pyc
+2026-02-12 01:53:34,478 - INFO - Starting upload: lib/wcwidth/__pycache__/control_codes.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc (1170 bytes)
+2026-02-12 01:53:34,479 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,480 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,480 - DEBUG - Progress: 1170/1170 bytes
+2026-02-12 01:53:34,480 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,481 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/control_codes.cpython-314.pyc')
+2026-02-12 01:53:34,482 - INFO - Upload completed successfully: control_codes.cpython-314.pyc
+2026-02-12 01:53:34,482 - INFO - Starting upload: lib/wcwidth/__pycache__/bisearch.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc (1308 bytes)
+2026-02-12 01:53:34,484 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,484 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,484 - DEBUG - Progress: 1308/1308 bytes
+2026-02-12 01:53:34,484 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,485 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/bisearch.cpython-314.pyc')
+2026-02-12 01:53:34,487 - INFO - Upload completed successfully: bisearch.cpython-314.pyc
+2026-02-12 01:53:34,487 - INFO - Starting upload: lib/wcwidth/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/wcwidth/__pycache__/__init__.cpython-314.pyc (1043 bytes)
+2026-02-12 01:53:34,489 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:34,490 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/wcwidth/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:34,490 - DEBUG - Progress: 1043/1043 bytes
+2026-02-12 01:53:34,490 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,491 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/wcwidth/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:34,492 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:34,492 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/rich-14.3.2.dist-info', 511)
+2026-02-12 01:53:34,493 - INFO - Created remote folder: /home/kevin/test/lib/rich-14.3.2.dist-info
+2026-02-12 01:53:34,493 - INFO - Starting upload: lib/rich-14.3.2.dist-info/RECORD -> /home/kevin/test/lib/rich-14.3.2.dist-info/RECORD (13199 bytes)
+2026-02-12 01:53:34,494 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/RECORD', 'wb')
+2026-02-12 01:53:34,495 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:34,495 - DEBUG - Progress: 13199/13199 bytes
+2026-02-12 01:53:34,495 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,497 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich-14.3.2.dist-info/RECORD')
+2026-02-12 01:53:34,499 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:34,499 - INFO - Starting upload: lib/rich-14.3.2.dist-info/REQUESTED -> /home/kevin/test/lib/rich-14.3.2.dist-info/REQUESTED (0 bytes)
+2026-02-12 01:53:34,500 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/REQUESTED', 'wb')
+2026-02-12 01:53:34,501 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/REQUESTED', 'wb') -> 00000000
+2026-02-12 01:53:34,501 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,501 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich-14.3.2.dist-info/REQUESTED')
+2026-02-12 01:53:34,503 - INFO - Upload completed successfully: REQUESTED
+2026-02-12 01:53:34,503 - INFO - Starting upload: lib/rich-14.3.2.dist-info/INSTALLER -> /home/kevin/test/lib/rich-14.3.2.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:34,504 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:34,505 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:34,505 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:34,505 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,506 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich-14.3.2.dist-info/INSTALLER')
+2026-02-12 01:53:34,507 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:34,507 - INFO - Starting upload: lib/rich-14.3.2.dist-info/WHEEL -> /home/kevin/test/lib/rich-14.3.2.dist-info/WHEEL (88 bytes)
+2026-02-12 01:53:34,508 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:34,509 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:34,509 - DEBUG - Progress: 88/88 bytes
+2026-02-12 01:53:34,509 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,510 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich-14.3.2.dist-info/WHEEL')
+2026-02-12 01:53:34,511 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:34,511 - INFO - Starting upload: lib/rich-14.3.2.dist-info/METADATA -> /home/kevin/test/lib/rich-14.3.2.dist-info/METADATA (18473 bytes)
+2026-02-12 01:53:34,513 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/METADATA', 'wb')
+2026-02-12 01:53:34,513 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:34,513 - DEBUG - Progress: 18473/18473 bytes
+2026-02-12 01:53:34,513 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,517 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich-14.3.2.dist-info/METADATA')
+2026-02-12 01:53:34,519 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:34,519 - INFO - Starting upload: lib/rich-14.3.2.dist-info/LICENSE -> /home/kevin/test/lib/rich-14.3.2.dist-info/LICENSE (1056 bytes)
+2026-02-12 01:53:34,521 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/LICENSE', 'wb')
+2026-02-12 01:53:34,521 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich-14.3.2.dist-info/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:34,521 - DEBUG - Progress: 1056/1056 bytes
+2026-02-12 01:53:34,521 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,522 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich-14.3.2.dist-info/LICENSE')
+2026-02-12 01:53:34,523 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:34,524 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/rich', 511)
+2026-02-12 01:53:34,524 - INFO - Created remote folder: /home/kevin/test/lib/rich
+2026-02-12 01:53:34,524 - INFO - Starting upload: lib/rich/tree.py -> /home/kevin/test/lib/rich/tree.py (9391 bytes)
+2026-02-12 01:53:34,526 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/tree.py', 'wb')
+2026-02-12 01:53:34,526 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/tree.py', 'wb') -> 00000000
+2026-02-12 01:53:34,526 - DEBUG - Progress: 9391/9391 bytes
+2026-02-12 01:53:34,526 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,530 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/tree.py')
+2026-02-12 01:53:34,531 - INFO - Upload completed successfully: tree.py
+2026-02-12 01:53:34,532 - INFO - Starting upload: lib/rich/traceback.py -> /home/kevin/test/lib/rich/traceback.py (37535 bytes)
+2026-02-12 01:53:34,533 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/traceback.py', 'wb')
+2026-02-12 01:53:34,534 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/traceback.py', 'wb') -> 00000000
+2026-02-12 01:53:34,534 - DEBUG - Progress: 32768/37535 bytes
+2026-02-12 01:53:34,534 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,539 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/traceback.py')
+2026-02-12 01:53:34,542 - INFO - Upload completed successfully: traceback.py
+2026-02-12 01:53:34,542 - INFO - Starting upload: lib/rich/themes.py -> /home/kevin/test/lib/rich/themes.py (102 bytes)
+2026-02-12 01:53:34,544 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/themes.py', 'wb')
+2026-02-12 01:53:34,545 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/themes.py', 'wb') -> 00000000
+2026-02-12 01:53:34,545 - DEBUG - Progress: 102/102 bytes
+2026-02-12 01:53:34,545 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,546 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/themes.py')
+2026-02-12 01:53:34,548 - INFO - Upload completed successfully: themes.py
+2026-02-12 01:53:34,548 - INFO - Starting upload: lib/rich/theme.py -> /home/kevin/test/lib/rich/theme.py (3771 bytes)
+2026-02-12 01:53:34,550 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/theme.py', 'wb')
+2026-02-12 01:53:34,550 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/theme.py', 'wb') -> 00000000
+2026-02-12 01:53:34,551 - DEBUG - Progress: 3771/3771 bytes
+2026-02-12 01:53:34,551 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,552 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/theme.py')
+2026-02-12 01:53:34,554 - INFO - Upload completed successfully: theme.py
+2026-02-12 01:53:34,554 - INFO - Starting upload: lib/rich/text.py -> /home/kevin/test/lib/rich/text.py (47655 bytes)
+2026-02-12 01:53:34,556 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/text.py', 'wb')
+2026-02-12 01:53:34,557 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/text.py', 'wb') -> 00000000
+2026-02-12 01:53:34,558 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,563 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/text.py')
+2026-02-12 01:53:34,565 - INFO - Upload completed successfully: text.py
+2026-02-12 01:53:34,565 - INFO - Starting upload: lib/rich/terminal_theme.py -> /home/kevin/test/lib/rich/terminal_theme.py (3370 bytes)
+2026-02-12 01:53:34,567 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/terminal_theme.py', 'wb')
+2026-02-12 01:53:34,567 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/terminal_theme.py', 'wb') -> 00000000
+2026-02-12 01:53:34,567 - DEBUG - Progress: 3370/3370 bytes
+2026-02-12 01:53:34,567 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,569 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/terminal_theme.py')
+2026-02-12 01:53:34,571 - INFO - Upload completed successfully: terminal_theme.py
+2026-02-12 01:53:34,571 - INFO - Starting upload: lib/rich/table.py -> /home/kevin/test/lib/rich/table.py (40033 bytes)
+2026-02-12 01:53:34,572 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/table.py', 'wb')
+2026-02-12 01:53:34,573 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/table.py', 'wb') -> 00000000
+2026-02-12 01:53:34,573 - DEBUG - Progress: 32768/40033 bytes
+2026-02-12 01:53:34,573 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,578 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/table.py')
+2026-02-12 01:53:34,580 - INFO - Upload completed successfully: table.py
+2026-02-12 01:53:34,581 - INFO - Starting upload: lib/rich/syntax.py -> /home/kevin/test/lib/rich/syntax.py (36263 bytes)
+2026-02-12 01:53:34,583 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/syntax.py', 'wb')
+2026-02-12 01:53:34,583 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/syntax.py', 'wb') -> 00000000
+2026-02-12 01:53:34,584 - DEBUG - Progress: 32768/36263 bytes
+2026-02-12 01:53:34,584 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,588 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/syntax.py')
+2026-02-12 01:53:34,590 - INFO - Upload completed successfully: syntax.py
+2026-02-12 01:53:34,590 - INFO - Starting upload: lib/rich/styled.py -> /home/kevin/test/lib/rich/styled.py (1234 bytes)
+2026-02-12 01:53:34,593 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/styled.py', 'wb')
+2026-02-12 01:53:34,594 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/styled.py', 'wb') -> 00000000
+2026-02-12 01:53:34,594 - DEBUG - Progress: 1234/1234 bytes
+2026-02-12 01:53:34,594 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,595 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/styled.py')
+2026-02-12 01:53:34,597 - INFO - Upload completed successfully: styled.py
+2026-02-12 01:53:34,597 - INFO - Starting upload: lib/rich/style.py -> /home/kevin/test/lib/rich/style.py (26990 bytes)
+2026-02-12 01:53:34,599 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/style.py', 'wb')
+2026-02-12 01:53:34,600 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/style.py', 'wb') -> 00000000
+2026-02-12 01:53:34,600 - DEBUG - Progress: 26990/26990 bytes
+2026-02-12 01:53:34,600 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,603 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/style.py')
+2026-02-12 01:53:34,606 - INFO - Upload completed successfully: style.py
+2026-02-12 01:53:34,606 - INFO - Starting upload: lib/rich/status.py -> /home/kevin/test/lib/rich/status.py (4424 bytes)
+2026-02-12 01:53:34,608 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/status.py', 'wb')
+2026-02-12 01:53:34,609 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/status.py', 'wb') -> 00000000
+2026-02-12 01:53:34,609 - DEBUG - Progress: 4424/4424 bytes
+2026-02-12 01:53:34,609 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,610 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/status.py')
+2026-02-12 01:53:34,613 - INFO - Upload completed successfully: status.py
+2026-02-12 01:53:34,613 - INFO - Starting upload: lib/rich/spinner.py -> /home/kevin/test/lib/rich/spinner.py (4214 bytes)
+2026-02-12 01:53:34,615 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/spinner.py', 'wb')
+2026-02-12 01:53:34,615 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/spinner.py', 'wb') -> 00000000
+2026-02-12 01:53:34,616 - DEBUG - Progress: 4214/4214 bytes
+2026-02-12 01:53:34,616 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,617 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/spinner.py')
+2026-02-12 01:53:34,620 - INFO - Upload completed successfully: spinner.py
+2026-02-12 01:53:34,620 - INFO - Starting upload: lib/rich/segment.py -> /home/kevin/test/lib/rich/segment.py (25795 bytes)
+2026-02-12 01:53:34,624 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/segment.py', 'wb')
+2026-02-12 01:53:34,625 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/segment.py', 'wb') -> 00000000
+2026-02-12 01:53:34,625 - DEBUG - Progress: 25795/25795 bytes
+2026-02-12 01:53:34,625 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,629 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/segment.py')
+2026-02-12 01:53:34,630 - INFO - Upload completed successfully: segment.py
+2026-02-12 01:53:34,631 - INFO - Starting upload: lib/rich/screen.py -> /home/kevin/test/lib/rich/screen.py (1579 bytes)
+2026-02-12 01:53:34,633 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/screen.py', 'wb')
+2026-02-12 01:53:34,634 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/screen.py', 'wb') -> 00000000
+2026-02-12 01:53:34,635 - DEBUG - Progress: 1579/1579 bytes
+2026-02-12 01:53:34,635 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,636 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/screen.py')
+2026-02-12 01:53:34,638 - INFO - Upload completed successfully: screen.py
+2026-02-12 01:53:34,638 - INFO - Starting upload: lib/rich/scope.py -> /home/kevin/test/lib/rich/scope.py (3239 bytes)
+2026-02-12 01:53:34,640 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/scope.py', 'wb')
+2026-02-12 01:53:34,641 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/scope.py', 'wb') -> 00000000
+2026-02-12 01:53:34,641 - DEBUG - Progress: 3239/3239 bytes
+2026-02-12 01:53:34,641 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,642 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/scope.py')
+2026-02-12 01:53:34,644 - INFO - Upload completed successfully: scope.py
+2026-02-12 01:53:34,644 - INFO - Starting upload: lib/rich/rule.py -> /home/kevin/test/lib/rich/rule.py (4590 bytes)
+2026-02-12 01:53:34,646 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/rule.py', 'wb')
+2026-02-12 01:53:34,646 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/rule.py', 'wb') -> 00000000
+2026-02-12 01:53:34,646 - DEBUG - Progress: 4590/4590 bytes
+2026-02-12 01:53:34,646 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,648 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/rule.py')
+2026-02-12 01:53:34,650 - INFO - Upload completed successfully: rule.py
+2026-02-12 01:53:34,650 - INFO - Starting upload: lib/rich/repr.py -> /home/kevin/test/lib/rich/repr.py (4419 bytes)
+2026-02-12 01:53:34,651 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/repr.py', 'wb')
+2026-02-12 01:53:34,652 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/repr.py', 'wb') -> 00000000
+2026-02-12 01:53:34,652 - DEBUG - Progress: 4419/4419 bytes
+2026-02-12 01:53:34,652 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,653 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/repr.py')
+2026-02-12 01:53:34,655 - INFO - Upload completed successfully: repr.py
+2026-02-12 01:53:34,655 - INFO - Starting upload: lib/rich/region.py -> /home/kevin/test/lib/rich/region.py (166 bytes)
+2026-02-12 01:53:34,657 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/region.py', 'wb')
+2026-02-12 01:53:34,657 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/region.py', 'wb') -> 00000000
+2026-02-12 01:53:34,658 - DEBUG - Progress: 166/166 bytes
+2026-02-12 01:53:34,658 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,658 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/region.py')
+2026-02-12 01:53:34,660 - INFO - Upload completed successfully: region.py
+2026-02-12 01:53:34,660 - INFO - Starting upload: lib/rich/py.typed -> /home/kevin/test/lib/rich/py.typed (0 bytes)
+2026-02-12 01:53:34,662 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/py.typed', 'wb')
+2026-02-12 01:53:34,663 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/py.typed', 'wb') -> 00000000
+2026-02-12 01:53:34,663 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,663 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/py.typed')
+2026-02-12 01:53:34,665 - INFO - Upload completed successfully: py.typed
+2026-02-12 01:53:34,665 - INFO - Starting upload: lib/rich/protocol.py -> /home/kevin/test/lib/rich/protocol.py (1367 bytes)
+2026-02-12 01:53:34,666 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/protocol.py', 'wb')
+2026-02-12 01:53:34,667 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/protocol.py', 'wb') -> 00000000
+2026-02-12 01:53:34,667 - DEBUG - Progress: 1367/1367 bytes
+2026-02-12 01:53:34,667 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,668 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/protocol.py')
+2026-02-12 01:53:34,670 - INFO - Upload completed successfully: protocol.py
+2026-02-12 01:53:34,670 - INFO - Starting upload: lib/rich/prompt.py -> /home/kevin/test/lib/rich/prompt.py (12448 bytes)
+2026-02-12 01:53:34,671 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/prompt.py', 'wb')
+2026-02-12 01:53:34,672 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/prompt.py', 'wb') -> 00000000
+2026-02-12 01:53:34,672 - DEBUG - Progress: 12448/12448 bytes
+2026-02-12 01:53:34,672 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,676 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/prompt.py')
+2026-02-12 01:53:34,678 - INFO - Upload completed successfully: prompt.py
+2026-02-12 01:53:34,678 - INFO - Starting upload: lib/rich/progress_bar.py -> /home/kevin/test/lib/rich/progress_bar.py (8162 bytes)
+2026-02-12 01:53:34,680 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/progress_bar.py', 'wb')
+2026-02-12 01:53:34,680 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/progress_bar.py', 'wb') -> 00000000
+2026-02-12 01:53:34,680 - DEBUG - Progress: 8162/8162 bytes
+2026-02-12 01:53:34,680 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,682 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/progress_bar.py')
+2026-02-12 01:53:34,684 - INFO - Upload completed successfully: progress_bar.py
+2026-02-12 01:53:34,684 - INFO - Starting upload: lib/rich/progress.py -> /home/kevin/test/lib/rich/progress.py (60393 bytes)
+2026-02-12 01:53:34,686 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/progress.py', 'wb')
+2026-02-12 01:53:34,686 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/progress.py', 'wb') -> 00000000
+2026-02-12 01:53:34,687 - DEBUG - Progress: 32768/60393 bytes
+2026-02-12 01:53:34,687 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,694 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/progress.py')
+2026-02-12 01:53:34,696 - INFO - Upload completed successfully: progress.py
+2026-02-12 01:53:34,696 - INFO - Starting upload: lib/rich/pretty.py -> /home/kevin/test/lib/rich/pretty.py (36349 bytes)
+2026-02-12 01:53:34,698 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/pretty.py', 'wb')
+2026-02-12 01:53:34,698 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/pretty.py', 'wb') -> 00000000
+2026-02-12 01:53:34,699 - DEBUG - Progress: 32768/36349 bytes
+2026-02-12 01:53:34,699 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,703 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/pretty.py')
+2026-02-12 01:53:34,704 - INFO - Upload completed successfully: pretty.py
+2026-02-12 01:53:34,704 - INFO - Starting upload: lib/rich/panel.py -> /home/kevin/test/lib/rich/panel.py (11157 bytes)
+2026-02-12 01:53:34,706 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/panel.py', 'wb')
+2026-02-12 01:53:34,707 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/panel.py', 'wb') -> 00000000
+2026-02-12 01:53:34,707 - DEBUG - Progress: 11157/11157 bytes
+2026-02-12 01:53:34,707 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,708 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/panel.py')
+2026-02-12 01:53:34,710 - INFO - Upload completed successfully: panel.py
+2026-02-12 01:53:34,710 - INFO - Starting upload: lib/rich/palette.py -> /home/kevin/test/lib/rich/palette.py (3288 bytes)
+2026-02-12 01:53:34,711 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/palette.py', 'wb')
+2026-02-12 01:53:34,712 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/palette.py', 'wb') -> 00000000
+2026-02-12 01:53:34,712 - DEBUG - Progress: 3288/3288 bytes
+2026-02-12 01:53:34,712 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,713 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/palette.py')
+2026-02-12 01:53:34,715 - INFO - Upload completed successfully: palette.py
+2026-02-12 01:53:34,715 - INFO - Starting upload: lib/rich/pager.py -> /home/kevin/test/lib/rich/pager.py (828 bytes)
+2026-02-12 01:53:34,717 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/pager.py', 'wb')
+2026-02-12 01:53:34,717 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/pager.py', 'wb') -> 00000000
+2026-02-12 01:53:34,718 - DEBUG - Progress: 828/828 bytes
+2026-02-12 01:53:34,718 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,718 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/pager.py')
+2026-02-12 01:53:34,720 - INFO - Upload completed successfully: pager.py
+2026-02-12 01:53:34,720 - INFO - Starting upload: lib/rich/padding.py -> /home/kevin/test/lib/rich/padding.py (4896 bytes)
+2026-02-12 01:53:34,721 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/padding.py', 'wb')
+2026-02-12 01:53:34,722 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/padding.py', 'wb') -> 00000000
+2026-02-12 01:53:34,722 - DEBUG - Progress: 4896/4896 bytes
+2026-02-12 01:53:34,722 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,723 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/padding.py')
+2026-02-12 01:53:34,725 - INFO - Upload completed successfully: padding.py
+2026-02-12 01:53:34,725 - INFO - Starting upload: lib/rich/measure.py -> /home/kevin/test/lib/rich/measure.py (5305 bytes)
+2026-02-12 01:53:34,726 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/measure.py', 'wb')
+2026-02-12 01:53:34,727 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/measure.py', 'wb') -> 00000000
+2026-02-12 01:53:34,727 - DEBUG - Progress: 5305/5305 bytes
+2026-02-12 01:53:34,727 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,728 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/measure.py')
+2026-02-12 01:53:34,730 - INFO - Upload completed successfully: measure.py
+2026-02-12 01:53:34,730 - INFO - Starting upload: lib/rich/markup.py -> /home/kevin/test/lib/rich/markup.py (8427 bytes)
+2026-02-12 01:53:34,731 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/markup.py', 'wb')
+2026-02-12 01:53:34,732 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/markup.py', 'wb') -> 00000000
+2026-02-12 01:53:34,732 - DEBUG - Progress: 8427/8427 bytes
+2026-02-12 01:53:34,732 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,734 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/markup.py')
+2026-02-12 01:53:34,735 - INFO - Upload completed successfully: markup.py
+2026-02-12 01:53:34,735 - INFO - Starting upload: lib/rich/markdown.py -> /home/kevin/test/lib/rich/markdown.py (26177 bytes)
+2026-02-12 01:53:34,737 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/markdown.py', 'wb')
+2026-02-12 01:53:34,737 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/markdown.py', 'wb') -> 00000000
+2026-02-12 01:53:34,737 - DEBUG - Progress: 26177/26177 bytes
+2026-02-12 01:53:34,737 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,741 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/markdown.py')
+2026-02-12 01:53:34,743 - INFO - Upload completed successfully: markdown.py
+2026-02-12 01:53:34,743 - INFO - Starting upload: lib/rich/logging.py -> /home/kevin/test/lib/rich/logging.py (12456 bytes)
+2026-02-12 01:53:34,745 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/logging.py', 'wb')
+2026-02-12 01:53:34,745 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/logging.py', 'wb') -> 00000000
+2026-02-12 01:53:34,745 - DEBUG - Progress: 12456/12456 bytes
+2026-02-12 01:53:34,745 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,749 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/logging.py')
+2026-02-12 01:53:34,751 - INFO - Upload completed successfully: logging.py
+2026-02-12 01:53:34,751 - INFO - Starting upload: lib/rich/live_render.py -> /home/kevin/test/lib/rich/live_render.py (3803 bytes)
+2026-02-12 01:53:34,753 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/live_render.py', 'wb')
+2026-02-12 01:53:34,753 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/live_render.py', 'wb') -> 00000000
+2026-02-12 01:53:34,753 - DEBUG - Progress: 3803/3803 bytes
+2026-02-12 01:53:34,753 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,754 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/live_render.py')
+2026-02-12 01:53:34,756 - INFO - Upload completed successfully: live_render.py
+2026-02-12 01:53:34,756 - INFO - Starting upload: lib/rich/live.py -> /home/kevin/test/lib/rich/live.py (15317 bytes)
+2026-02-12 01:53:34,758 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/live.py', 'wb')
+2026-02-12 01:53:34,759 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/live.py', 'wb') -> 00000000
+2026-02-12 01:53:34,759 - DEBUG - Progress: 15317/15317 bytes
+2026-02-12 01:53:34,759 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,762 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/live.py')
+2026-02-12 01:53:34,764 - INFO - Upload completed successfully: live.py
+2026-02-12 01:53:34,764 - INFO - Starting upload: lib/rich/layout.py -> /home/kevin/test/lib/rich/layout.py (13944 bytes)
+2026-02-12 01:53:34,766 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/layout.py', 'wb')
+2026-02-12 01:53:34,766 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/layout.py', 'wb') -> 00000000
+2026-02-12 01:53:34,766 - DEBUG - Progress: 13944/13944 bytes
+2026-02-12 01:53:34,766 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,768 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/layout.py')
+2026-02-12 01:53:34,770 - INFO - Upload completed successfully: layout.py
+2026-02-12 01:53:34,770 - INFO - Starting upload: lib/rich/jupyter.py -> /home/kevin/test/lib/rich/jupyter.py (3228 bytes)
+2026-02-12 01:53:34,771 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/jupyter.py', 'wb')
+2026-02-12 01:53:34,772 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/jupyter.py', 'wb') -> 00000000
+2026-02-12 01:53:34,772 - DEBUG - Progress: 3228/3228 bytes
+2026-02-12 01:53:34,772 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,773 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/jupyter.py')
+2026-02-12 01:53:34,775 - INFO - Upload completed successfully: jupyter.py
+2026-02-12 01:53:34,775 - INFO - Starting upload: lib/rich/json.py -> /home/kevin/test/lib/rich/json.py (5019 bytes)
+2026-02-12 01:53:34,776 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/json.py', 'wb')
+2026-02-12 01:53:34,777 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/json.py', 'wb') -> 00000000
+2026-02-12 01:53:34,777 - DEBUG - Progress: 5019/5019 bytes
+2026-02-12 01:53:34,777 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,778 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/json.py')
+2026-02-12 01:53:34,780 - INFO - Upload completed successfully: json.py
+2026-02-12 01:53:34,780 - INFO - Starting upload: lib/rich/highlighter.py -> /home/kevin/test/lib/rich/highlighter.py (9729 bytes)
+2026-02-12 01:53:34,782 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/highlighter.py', 'wb')
+2026-02-12 01:53:34,782 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/highlighter.py', 'wb') -> 00000000
+2026-02-12 01:53:34,783 - DEBUG - Progress: 9729/9729 bytes
+2026-02-12 01:53:34,783 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,784 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/highlighter.py')
+2026-02-12 01:53:34,786 - INFO - Upload completed successfully: highlighter.py
+2026-02-12 01:53:34,786 - INFO - Starting upload: lib/rich/filesize.py -> /home/kevin/test/lib/rich/filesize.py (2484 bytes)
+2026-02-12 01:53:34,787 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/filesize.py', 'wb')
+2026-02-12 01:53:34,788 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/filesize.py', 'wb') -> 00000000
+2026-02-12 01:53:34,788 - DEBUG - Progress: 2484/2484 bytes
+2026-02-12 01:53:34,788 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,789 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/filesize.py')
+2026-02-12 01:53:34,791 - INFO - Upload completed successfully: filesize.py
+2026-02-12 01:53:34,791 - INFO - Starting upload: lib/rich/file_proxy.py -> /home/kevin/test/lib/rich/file_proxy.py (1683 bytes)
+2026-02-12 01:53:34,792 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/file_proxy.py', 'wb')
+2026-02-12 01:53:34,793 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/file_proxy.py', 'wb') -> 00000000
+2026-02-12 01:53:34,793 - DEBUG - Progress: 1683/1683 bytes
+2026-02-12 01:53:34,793 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,794 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/file_proxy.py')
+2026-02-12 01:53:34,796 - INFO - Upload completed successfully: file_proxy.py
+2026-02-12 01:53:34,796 - INFO - Starting upload: lib/rich/errors.py -> /home/kevin/test/lib/rich/errors.py (642 bytes)
+2026-02-12 01:53:34,797 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/errors.py', 'wb')
+2026-02-12 01:53:34,798 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/errors.py', 'wb') -> 00000000
+2026-02-12 01:53:34,798 - DEBUG - Progress: 642/642 bytes
+2026-02-12 01:53:34,798 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,799 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/errors.py')
+2026-02-12 01:53:34,800 - INFO - Upload completed successfully: errors.py
+2026-02-12 01:53:34,801 - INFO - Starting upload: lib/rich/emoji.py -> /home/kevin/test/lib/rich/emoji.py (2343 bytes)
+2026-02-12 01:53:34,802 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/emoji.py', 'wb')
+2026-02-12 01:53:34,803 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/emoji.py', 'wb') -> 00000000
+2026-02-12 01:53:34,803 - DEBUG - Progress: 2343/2343 bytes
+2026-02-12 01:53:34,803 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,804 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/emoji.py')
+2026-02-12 01:53:34,806 - INFO - Upload completed successfully: emoji.py
+2026-02-12 01:53:34,806 - INFO - Starting upload: lib/rich/diagnose.py -> /home/kevin/test/lib/rich/diagnose.py (977 bytes)
+2026-02-12 01:53:34,808 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/diagnose.py', 'wb')
+2026-02-12 01:53:34,809 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/diagnose.py', 'wb') -> 00000000
+2026-02-12 01:53:34,809 - DEBUG - Progress: 977/977 bytes
+2026-02-12 01:53:34,809 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,810 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/diagnose.py')
+2026-02-12 01:53:34,812 - INFO - Upload completed successfully: diagnose.py
+2026-02-12 01:53:34,812 - INFO - Starting upload: lib/rich/default_styles.py -> /home/kevin/test/lib/rich/default_styles.py (8340 bytes)
+2026-02-12 01:53:34,814 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/default_styles.py', 'wb')
+2026-02-12 01:53:34,814 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/default_styles.py', 'wb') -> 00000000
+2026-02-12 01:53:34,815 - DEBUG - Progress: 8340/8340 bytes
+2026-02-12 01:53:34,815 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,816 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/default_styles.py')
+2026-02-12 01:53:34,818 - INFO - Upload completed successfully: default_styles.py
+2026-02-12 01:53:34,819 - INFO - Starting upload: lib/rich/control.py -> /home/kevin/test/lib/rich/control.py (6475 bytes)
+2026-02-12 01:53:34,820 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/control.py', 'wb')
+2026-02-12 01:53:34,821 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/control.py', 'wb') -> 00000000
+2026-02-12 01:53:34,821 - DEBUG - Progress: 6475/6475 bytes
+2026-02-12 01:53:34,822 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,823 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/control.py')
+2026-02-12 01:53:34,824 - INFO - Upload completed successfully: control.py
+2026-02-12 01:53:34,825 - INFO - Starting upload: lib/rich/containers.py -> /home/kevin/test/lib/rich/containers.py (5502 bytes)
+2026-02-12 01:53:34,826 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/containers.py', 'wb')
+2026-02-12 01:53:34,827 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/containers.py', 'wb') -> 00000000
+2026-02-12 01:53:34,827 - DEBUG - Progress: 5502/5502 bytes
+2026-02-12 01:53:34,827 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,828 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/containers.py')
+2026-02-12 01:53:34,831 - INFO - Upload completed successfully: containers.py
+2026-02-12 01:53:34,831 - INFO - Starting upload: lib/rich/constrain.py -> /home/kevin/test/lib/rich/constrain.py (1288 bytes)
+2026-02-12 01:53:34,833 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/constrain.py', 'wb')
+2026-02-12 01:53:34,834 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/constrain.py', 'wb') -> 00000000
+2026-02-12 01:53:34,834 - DEBUG - Progress: 1288/1288 bytes
+2026-02-12 01:53:34,834 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,835 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/constrain.py')
+2026-02-12 01:53:34,837 - INFO - Upload completed successfully: constrain.py
+2026-02-12 01:53:34,838 - INFO - Starting upload: lib/rich/console.py -> /home/kevin/test/lib/rich/console.py (101009 bytes)
+2026-02-12 01:53:34,840 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/console.py', 'wb')
+2026-02-12 01:53:34,841 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/console.py', 'wb') -> 00000000
+2026-02-12 01:53:34,841 - DEBUG - Progress: 32768/101009 bytes
+2026-02-12 01:53:34,842 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,852 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/console.py')
+2026-02-12 01:53:34,854 - INFO - Upload completed successfully: console.py
+2026-02-12 01:53:34,855 - INFO - Starting upload: lib/rich/columns.py -> /home/kevin/test/lib/rich/columns.py (7131 bytes)
+2026-02-12 01:53:34,856 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/columns.py', 'wb')
+2026-02-12 01:53:34,857 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/columns.py', 'wb') -> 00000000
+2026-02-12 01:53:34,857 - DEBUG - Progress: 7131/7131 bytes
+2026-02-12 01:53:34,857 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,859 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/columns.py')
+2026-02-12 01:53:34,861 - INFO - Upload completed successfully: columns.py
+2026-02-12 01:53:34,861 - INFO - Starting upload: lib/rich/color_triplet.py -> /home/kevin/test/lib/rich/color_triplet.py (1054 bytes)
+2026-02-12 01:53:34,863 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/color_triplet.py', 'wb')
+2026-02-12 01:53:34,863 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/color_triplet.py', 'wb') -> 00000000
+2026-02-12 01:53:34,864 - DEBUG - Progress: 1054/1054 bytes
+2026-02-12 01:53:34,864 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,864 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/color_triplet.py')
+2026-02-12 01:53:34,866 - INFO - Upload completed successfully: color_triplet.py
+2026-02-12 01:53:34,866 - INFO - Starting upload: lib/rich/color.py -> /home/kevin/test/lib/rich/color.py (18211 bytes)
+2026-02-12 01:53:34,868 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/color.py', 'wb')
+2026-02-12 01:53:34,869 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/color.py', 'wb') -> 00000000
+2026-02-12 01:53:34,869 - DEBUG - Progress: 18211/18211 bytes
+2026-02-12 01:53:34,869 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,871 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/color.py')
+2026-02-12 01:53:34,874 - INFO - Upload completed successfully: color.py
+2026-02-12 01:53:34,874 - INFO - Starting upload: lib/rich/cells.py -> /home/kevin/test/lib/rich/cells.py (10850 bytes)
+2026-02-12 01:53:34,875 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/cells.py', 'wb')
+2026-02-12 01:53:34,876 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/cells.py', 'wb') -> 00000000
+2026-02-12 01:53:34,876 - DEBUG - Progress: 10850/10850 bytes
+2026-02-12 01:53:34,876 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,878 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/cells.py')
+2026-02-12 01:53:34,880 - INFO - Upload completed successfully: cells.py
+2026-02-12 01:53:34,880 - INFO - Starting upload: lib/rich/box.py -> /home/kevin/test/lib/rich/box.py (10650 bytes)
+2026-02-12 01:53:34,882 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/box.py', 'wb')
+2026-02-12 01:53:34,883 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/box.py', 'wb') -> 00000000
+2026-02-12 01:53:34,883 - DEBUG - Progress: 10650/10650 bytes
+2026-02-12 01:53:34,883 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,884 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/box.py')
+2026-02-12 01:53:34,887 - INFO - Upload completed successfully: box.py
+2026-02-12 01:53:34,887 - INFO - Starting upload: lib/rich/bar.py -> /home/kevin/test/lib/rich/bar.py (3263 bytes)
+2026-02-12 01:53:34,889 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/bar.py', 'wb')
+2026-02-12 01:53:34,889 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/bar.py', 'wb') -> 00000000
+2026-02-12 01:53:34,889 - DEBUG - Progress: 3263/3263 bytes
+2026-02-12 01:53:34,890 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,891 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/bar.py')
+2026-02-12 01:53:34,893 - INFO - Upload completed successfully: bar.py
+2026-02-12 01:53:34,893 - INFO - Starting upload: lib/rich/ansi.py -> /home/kevin/test/lib/rich/ansi.py (6921 bytes)
+2026-02-12 01:53:34,895 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/ansi.py', 'wb')
+2026-02-12 01:53:34,896 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/ansi.py', 'wb') -> 00000000
+2026-02-12 01:53:34,896 - DEBUG - Progress: 6921/6921 bytes
+2026-02-12 01:53:34,896 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,897 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/ansi.py')
+2026-02-12 01:53:34,899 - INFO - Upload completed successfully: ansi.py
+2026-02-12 01:53:34,899 - INFO - Starting upload: lib/rich/align.py -> /home/kevin/test/lib/rich/align.py (10726 bytes)
+2026-02-12 01:53:34,901 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/align.py', 'wb')
+2026-02-12 01:53:34,902 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/align.py', 'wb') -> 00000000
+2026-02-12 01:53:34,902 - DEBUG - Progress: 10726/10726 bytes
+2026-02-12 01:53:34,902 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,904 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/align.py')
+2026-02-12 01:53:34,906 - INFO - Upload completed successfully: align.py
+2026-02-12 01:53:34,906 - INFO - Starting upload: lib/rich/abc.py -> /home/kevin/test/lib/rich/abc.py (878 bytes)
+2026-02-12 01:53:34,908 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/abc.py', 'wb')
+2026-02-12 01:53:34,909 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/abc.py', 'wb') -> 00000000
+2026-02-12 01:53:34,909 - DEBUG - Progress: 878/878 bytes
+2026-02-12 01:53:34,909 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,910 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/abc.py')
+2026-02-12 01:53:34,911 - INFO - Upload completed successfully: abc.py
+2026-02-12 01:53:34,912 - INFO - Starting upload: lib/rich/_wrap.py -> /home/kevin/test/lib/rich/_wrap.py (3404 bytes)
+2026-02-12 01:53:34,913 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_wrap.py', 'wb')
+2026-02-12 01:53:34,914 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_wrap.py', 'wb') -> 00000000
+2026-02-12 01:53:34,914 - DEBUG - Progress: 3404/3404 bytes
+2026-02-12 01:53:34,914 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,916 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_wrap.py')
+2026-02-12 01:53:34,918 - INFO - Upload completed successfully: _wrap.py
+2026-02-12 01:53:34,918 - INFO - Starting upload: lib/rich/_windows_renderer.py -> /home/kevin/test/lib/rich/_windows_renderer.py (2759 bytes)
+2026-02-12 01:53:34,920 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_windows_renderer.py', 'wb')
+2026-02-12 01:53:34,920 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_windows_renderer.py', 'wb') -> 00000000
+2026-02-12 01:53:34,921 - DEBUG - Progress: 2759/2759 bytes
+2026-02-12 01:53:34,921 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,922 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_windows_renderer.py')
+2026-02-12 01:53:34,924 - INFO - Upload completed successfully: _windows_renderer.py
+2026-02-12 01:53:34,924 - INFO - Starting upload: lib/rich/_windows.py -> /home/kevin/test/lib/rich/_windows.py (1901 bytes)
+2026-02-12 01:53:34,926 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_windows.py', 'wb')
+2026-02-12 01:53:34,927 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_windows.py', 'wb') -> 00000000
+2026-02-12 01:53:34,927 - DEBUG - Progress: 1901/1901 bytes
+2026-02-12 01:53:34,927 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,928 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_windows.py')
+2026-02-12 01:53:34,931 - INFO - Upload completed successfully: _windows.py
+2026-02-12 01:53:34,931 - INFO - Starting upload: lib/rich/_win32_console.py -> /home/kevin/test/lib/rich/_win32_console.py (22719 bytes)
+2026-02-12 01:53:34,932 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_win32_console.py', 'wb')
+2026-02-12 01:53:34,933 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_win32_console.py', 'wb') -> 00000000
+2026-02-12 01:53:34,933 - DEBUG - Progress: 22719/22719 bytes
+2026-02-12 01:53:34,933 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,937 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_win32_console.py')
+2026-02-12 01:53:34,939 - INFO - Upload completed successfully: _win32_console.py
+2026-02-12 01:53:34,939 - INFO - Starting upload: lib/rich/_timer.py -> /home/kevin/test/lib/rich/_timer.py (417 bytes)
+2026-02-12 01:53:34,941 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_timer.py', 'wb')
+2026-02-12 01:53:34,942 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_timer.py', 'wb') -> 00000000
+2026-02-12 01:53:34,942 - DEBUG - Progress: 417/417 bytes
+2026-02-12 01:53:34,942 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,943 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_timer.py')
+2026-02-12 01:53:34,945 - INFO - Upload completed successfully: _timer.py
+2026-02-12 01:53:34,945 - INFO - Starting upload: lib/rich/_stack.py -> /home/kevin/test/lib/rich/_stack.py (351 bytes)
+2026-02-12 01:53:34,946 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_stack.py', 'wb')
+2026-02-12 01:53:34,947 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_stack.py', 'wb') -> 00000000
+2026-02-12 01:53:34,947 - DEBUG - Progress: 351/351 bytes
+2026-02-12 01:53:34,947 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,948 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_stack.py')
+2026-02-12 01:53:34,950 - INFO - Upload completed successfully: _stack.py
+2026-02-12 01:53:34,950 - INFO - Starting upload: lib/rich/_spinners.py -> /home/kevin/test/lib/rich/_spinners.py (19919 bytes)
+2026-02-12 01:53:34,951 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_spinners.py', 'wb')
+2026-02-12 01:53:34,952 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_spinners.py', 'wb') -> 00000000
+2026-02-12 01:53:34,952 - DEBUG - Progress: 19919/19919 bytes
+2026-02-12 01:53:34,952 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,956 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_spinners.py')
+2026-02-12 01:53:34,958 - INFO - Upload completed successfully: _spinners.py
+2026-02-12 01:53:34,958 - INFO - Starting upload: lib/rich/_ratio.py -> /home/kevin/test/lib/rich/_ratio.py (5325 bytes)
+2026-02-12 01:53:34,960 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_ratio.py', 'wb')
+2026-02-12 01:53:34,961 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_ratio.py', 'wb') -> 00000000
+2026-02-12 01:53:34,961 - DEBUG - Progress: 5325/5325 bytes
+2026-02-12 01:53:34,961 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,962 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_ratio.py')
+2026-02-12 01:53:34,964 - INFO - Upload completed successfully: _ratio.py
+2026-02-12 01:53:34,964 - INFO - Starting upload: lib/rich/_pick.py -> /home/kevin/test/lib/rich/_pick.py (423 bytes)
+2026-02-12 01:53:34,966 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_pick.py', 'wb')
+2026-02-12 01:53:34,966 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_pick.py', 'wb') -> 00000000
+2026-02-12 01:53:34,967 - DEBUG - Progress: 423/423 bytes
+2026-02-12 01:53:34,967 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,967 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_pick.py')
+2026-02-12 01:53:34,970 - INFO - Upload completed successfully: _pick.py
+2026-02-12 01:53:34,970 - INFO - Starting upload: lib/rich/_palettes.py -> /home/kevin/test/lib/rich/_palettes.py (7063 bytes)
+2026-02-12 01:53:34,972 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_palettes.py', 'wb')
+2026-02-12 01:53:34,972 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_palettes.py', 'wb') -> 00000000
+2026-02-12 01:53:34,973 - DEBUG - Progress: 7063/7063 bytes
+2026-02-12 01:53:34,973 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,974 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_palettes.py')
+2026-02-12 01:53:34,976 - INFO - Upload completed successfully: _palettes.py
+2026-02-12 01:53:34,976 - INFO - Starting upload: lib/rich/_null_file.py -> /home/kevin/test/lib/rich/_null_file.py (1394 bytes)
+2026-02-12 01:53:34,978 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_null_file.py', 'wb')
+2026-02-12 01:53:34,979 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_null_file.py', 'wb') -> 00000000
+2026-02-12 01:53:34,979 - DEBUG - Progress: 1394/1394 bytes
+2026-02-12 01:53:34,979 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,980 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_null_file.py')
+2026-02-12 01:53:34,982 - INFO - Upload completed successfully: _null_file.py
+2026-02-12 01:53:34,982 - INFO - Starting upload: lib/rich/_loop.py -> /home/kevin/test/lib/rich/_loop.py (1236 bytes)
+2026-02-12 01:53:34,983 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_loop.py', 'wb')
+2026-02-12 01:53:34,984 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_loop.py', 'wb') -> 00000000
+2026-02-12 01:53:34,984 - DEBUG - Progress: 1236/1236 bytes
+2026-02-12 01:53:34,984 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,985 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_loop.py')
+2026-02-12 01:53:34,986 - INFO - Upload completed successfully: _loop.py
+2026-02-12 01:53:34,987 - INFO - Starting upload: lib/rich/_log_render.py -> /home/kevin/test/lib/rich/_log_render.py (3213 bytes)
+2026-02-12 01:53:34,988 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_log_render.py', 'wb')
+2026-02-12 01:53:34,989 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_log_render.py', 'wb') -> 00000000
+2026-02-12 01:53:34,989 - DEBUG - Progress: 3213/3213 bytes
+2026-02-12 01:53:34,989 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,990 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_log_render.py')
+2026-02-12 01:53:34,992 - INFO - Upload completed successfully: _log_render.py
+2026-02-12 01:53:34,992 - INFO - Starting upload: lib/rich/_inspect.py -> /home/kevin/test/lib/rich/_inspect.py (9894 bytes)
+2026-02-12 01:53:34,993 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_inspect.py', 'wb')
+2026-02-12 01:53:34,994 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_inspect.py', 'wb') -> 00000000
+2026-02-12 01:53:34,994 - DEBUG - Progress: 9894/9894 bytes
+2026-02-12 01:53:34,994 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:34,996 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_inspect.py')
+2026-02-12 01:53:34,998 - INFO - Upload completed successfully: _inspect.py
+2026-02-12 01:53:34,998 - INFO - Starting upload: lib/rich/_fileno.py -> /home/kevin/test/lib/rich/_fileno.py (799 bytes)
+2026-02-12 01:53:34,999 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_fileno.py', 'wb')
+2026-02-12 01:53:35,000 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_fileno.py', 'wb') -> 00000000
+2026-02-12 01:53:35,000 - DEBUG - Progress: 799/799 bytes
+2026-02-12 01:53:35,000 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,001 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_fileno.py')
+2026-02-12 01:53:35,002 - INFO - Upload completed successfully: _fileno.py
+2026-02-12 01:53:35,002 - INFO - Starting upload: lib/rich/_extension.py -> /home/kevin/test/lib/rich/_extension.py (241 bytes)
+2026-02-12 01:53:35,004 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_extension.py', 'wb')
+2026-02-12 01:53:35,004 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_extension.py', 'wb') -> 00000000
+2026-02-12 01:53:35,004 - DEBUG - Progress: 241/241 bytes
+2026-02-12 01:53:35,004 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,005 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_extension.py')
+2026-02-12 01:53:35,007 - INFO - Upload completed successfully: _extension.py
+2026-02-12 01:53:35,007 - INFO - Starting upload: lib/rich/_export_format.py -> /home/kevin/test/lib/rich/_export_format.py (2128 bytes)
+2026-02-12 01:53:35,008 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_export_format.py', 'wb')
+2026-02-12 01:53:35,009 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_export_format.py', 'wb') -> 00000000
+2026-02-12 01:53:35,009 - DEBUG - Progress: 2128/2128 bytes
+2026-02-12 01:53:35,009 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,010 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_export_format.py')
+2026-02-12 01:53:35,012 - INFO - Upload completed successfully: _export_format.py
+2026-02-12 01:53:35,012 - INFO - Starting upload: lib/rich/_emoji_replace.py -> /home/kevin/test/lib/rich/_emoji_replace.py (1064 bytes)
+2026-02-12 01:53:35,014 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_emoji_replace.py', 'wb')
+2026-02-12 01:53:35,014 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_emoji_replace.py', 'wb') -> 00000000
+2026-02-12 01:53:35,014 - DEBUG - Progress: 1064/1064 bytes
+2026-02-12 01:53:35,014 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,015 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_emoji_replace.py')
+2026-02-12 01:53:35,017 - INFO - Upload completed successfully: _emoji_replace.py
+2026-02-12 01:53:35,017 - INFO - Starting upload: lib/rich/_emoji_codes.py -> /home/kevin/test/lib/rich/_emoji_codes.py (140235 bytes)
+2026-02-12 01:53:35,018 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_emoji_codes.py', 'wb')
+2026-02-12 01:53:35,019 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_emoji_codes.py', 'wb') -> 00000000
+2026-02-12 01:53:35,019 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,034 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_emoji_codes.py')
+2026-02-12 01:53:35,037 - INFO - Upload completed successfully: _emoji_codes.py
+2026-02-12 01:53:35,037 - INFO - Starting upload: lib/rich/__main__.py -> /home/kevin/test/lib/rich/__main__.py (7725 bytes)
+2026-02-12 01:53:35,039 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__main__.py', 'wb')
+2026-02-12 01:53:35,040 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__main__.py', 'wb') -> 00000000
+2026-02-12 01:53:35,040 - DEBUG - Progress: 7725/7725 bytes
+2026-02-12 01:53:35,040 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,042 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__main__.py')
+2026-02-12 01:53:35,044 - INFO - Upload completed successfully: __main__.py
+2026-02-12 01:53:35,044 - INFO - Starting upload: lib/rich/__init__.py -> /home/kevin/test/lib/rich/__init__.py (6131 bytes)
+2026-02-12 01:53:35,046 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__init__.py', 'wb')
+2026-02-12 01:53:35,046 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:35,047 - DEBUG - Progress: 6131/6131 bytes
+2026-02-12 01:53:35,047 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,048 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__init__.py')
+2026-02-12 01:53:35,050 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:35,051 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/rich/__pycache__', 511)
+2026-02-12 01:53:35,051 - INFO - Created remote folder: /home/kevin/test/lib/rich/__pycache__
+2026-02-12 01:53:35,051 - INFO - Starting upload: lib/rich/__pycache__/tree.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/tree.cpython-314.pyc (13118 bytes)
+2026-02-12 01:53:35,053 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/tree.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,054 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/tree.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,054 - DEBUG - Progress: 13118/13118 bytes
+2026-02-12 01:53:35,054 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,057 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/tree.cpython-314.pyc')
+2026-02-12 01:53:35,058 - INFO - Upload completed successfully: tree.cpython-314.pyc
+2026-02-12 01:53:35,059 - INFO - Starting upload: lib/rich/__pycache__/traceback.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/traceback.cpython-314.pyc (44364 bytes)
+2026-02-12 01:53:35,061 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/traceback.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,061 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/traceback.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,062 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,067 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/traceback.cpython-314.pyc')
+2026-02-12 01:53:35,069 - INFO - Upload completed successfully: traceback.cpython-314.pyc
+2026-02-12 01:53:35,069 - INFO - Starting upload: lib/rich/__pycache__/themes.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/themes.cpython-314.pyc (278 bytes)
+2026-02-12 01:53:35,071 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/themes.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,072 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/themes.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,072 - DEBUG - Progress: 278/278 bytes
+2026-02-12 01:53:35,072 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,073 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/themes.cpython-314.pyc')
+2026-02-12 01:53:35,077 - INFO - Upload completed successfully: themes.cpython-314.pyc
+2026-02-12 01:53:35,077 - INFO - Starting upload: lib/rich/__pycache__/theme.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/theme.cpython-314.pyc (7755 bytes)
+2026-02-12 01:53:35,081 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/theme.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,082 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/theme.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,082 - DEBUG - Progress: 7755/7755 bytes
+2026-02-12 01:53:35,082 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,084 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/theme.cpython-314.pyc')
+2026-02-12 01:53:35,086 - INFO - Upload completed successfully: theme.cpython-314.pyc
+2026-02-12 01:53:35,086 - INFO - Starting upload: lib/rich/__pycache__/text.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/text.cpython-314.pyc (73093 bytes)
+2026-02-12 01:53:35,089 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/text.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,090 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/text.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,091 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,098 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/text.cpython-314.pyc')
+2026-02-12 01:53:35,102 - INFO - Upload completed successfully: text.cpython-314.pyc
+2026-02-12 01:53:35,102 - INFO - Starting upload: lib/rich/__pycache__/terminal_theme.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/terminal_theme.cpython-314.pyc (3639 bytes)
+2026-02-12 01:53:35,106 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/terminal_theme.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,107 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/terminal_theme.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,108 - DEBUG - Progress: 3639/3639 bytes
+2026-02-12 01:53:35,108 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,109 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/terminal_theme.cpython-314.pyc')
+2026-02-12 01:53:35,113 - INFO - Upload completed successfully: terminal_theme.cpython-314.pyc
+2026-02-12 01:53:35,113 - INFO - Starting upload: lib/rich/__pycache__/table.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/table.cpython-314.pyc (51512 bytes)
+2026-02-12 01:53:35,117 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/table.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,118 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/table.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,119 - DEBUG - Progress: 32768/51512 bytes
+2026-02-12 01:53:35,119 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,125 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/table.cpython-314.pyc')
+2026-02-12 01:53:35,128 - INFO - Upload completed successfully: table.cpython-314.pyc
+2026-02-12 01:53:35,128 - INFO - Starting upload: lib/rich/__pycache__/syntax.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/syntax.cpython-314.pyc (46071 bytes)
+2026-02-12 01:53:35,130 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/syntax.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,131 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/syntax.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,132 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,136 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/syntax.cpython-314.pyc')
+2026-02-12 01:53:35,140 - INFO - Upload completed successfully: syntax.cpython-314.pyc
+2026-02-12 01:53:35,140 - INFO - Starting upload: lib/rich/__pycache__/styled.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/styled.cpython-314.pyc (2644 bytes)
+2026-02-12 01:53:35,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/styled.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,145 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/styled.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,146 - DEBUG - Progress: 2644/2644 bytes
+2026-02-12 01:53:35,146 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,147 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/styled.cpython-314.pyc')
+2026-02-12 01:53:35,149 - INFO - Upload completed successfully: styled.cpython-314.pyc
+2026-02-12 01:53:35,149 - INFO - Starting upload: lib/rich/__pycache__/style.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/style.cpython-314.pyc (42013 bytes)
+2026-02-12 01:53:35,151 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/style.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,152 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/style.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,152 - DEBUG - Progress: 32768/42013 bytes
+2026-02-12 01:53:35,152 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,160 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/style.cpython-314.pyc')
+2026-02-12 01:53:35,162 - INFO - Upload completed successfully: style.cpython-314.pyc
+2026-02-12 01:53:35,162 - INFO - Starting upload: lib/rich/__pycache__/status.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/status.cpython-314.pyc (7603 bytes)
+2026-02-12 01:53:35,164 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/status.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,165 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/status.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,165 - DEBUG - Progress: 7603/7603 bytes
+2026-02-12 01:53:35,165 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,167 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/status.cpython-314.pyc')
+2026-02-12 01:53:35,170 - INFO - Upload completed successfully: status.cpython-314.pyc
+2026-02-12 01:53:35,171 - INFO - Starting upload: lib/rich/__pycache__/spinner.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/spinner.cpython-314.pyc (7001 bytes)
+2026-02-12 01:53:35,174 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/spinner.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,175 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/spinner.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,176 - DEBUG - Progress: 7001/7001 bytes
+2026-02-12 01:53:35,176 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,177 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/spinner.cpython-314.pyc')
+2026-02-12 01:53:35,181 - INFO - Upload completed successfully: spinner.cpython-314.pyc
+2026-02-12 01:53:35,181 - INFO - Starting upload: lib/rich/__pycache__/segment.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/segment.cpython-314.pyc (34877 bytes)
+2026-02-12 01:53:35,185 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/segment.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,186 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/segment.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,187 - DEBUG - Progress: 32768/34877 bytes
+2026-02-12 01:53:35,187 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,191 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/segment.cpython-314.pyc')
+2026-02-12 01:53:35,192 - INFO - Upload completed successfully: segment.cpython-314.pyc
+2026-02-12 01:53:35,192 - INFO - Starting upload: lib/rich/__pycache__/screen.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/screen.cpython-314.pyc (3060 bytes)
+2026-02-12 01:53:35,194 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/screen.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,195 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/screen.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,195 - DEBUG - Progress: 3060/3060 bytes
+2026-02-12 01:53:35,195 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,196 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/screen.cpython-314.pyc')
+2026-02-12 01:53:35,198 - INFO - Upload completed successfully: screen.cpython-314.pyc
+2026-02-12 01:53:35,198 - INFO - Starting upload: lib/rich/__pycache__/scope.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/scope.cpython-314.pyc (4810 bytes)
+2026-02-12 01:53:35,200 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/scope.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,201 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/scope.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,201 - DEBUG - Progress: 4810/4810 bytes
+2026-02-12 01:53:35,201 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,202 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/scope.cpython-314.pyc')
+2026-02-12 01:53:35,204 - INFO - Upload completed successfully: scope.cpython-314.pyc
+2026-02-12 01:53:35,204 - INFO - Starting upload: lib/rich/__pycache__/rule.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/rule.cpython-314.pyc (7649 bytes)
+2026-02-12 01:53:35,205 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/rule.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,206 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/rule.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,206 - DEBUG - Progress: 7649/7649 bytes
+2026-02-12 01:53:35,206 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,208 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/rule.cpython-314.pyc')
+2026-02-12 01:53:35,210 - INFO - Upload completed successfully: rule.cpython-314.pyc
+2026-02-12 01:53:35,210 - INFO - Starting upload: lib/rich/__pycache__/repr.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/repr.cpython-314.pyc (8820 bytes)
+2026-02-12 01:53:35,211 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/repr.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,212 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/repr.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,212 - DEBUG - Progress: 8820/8820 bytes
+2026-02-12 01:53:35,212 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,214 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/repr.cpython-314.pyc')
+2026-02-12 01:53:35,216 - INFO - Upload completed successfully: repr.cpython-314.pyc
+2026-02-12 01:53:35,216 - INFO - Starting upload: lib/rich/__pycache__/region.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/region.cpython-314.pyc (828 bytes)
+2026-02-12 01:53:35,217 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/region.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,218 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/region.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,218 - DEBUG - Progress: 828/828 bytes
+2026-02-12 01:53:35,218 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,219 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/region.cpython-314.pyc')
+2026-02-12 01:53:35,221 - INFO - Upload completed successfully: region.cpython-314.pyc
+2026-02-12 01:53:35,221 - INFO - Starting upload: lib/rich/__pycache__/protocol.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/protocol.cpython-314.pyc (2022 bytes)
+2026-02-12 01:53:35,222 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/protocol.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,223 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/protocol.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,223 - DEBUG - Progress: 2022/2022 bytes
+2026-02-12 01:53:35,223 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,225 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/protocol.cpython-314.pyc')
+2026-02-12 01:53:35,226 - INFO - Upload completed successfully: protocol.cpython-314.pyc
+2026-02-12 01:53:35,226 - INFO - Starting upload: lib/rich/__pycache__/prompt.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/prompt.cpython-314.pyc (19762 bytes)
+2026-02-12 01:53:35,228 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/prompt.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,228 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/prompt.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,229 - DEBUG - Progress: 19762/19762 bytes
+2026-02-12 01:53:35,229 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,233 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/prompt.cpython-314.pyc')
+2026-02-12 01:53:35,235 - INFO - Upload completed successfully: prompt.cpython-314.pyc
+2026-02-12 01:53:35,236 - INFO - Starting upload: lib/rich/__pycache__/progress_bar.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/progress_bar.cpython-314.pyc (12150 bytes)
+2026-02-12 01:53:35,237 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/progress_bar.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,238 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/progress_bar.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,238 - DEBUG - Progress: 12150/12150 bytes
+2026-02-12 01:53:35,238 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,240 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/progress_bar.cpython-314.pyc')
+2026-02-12 01:53:35,242 - INFO - Upload completed successfully: progress_bar.cpython-314.pyc
+2026-02-12 01:53:35,242 - INFO - Starting upload: lib/rich/__pycache__/progress.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/progress.cpython-314.pyc (88705 bytes)
+2026-02-12 01:53:35,246 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/progress.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,246 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/progress.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,248 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,256 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/progress.cpython-314.pyc')
+2026-02-12 01:53:35,260 - INFO - Upload completed successfully: progress.cpython-314.pyc
+2026-02-12 01:53:35,260 - INFO - Starting upload: lib/rich/__pycache__/pretty.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/pretty.cpython-314.pyc (49494 bytes)
+2026-02-12 01:53:35,264 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/pretty.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,265 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/pretty.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,266 - DEBUG - Progress: 32768/49494 bytes
+2026-02-12 01:53:35,266 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,271 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/pretty.cpython-314.pyc')
+2026-02-12 01:53:35,275 - INFO - Upload completed successfully: pretty.cpython-314.pyc
+2026-02-12 01:53:35,275 - INFO - Starting upload: lib/rich/__pycache__/panel.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/panel.cpython-314.pyc (14465 bytes)
+2026-02-12 01:53:35,278 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/panel.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,278 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/panel.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,279 - DEBUG - Progress: 14465/14465 bytes
+2026-02-12 01:53:35,279 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,282 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/panel.cpython-314.pyc')
+2026-02-12 01:53:35,286 - INFO - Upload completed successfully: panel.cpython-314.pyc
+2026-02-12 01:53:35,286 - INFO - Starting upload: lib/rich/__pycache__/palette.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/palette.cpython-314.pyc (6483 bytes)
+2026-02-12 01:53:35,290 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/palette.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,291 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/palette.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,291 - DEBUG - Progress: 6483/6483 bytes
+2026-02-12 01:53:35,292 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,293 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/palette.cpython-314.pyc')
+2026-02-12 01:53:35,297 - INFO - Upload completed successfully: palette.cpython-314.pyc
+2026-02-12 01:53:35,297 - INFO - Starting upload: lib/rich/__pycache__/pager.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/pager.cpython-314.pyc (2401 bytes)
+2026-02-12 01:53:35,300 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/pager.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,301 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/pager.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,301 - DEBUG - Progress: 2401/2401 bytes
+2026-02-12 01:53:35,301 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,303 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/pager.cpython-314.pyc')
+2026-02-12 01:53:35,306 - INFO - Upload completed successfully: pager.cpython-314.pyc
+2026-02-12 01:53:35,307 - INFO - Starting upload: lib/rich/__pycache__/padding.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/padding.cpython-314.pyc (8101 bytes)
+2026-02-12 01:53:35,309 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/padding.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,310 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/padding.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,310 - DEBUG - Progress: 8101/8101 bytes
+2026-02-12 01:53:35,310 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,312 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/padding.cpython-314.pyc')
+2026-02-12 01:53:35,313 - INFO - Upload completed successfully: padding.cpython-314.pyc
+2026-02-12 01:53:35,313 - INFO - Starting upload: lib/rich/__pycache__/measure.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/measure.cpython-314.pyc (7415 bytes)
+2026-02-12 01:53:35,315 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/measure.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,316 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/measure.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,316 - DEBUG - Progress: 7415/7415 bytes
+2026-02-12 01:53:35,316 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,318 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/measure.cpython-314.pyc')
+2026-02-12 01:53:35,319 - INFO - Upload completed successfully: measure.cpython-314.pyc
+2026-02-12 01:53:35,319 - INFO - Starting upload: lib/rich/__pycache__/markup.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/markup.cpython-314.pyc (11256 bytes)
+2026-02-12 01:53:35,321 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/markup.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,322 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/markup.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,322 - DEBUG - Progress: 11256/11256 bytes
+2026-02-12 01:53:35,322 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,324 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/markup.cpython-314.pyc')
+2026-02-12 01:53:35,326 - INFO - Upload completed successfully: markup.cpython-314.pyc
+2026-02-12 01:53:35,326 - INFO - Starting upload: lib/rich/__pycache__/markdown.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/markdown.cpython-314.pyc (45855 bytes)
+2026-02-12 01:53:35,327 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/markdown.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,328 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/markdown.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,328 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,334 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/markdown.cpython-314.pyc')
+2026-02-12 01:53:35,336 - INFO - Upload completed successfully: markdown.cpython-314.pyc
+2026-02-12 01:53:35,336 - INFO - Starting upload: lib/rich/__pycache__/logging.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/logging.cpython-314.pyc (15608 bytes)
+2026-02-12 01:53:35,337 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/logging.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,338 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/logging.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,338 - DEBUG - Progress: 15608/15608 bytes
+2026-02-12 01:53:35,338 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,342 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/logging.cpython-314.pyc')
+2026-02-12 01:53:35,344 - INFO - Upload completed successfully: logging.cpython-314.pyc
+2026-02-12 01:53:35,345 - INFO - Starting upload: lib/rich/__pycache__/live_render.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/live_render.cpython-314.pyc (6039 bytes)
+2026-02-12 01:53:35,347 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/live_render.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,347 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/live_render.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,348 - DEBUG - Progress: 6039/6039 bytes
+2026-02-12 01:53:35,348 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,349 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/live_render.cpython-314.pyc')
+2026-02-12 01:53:35,351 - INFO - Upload completed successfully: live_render.cpython-314.pyc
+2026-02-12 01:53:35,351 - INFO - Starting upload: lib/rich/__pycache__/live.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/live.cpython-314.pyc (23704 bytes)
+2026-02-12 01:53:35,353 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/live.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/live.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,354 - DEBUG - Progress: 23704/23704 bytes
+2026-02-12 01:53:35,354 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,357 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/live.cpython-314.pyc')
+2026-02-12 01:53:35,359 - INFO - Upload completed successfully: live.cpython-314.pyc
+2026-02-12 01:53:35,359 - INFO - Starting upload: lib/rich/__pycache__/layout.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/layout.cpython-314.pyc (25210 bytes)
+2026-02-12 01:53:35,362 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/layout.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,362 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/layout.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,362 - DEBUG - Progress: 25210/25210 bytes
+2026-02-12 01:53:35,363 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,366 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/layout.cpython-314.pyc')
+2026-02-12 01:53:35,368 - INFO - Upload completed successfully: layout.cpython-314.pyc
+2026-02-12 01:53:35,368 - INFO - Starting upload: lib/rich/__pycache__/jupyter.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/jupyter.cpython-314.pyc (6589 bytes)
+2026-02-12 01:53:35,370 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/jupyter.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,371 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/jupyter.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,371 - DEBUG - Progress: 6589/6589 bytes
+2026-02-12 01:53:35,371 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,372 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/jupyter.cpython-314.pyc')
+2026-02-12 01:53:35,375 - INFO - Upload completed successfully: jupyter.cpython-314.pyc
+2026-02-12 01:53:35,375 - INFO - Starting upload: lib/rich/__pycache__/json.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/json.cpython-314.pyc (6507 bytes)
+2026-02-12 01:53:35,377 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/json.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,377 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/json.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,378 - DEBUG - Progress: 6507/6507 bytes
+2026-02-12 01:53:35,378 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,379 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/json.cpython-314.pyc')
+2026-02-12 01:53:35,381 - INFO - Upload completed successfully: json.cpython-314.pyc
+2026-02-12 01:53:35,381 - INFO - Starting upload: lib/rich/__pycache__/highlighter.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/highlighter.cpython-314.pyc (12055 bytes)
+2026-02-12 01:53:35,384 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/highlighter.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,384 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/highlighter.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,384 - DEBUG - Progress: 12055/12055 bytes
+2026-02-12 01:53:35,385 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,387 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/highlighter.cpython-314.pyc')
+2026-02-12 01:53:35,388 - INFO - Upload completed successfully: highlighter.cpython-314.pyc
+2026-02-12 01:53:35,389 - INFO - Starting upload: lib/rich/__pycache__/filesize.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/filesize.cpython-314.pyc (3676 bytes)
+2026-02-12 01:53:35,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/filesize.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,391 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/filesize.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,391 - DEBUG - Progress: 3676/3676 bytes
+2026-02-12 01:53:35,391 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,392 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/filesize.cpython-314.pyc')
+2026-02-12 01:53:35,395 - INFO - Upload completed successfully: filesize.cpython-314.pyc
+2026-02-12 01:53:35,395 - INFO - Starting upload: lib/rich/__pycache__/file_proxy.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/file_proxy.cpython-314.pyc (4643 bytes)
+2026-02-12 01:53:35,397 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/file_proxy.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,398 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/file_proxy.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,398 - DEBUG - Progress: 4643/4643 bytes
+2026-02-12 01:53:35,398 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,399 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/file_proxy.cpython-314.pyc')
+2026-02-12 01:53:35,401 - INFO - Upload completed successfully: file_proxy.cpython-314.pyc
+2026-02-12 01:53:35,401 - INFO - Starting upload: lib/rich/__pycache__/errors.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/errors.cpython-314.pyc (1984 bytes)
+2026-02-12 01:53:35,403 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/errors.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,404 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/errors.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,404 - DEBUG - Progress: 1984/1984 bytes
+2026-02-12 01:53:35,404 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,405 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/errors.cpython-314.pyc')
+2026-02-12 01:53:35,407 - INFO - Upload completed successfully: errors.cpython-314.pyc
+2026-02-12 01:53:35,407 - INFO - Starting upload: lib/rich/__pycache__/emoji.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/emoji.cpython-314.pyc (4905 bytes)
+2026-02-12 01:53:35,409 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/emoji.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,409 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/emoji.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,410 - DEBUG - Progress: 4905/4905 bytes
+2026-02-12 01:53:35,410 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,411 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/emoji.cpython-314.pyc')
+2026-02-12 01:53:35,413 - INFO - Upload completed successfully: emoji.cpython-314.pyc
+2026-02-12 01:53:35,413 - INFO - Starting upload: lib/rich/__pycache__/diagnose.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/diagnose.cpython-314.pyc (1592 bytes)
+2026-02-12 01:53:35,414 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/diagnose.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,415 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/diagnose.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,415 - DEBUG - Progress: 1592/1592 bytes
+2026-02-12 01:53:35,415 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,416 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/diagnose.cpython-314.pyc')
+2026-02-12 01:53:35,417 - INFO - Upload completed successfully: diagnose.cpython-314.pyc
+2026-02-12 01:53:35,418 - INFO - Starting upload: lib/rich/__pycache__/default_styles.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/default_styles.cpython-314.pyc (11114 bytes)
+2026-02-12 01:53:35,419 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/default_styles.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,419 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/default_styles.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,420 - DEBUG - Progress: 11114/11114 bytes
+2026-02-12 01:53:35,420 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,421 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/default_styles.cpython-314.pyc')
+2026-02-12 01:53:35,423 - INFO - Upload completed successfully: default_styles.cpython-314.pyc
+2026-02-12 01:53:35,423 - INFO - Starting upload: lib/rich/__pycache__/control.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/control.cpython-314.pyc (13507 bytes)
+2026-02-12 01:53:35,424 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/control.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,425 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/control.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,425 - DEBUG - Progress: 13507/13507 bytes
+2026-02-12 01:53:35,425 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,429 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/control.cpython-314.pyc')
+2026-02-12 01:53:35,431 - INFO - Upload completed successfully: control.cpython-314.pyc
+2026-02-12 01:53:35,431 - INFO - Starting upload: lib/rich/__pycache__/containers.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/containers.cpython-314.pyc (12145 bytes)
+2026-02-12 01:53:35,432 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/containers.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,433 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/containers.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,433 - DEBUG - Progress: 12145/12145 bytes
+2026-02-12 01:53:35,433 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,435 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/containers.cpython-314.pyc')
+2026-02-12 01:53:35,436 - INFO - Upload completed successfully: containers.cpython-314.pyc
+2026-02-12 01:53:35,437 - INFO - Starting upload: lib/rich/__pycache__/constrain.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/constrain.cpython-314.pyc (2778 bytes)
+2026-02-12 01:53:35,438 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/constrain.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,439 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/constrain.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,439 - DEBUG - Progress: 2778/2778 bytes
+2026-02-12 01:53:35,439 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,440 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/constrain.cpython-314.pyc')
+2026-02-12 01:53:35,441 - INFO - Upload completed successfully: constrain.cpython-314.pyc
+2026-02-12 01:53:35,441 - INFO - Starting upload: lib/rich/__pycache__/console.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/console.cpython-314.pyc (138154 bytes)
+2026-02-12 01:53:35,443 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/console.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,443 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/console.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,444 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,461 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/console.cpython-314.pyc')
+2026-02-12 01:53:35,464 - INFO - Upload completed successfully: console.cpython-314.pyc
+2026-02-12 01:53:35,464 - INFO - Starting upload: lib/rich/__pycache__/columns.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/columns.cpython-314.pyc (9781 bytes)
+2026-02-12 01:53:35,466 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/columns.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,467 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/columns.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,467 - DEBUG - Progress: 9781/9781 bytes
+2026-02-12 01:53:35,467 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,469 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/columns.cpython-314.pyc')
+2026-02-12 01:53:35,471 - INFO - Upload completed successfully: columns.cpython-314.pyc
+2026-02-12 01:53:35,471 - INFO - Starting upload: lib/rich/__pycache__/color_triplet.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/color_triplet.cpython-314.pyc (2340 bytes)
+2026-02-12 01:53:35,474 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/color_triplet.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,475 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/color_triplet.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,476 - DEBUG - Progress: 2340/2340 bytes
+2026-02-12 01:53:35,476 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,477 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/color_triplet.cpython-314.pyc')
+2026-02-12 01:53:35,479 - INFO - Upload completed successfully: color_triplet.cpython-314.pyc
+2026-02-12 01:53:35,479 - INFO - Starting upload: lib/rich/__pycache__/color.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/color.cpython-314.pyc (28640 bytes)
+2026-02-12 01:53:35,481 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/color.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,482 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/color.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,482 - DEBUG - Progress: 28640/28640 bytes
+2026-02-12 01:53:35,482 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,485 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/color.cpython-314.pyc')
+2026-02-12 01:53:35,487 - INFO - Upload completed successfully: color.cpython-314.pyc
+2026-02-12 01:53:35,487 - INFO - Starting upload: lib/rich/__pycache__/cells.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/cells.cpython-314.pyc (11589 bytes)
+2026-02-12 01:53:35,489 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/cells.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,490 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/cells.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,490 - DEBUG - Progress: 11589/11589 bytes
+2026-02-12 01:53:35,490 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,492 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/cells.cpython-314.pyc')
+2026-02-12 01:53:35,494 - INFO - Upload completed successfully: cells.cpython-314.pyc
+2026-02-12 01:53:35,494 - INFO - Starting upload: lib/rich/__pycache__/box.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/box.cpython-314.pyc (14294 bytes)
+2026-02-12 01:53:35,496 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/box.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,497 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/box.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,497 - DEBUG - Progress: 14294/14294 bytes
+2026-02-12 01:53:35,497 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,499 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/box.cpython-314.pyc')
+2026-02-12 01:53:35,502 - INFO - Upload completed successfully: box.cpython-314.pyc
+2026-02-12 01:53:35,502 - INFO - Starting upload: lib/rich/__pycache__/bar.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/bar.cpython-314.pyc (5172 bytes)
+2026-02-12 01:53:35,504 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/bar.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,505 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/bar.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,505 - DEBUG - Progress: 5172/5172 bytes
+2026-02-12 01:53:35,505 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,506 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/bar.cpython-314.pyc')
+2026-02-12 01:53:35,508 - INFO - Upload completed successfully: bar.cpython-314.pyc
+2026-02-12 01:53:35,508 - INFO - Starting upload: lib/rich/__pycache__/ansi.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/ansi.cpython-314.pyc (10187 bytes)
+2026-02-12 01:53:35,510 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/ansi.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,511 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/ansi.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,511 - DEBUG - Progress: 10187/10187 bytes
+2026-02-12 01:53:35,511 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,513 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/ansi.cpython-314.pyc')
+2026-02-12 01:53:35,515 - INFO - Upload completed successfully: ansi.cpython-314.pyc
+2026-02-12 01:53:35,516 - INFO - Starting upload: lib/rich/__pycache__/align.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/align.cpython-314.pyc (15265 bytes)
+2026-02-12 01:53:35,518 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/align.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,518 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/align.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,518 - DEBUG - Progress: 15265/15265 bytes
+2026-02-12 01:53:35,519 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,521 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/align.cpython-314.pyc')
+2026-02-12 01:53:35,522 - INFO - Upload completed successfully: align.cpython-314.pyc
+2026-02-12 01:53:35,523 - INFO - Starting upload: lib/rich/__pycache__/abc.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/abc.cpython-314.pyc (1860 bytes)
+2026-02-12 01:53:35,525 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/abc.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,525 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/abc.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,526 - DEBUG - Progress: 1860/1860 bytes
+2026-02-12 01:53:35,526 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,527 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/abc.cpython-314.pyc')
+2026-02-12 01:53:35,529 - INFO - Upload completed successfully: abc.cpython-314.pyc
+2026-02-12 01:53:35,529 - INFO - Starting upload: lib/rich/__pycache__/_wrap.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_wrap.cpython-314.pyc (3624 bytes)
+2026-02-12 01:53:35,531 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_wrap.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,532 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_wrap.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,532 - DEBUG - Progress: 3624/3624 bytes
+2026-02-12 01:53:35,532 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,533 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_wrap.cpython-314.pyc')
+2026-02-12 01:53:35,536 - INFO - Upload completed successfully: _wrap.cpython-314.pyc
+2026-02-12 01:53:35,536 - INFO - Starting upload: lib/rich/__pycache__/_windows_renderer.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_windows_renderer.cpython-314.pyc (3819 bytes)
+2026-02-12 01:53:35,538 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_windows_renderer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,538 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_windows_renderer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,539 - DEBUG - Progress: 3819/3819 bytes
+2026-02-12 01:53:35,539 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,540 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_windows_renderer.cpython-314.pyc')
+2026-02-12 01:53:35,542 - INFO - Upload completed successfully: _windows_renderer.cpython-314.pyc
+2026-02-12 01:53:35,542 - INFO - Starting upload: lib/rich/__pycache__/_windows.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_windows.cpython-314.pyc (3030 bytes)
+2026-02-12 01:53:35,545 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_windows.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,545 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_windows.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,545 - DEBUG - Progress: 3030/3030 bytes
+2026-02-12 01:53:35,545 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,547 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_windows.cpython-314.pyc')
+2026-02-12 01:53:35,549 - INFO - Upload completed successfully: _windows.cpython-314.pyc
+2026-02-12 01:53:35,549 - INFO - Starting upload: lib/rich/__pycache__/_win32_console.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_win32_console.cpython-314.pyc (33456 bytes)
+2026-02-12 01:53:35,551 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_win32_console.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,552 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_win32_console.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,552 - DEBUG - Progress: 32768/33456 bytes
+2026-02-12 01:53:35,552 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,556 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_win32_console.cpython-314.pyc')
+2026-02-12 01:53:35,558 - INFO - Upload completed successfully: _win32_console.cpython-314.pyc
+2026-02-12 01:53:35,558 - INFO - Starting upload: lib/rich/__pycache__/_timer.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_timer.cpython-314.pyc (1001 bytes)
+2026-02-12 01:53:35,560 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_timer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,560 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_timer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,560 - DEBUG - Progress: 1001/1001 bytes
+2026-02-12 01:53:35,560 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,561 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_timer.cpython-314.pyc')
+2026-02-12 01:53:35,563 - INFO - Upload completed successfully: _timer.cpython-314.pyc
+2026-02-12 01:53:35,563 - INFO - Starting upload: lib/rich/__pycache__/_stack.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_stack.cpython-314.pyc (1356 bytes)
+2026-02-12 01:53:35,564 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_stack.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,565 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_stack.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,565 - DEBUG - Progress: 1356/1356 bytes
+2026-02-12 01:53:35,565 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,566 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_stack.cpython-314.pyc')
+2026-02-12 01:53:35,567 - INFO - Upload completed successfully: _stack.cpython-314.pyc
+2026-02-12 01:53:35,567 - INFO - Starting upload: lib/rich/__pycache__/_spinners.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_spinners.cpython-314.pyc (15679 bytes)
+2026-02-12 01:53:35,569 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_spinners.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,570 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_spinners.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,570 - DEBUG - Progress: 15679/15679 bytes
+2026-02-12 01:53:35,570 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,574 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_spinners.cpython-314.pyc')
+2026-02-12 01:53:35,576 - INFO - Upload completed successfully: _spinners.cpython-314.pyc
+2026-02-12 01:53:35,576 - INFO - Starting upload: lib/rich/__pycache__/_ratio.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_ratio.cpython-314.pyc (7606 bytes)
+2026-02-12 01:53:35,577 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_ratio.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,578 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_ratio.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,578 - DEBUG - Progress: 7606/7606 bytes
+2026-02-12 01:53:35,578 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,580 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_ratio.cpython-314.pyc')
+2026-02-12 01:53:35,582 - INFO - Upload completed successfully: _ratio.cpython-314.pyc
+2026-02-12 01:53:35,582 - INFO - Starting upload: lib/rich/__pycache__/_pick.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_pick.cpython-314.pyc (851 bytes)
+2026-02-12 01:53:35,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_pick.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,585 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_pick.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,585 - DEBUG - Progress: 851/851 bytes
+2026-02-12 01:53:35,585 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,586 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_pick.cpython-314.pyc')
+2026-02-12 01:53:35,588 - INFO - Upload completed successfully: _pick.cpython-314.pyc
+2026-02-12 01:53:35,588 - INFO - Starting upload: lib/rich/__pycache__/_palettes.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_palettes.cpython-314.pyc (9356 bytes)
+2026-02-12 01:53:35,590 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_palettes.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,591 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_palettes.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,591 - DEBUG - Progress: 9356/9356 bytes
+2026-02-12 01:53:35,591 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,593 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_palettes.cpython-314.pyc')
+2026-02-12 01:53:35,594 - INFO - Upload completed successfully: _palettes.cpython-314.pyc
+2026-02-12 01:53:35,594 - INFO - Starting upload: lib/rich/__pycache__/_null_file.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_null_file.cpython-314.pyc (6582 bytes)
+2026-02-12 01:53:35,597 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_null_file.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,598 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_null_file.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,598 - DEBUG - Progress: 6582/6582 bytes
+2026-02-12 01:53:35,598 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,600 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_null_file.cpython-314.pyc')
+2026-02-12 01:53:35,601 - INFO - Upload completed successfully: _null_file.cpython-314.pyc
+2026-02-12 01:53:35,601 - INFO - Starting upload: lib/rich/__pycache__/_loop.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_loop.cpython-314.pyc (2487 bytes)
+2026-02-12 01:53:35,603 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_loop.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,603 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_loop.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,603 - DEBUG - Progress: 2487/2487 bytes
+2026-02-12 01:53:35,603 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,605 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_loop.cpython-314.pyc')
+2026-02-12 01:53:35,606 - INFO - Upload completed successfully: _loop.cpython-314.pyc
+2026-02-12 01:53:35,606 - INFO - Starting upload: lib/rich/__pycache__/_log_render.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_log_render.cpython-314.pyc (4837 bytes)
+2026-02-12 01:53:35,608 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_log_render.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,608 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_log_render.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,608 - DEBUG - Progress: 4837/4837 bytes
+2026-02-12 01:53:35,608 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,610 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_log_render.cpython-314.pyc')
+2026-02-12 01:53:35,611 - INFO - Upload completed successfully: _log_render.cpython-314.pyc
+2026-02-12 01:53:35,611 - INFO - Starting upload: lib/rich/__pycache__/_inspect.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_inspect.cpython-314.pyc (14841 bytes)
+2026-02-12 01:53:35,613 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_inspect.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,613 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_inspect.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,613 - DEBUG - Progress: 14841/14841 bytes
+2026-02-12 01:53:35,614 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,617 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_inspect.cpython-314.pyc')
+2026-02-12 01:53:35,619 - INFO - Upload completed successfully: _inspect.cpython-314.pyc
+2026-02-12 01:53:35,619 - INFO - Starting upload: lib/rich/__pycache__/_fileno.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_fileno.cpython-314.pyc (963 bytes)
+2026-02-12 01:53:35,621 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_fileno.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,622 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_fileno.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,622 - DEBUG - Progress: 963/963 bytes
+2026-02-12 01:53:35,622 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,623 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_fileno.cpython-314.pyc')
+2026-02-12 01:53:35,624 - INFO - Upload completed successfully: _fileno.cpython-314.pyc
+2026-02-12 01:53:35,624 - INFO - Starting upload: lib/rich/__pycache__/_extension.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_extension.cpython-314.pyc (620 bytes)
+2026-02-12 01:53:35,626 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_extension.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,626 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_extension.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,627 - DEBUG - Progress: 620/620 bytes
+2026-02-12 01:53:35,627 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,627 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_extension.cpython-314.pyc')
+2026-02-12 01:53:35,629 - INFO - Upload completed successfully: _extension.cpython-314.pyc
+2026-02-12 01:53:35,629 - INFO - Starting upload: lib/rich/__pycache__/_export_format.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_export_format.cpython-314.pyc (2313 bytes)
+2026-02-12 01:53:35,631 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_export_format.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,631 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_export_format.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,631 - DEBUG - Progress: 2313/2313 bytes
+2026-02-12 01:53:35,631 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,633 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_export_format.cpython-314.pyc')
+2026-02-12 01:53:35,634 - INFO - Upload completed successfully: _export_format.cpython-314.pyc
+2026-02-12 01:53:35,634 - INFO - Starting upload: lib/rich/__pycache__/_emoji_replace.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_emoji_replace.cpython-314.pyc (2094 bytes)
+2026-02-12 01:53:35,636 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_emoji_replace.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,636 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_emoji_replace.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,636 - DEBUG - Progress: 2094/2094 bytes
+2026-02-12 01:53:35,636 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,638 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_emoji_replace.cpython-314.pyc')
+2026-02-12 01:53:35,639 - INFO - Upload completed successfully: _emoji_replace.cpython-314.pyc
+2026-02-12 01:53:35,639 - INFO - Starting upload: lib/rich/__pycache__/_emoji_codes.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/_emoji_codes.cpython-314.pyc (205958 bytes)
+2026-02-12 01:53:35,641 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_emoji_codes.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,641 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/_emoji_codes.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,653 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,662 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/_emoji_codes.cpython-314.pyc')
+2026-02-12 01:53:35,665 - INFO - Upload completed successfully: _emoji_codes.cpython-314.pyc
+2026-02-12 01:53:35,665 - INFO - Starting upload: lib/rich/__pycache__/__main__.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/__main__.cpython-314.pyc (10141 bytes)
+2026-02-12 01:53:35,667 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/__main__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,667 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/__main__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,668 - DEBUG - Progress: 10141/10141 bytes
+2026-02-12 01:53:35,668 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,669 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/__main__.cpython-314.pyc')
+2026-02-12 01:53:35,672 - INFO - Upload completed successfully: __main__.cpython-314.pyc
+2026-02-12 01:53:35,672 - INFO - Starting upload: lib/rich/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/rich/__pycache__/__init__.cpython-314.pyc (8397 bytes)
+2026-02-12 01:53:35,674 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,675 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,675 - DEBUG - Progress: 8397/8397 bytes
+2026-02-12 01:53:35,675 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,677 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:35,679 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:35,679 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/rich/_unicode_data', 511)
+2026-02-12 01:53:35,680 - INFO - Created remote folder: /home/kevin/test/lib/rich/_unicode_data
+2026-02-12 01:53:35,680 - INFO - Starting upload: lib/rich/_unicode_data/unicode9-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode9-0-0.py (14148 bytes)
+2026-02-12 01:53:35,682 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode9-0-0.py', 'wb')
+2026-02-12 01:53:35,683 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode9-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,683 - DEBUG - Progress: 14148/14148 bytes
+2026-02-12 01:53:35,683 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,685 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode9-0-0.py')
+2026-02-12 01:53:35,687 - INFO - Upload completed successfully: unicode9-0-0.py
+2026-02-12 01:53:35,687 - INFO - Starting upload: lib/rich/_unicode_data/unicode8-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode8-0-0.py (11864 bytes)
+2026-02-12 01:53:35,689 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode8-0-0.py', 'wb')
+2026-02-12 01:53:35,690 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode8-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,690 - DEBUG - Progress: 11864/11864 bytes
+2026-02-12 01:53:35,690 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,693 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode8-0-0.py')
+2026-02-12 01:53:35,695 - INFO - Upload completed successfully: unicode8-0-0.py
+2026-02-12 01:53:35,695 - INFO - Starting upload: lib/rich/_unicode_data/unicode7-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode7-0-0.py (11630 bytes)
+2026-02-12 01:53:35,697 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode7-0-0.py', 'wb')
+2026-02-12 01:53:35,697 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode7-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,698 - DEBUG - Progress: 11630/11630 bytes
+2026-02-12 01:53:35,698 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,700 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode7-0-0.py')
+2026-02-12 01:53:35,702 - INFO - Upload completed successfully: unicode7-0-0.py
+2026-02-12 01:53:35,702 - INFO - Starting upload: lib/rich/_unicode_data/unicode6-3-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode6-3-0.py (10924 bytes)
+2026-02-12 01:53:35,704 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-3-0.py', 'wb')
+2026-02-12 01:53:35,705 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-3-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,705 - DEBUG - Progress: 10924/10924 bytes
+2026-02-12 01:53:35,705 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,707 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-3-0.py')
+2026-02-12 01:53:35,709 - INFO - Upload completed successfully: unicode6-3-0.py
+2026-02-12 01:53:35,710 - INFO - Starting upload: lib/rich/_unicode_data/unicode6-2-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode6-2-0.py (10899 bytes)
+2026-02-12 01:53:35,712 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-2-0.py', 'wb')
+2026-02-12 01:53:35,712 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-2-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,712 - DEBUG - Progress: 10899/10899 bytes
+2026-02-12 01:53:35,712 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,714 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-2-0.py')
+2026-02-12 01:53:35,716 - INFO - Upload completed successfully: unicode6-2-0.py
+2026-02-12 01:53:35,717 - INFO - Starting upload: lib/rich/_unicode_data/unicode6-1-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode6-1-0.py (10899 bytes)
+2026-02-12 01:53:35,719 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-1-0.py', 'wb')
+2026-02-12 01:53:35,719 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-1-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,719 - DEBUG - Progress: 10899/10899 bytes
+2026-02-12 01:53:35,719 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,721 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-1-0.py')
+2026-02-12 01:53:35,723 - INFO - Upload completed successfully: unicode6-1-0.py
+2026-02-12 01:53:35,723 - INFO - Starting upload: lib/rich/_unicode_data/unicode6-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode6-0-0.py (10604 bytes)
+2026-02-12 01:53:35,725 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-0-0.py', 'wb')
+2026-02-12 01:53:35,726 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,726 - DEBUG - Progress: 10604/10604 bytes
+2026-02-12 01:53:35,726 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,728 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode6-0-0.py')
+2026-02-12 01:53:35,730 - INFO - Upload completed successfully: unicode6-0-0.py
+2026-02-12 01:53:35,731 - INFO - Starting upload: lib/rich/_unicode_data/unicode5-2-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode5-2-0.py (10390 bytes)
+2026-02-12 01:53:35,733 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-2-0.py', 'wb')
+2026-02-12 01:53:35,733 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-2-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,733 - DEBUG - Progress: 10390/10390 bytes
+2026-02-12 01:53:35,733 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,735 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-2-0.py')
+2026-02-12 01:53:35,737 - INFO - Upload completed successfully: unicode5-2-0.py
+2026-02-12 01:53:35,737 - INFO - Starting upload: lib/rich/_unicode_data/unicode5-1-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode5-1-0.py (9650 bytes)
+2026-02-12 01:53:35,739 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-1-0.py', 'wb')
+2026-02-12 01:53:35,740 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-1-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,740 - DEBUG - Progress: 9650/9650 bytes
+2026-02-12 01:53:35,741 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,742 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-1-0.py')
+2026-02-12 01:53:35,745 - INFO - Upload completed successfully: unicode5-1-0.py
+2026-02-12 01:53:35,745 - INFO - Starting upload: lib/rich/_unicode_data/unicode5-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode5-0-0.py (9613 bytes)
+2026-02-12 01:53:35,747 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-0-0.py', 'wb')
+2026-02-12 01:53:35,748 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,748 - DEBUG - Progress: 9613/9613 bytes
+2026-02-12 01:53:35,748 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,749 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode5-0-0.py')
+2026-02-12 01:53:35,751 - INFO - Upload completed successfully: unicode5-0-0.py
+2026-02-12 01:53:35,751 - INFO - Starting upload: lib/rich/_unicode_data/unicode4-1-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode4-1-0.py (9488 bytes)
+2026-02-12 01:53:35,753 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode4-1-0.py', 'wb')
+2026-02-12 01:53:35,753 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode4-1-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,754 - DEBUG - Progress: 9488/9488 bytes
+2026-02-12 01:53:35,754 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,755 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode4-1-0.py')
+2026-02-12 01:53:35,759 - INFO - Upload completed successfully: unicode4-1-0.py
+2026-02-12 01:53:35,759 - INFO - Starting upload: lib/rich/_unicode_data/unicode17-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode17-0-0.py (16704 bytes)
+2026-02-12 01:53:35,763 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode17-0-0.py', 'wb')
+2026-02-12 01:53:35,764 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode17-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,764 - DEBUG - Progress: 16704/16704 bytes
+2026-02-12 01:53:35,764 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,767 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode17-0-0.py')
+2026-02-12 01:53:35,769 - INFO - Upload completed successfully: unicode17-0-0.py
+2026-02-12 01:53:35,769 - INFO - Starting upload: lib/rich/_unicode_data/unicode16-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode16-0-0.py (16480 bytes)
+2026-02-12 01:53:35,771 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode16-0-0.py', 'wb')
+2026-02-12 01:53:35,771 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode16-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,772 - DEBUG - Progress: 16480/16480 bytes
+2026-02-12 01:53:35,772 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,774 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode16-0-0.py')
+2026-02-12 01:53:35,776 - INFO - Upload completed successfully: unicode16-0-0.py
+2026-02-12 01:53:35,776 - INFO - Starting upload: lib/rich/_unicode_data/unicode15-1-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode15-1-0.py (16129 bytes)
+2026-02-12 01:53:35,777 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode15-1-0.py', 'wb')
+2026-02-12 01:53:35,778 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode15-1-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,778 - DEBUG - Progress: 16129/16129 bytes
+2026-02-12 01:53:35,778 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,782 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode15-1-0.py')
+2026-02-12 01:53:35,784 - INFO - Upload completed successfully: unicode15-1-0.py
+2026-02-12 01:53:35,784 - INFO - Starting upload: lib/rich/_unicode_data/unicode15-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode15-0-0.py (16156 bytes)
+2026-02-12 01:53:35,786 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode15-0-0.py', 'wb')
+2026-02-12 01:53:35,786 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode15-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,787 - DEBUG - Progress: 16156/16156 bytes
+2026-02-12 01:53:35,787 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,791 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode15-0-0.py')
+2026-02-12 01:53:35,792 - INFO - Upload completed successfully: unicode15-0-0.py
+2026-02-12 01:53:35,792 - INFO - Starting upload: lib/rich/_unicode_data/unicode14-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode14-0-0.py (15884 bytes)
+2026-02-12 01:53:35,794 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode14-0-0.py', 'wb')
+2026-02-12 01:53:35,795 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode14-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,795 - DEBUG - Progress: 15884/15884 bytes
+2026-02-12 01:53:35,795 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,799 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode14-0-0.py')
+2026-02-12 01:53:35,801 - INFO - Upload completed successfully: unicode14-0-0.py
+2026-02-12 01:53:35,801 - INFO - Starting upload: lib/rich/_unicode_data/unicode13-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode13-0-0.py (15519 bytes)
+2026-02-12 01:53:35,802 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode13-0-0.py', 'wb')
+2026-02-12 01:53:35,803 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode13-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,803 - DEBUG - Progress: 15519/15519 bytes
+2026-02-12 01:53:35,803 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,805 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode13-0-0.py')
+2026-02-12 01:53:35,807 - INFO - Upload completed successfully: unicode13-0-0.py
+2026-02-12 01:53:35,807 - INFO - Starting upload: lib/rich/_unicode_data/unicode12-1-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode12-1-0.py (15189 bytes)
+2026-02-12 01:53:35,809 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode12-1-0.py', 'wb')
+2026-02-12 01:53:35,810 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode12-1-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,810 - DEBUG - Progress: 15189/15189 bytes
+2026-02-12 01:53:35,810 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,812 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode12-1-0.py')
+2026-02-12 01:53:35,814 - INFO - Upload completed successfully: unicode12-1-0.py
+2026-02-12 01:53:35,814 - INFO - Starting upload: lib/rich/_unicode_data/unicode12-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode12-0-0.py (15216 bytes)
+2026-02-12 01:53:35,816 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode12-0-0.py', 'wb')
+2026-02-12 01:53:35,817 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode12-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,817 - DEBUG - Progress: 15216/15216 bytes
+2026-02-12 01:53:35,817 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,819 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode12-0-0.py')
+2026-02-12 01:53:35,821 - INFO - Upload completed successfully: unicode12-0-0.py
+2026-02-12 01:53:35,821 - INFO - Starting upload: lib/rich/_unicode_data/unicode11-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode11-0-0.py (14874 bytes)
+2026-02-12 01:53:35,823 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode11-0-0.py', 'wb')
+2026-02-12 01:53:35,824 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode11-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,824 - DEBUG - Progress: 14874/14874 bytes
+2026-02-12 01:53:35,824 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,826 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode11-0-0.py')
+2026-02-12 01:53:35,828 - INFO - Upload completed successfully: unicode11-0-0.py
+2026-02-12 01:53:35,828 - INFO - Starting upload: lib/rich/_unicode_data/unicode10-0-0.py -> /home/kevin/test/lib/rich/_unicode_data/unicode10-0-0.py (14496 bytes)
+2026-02-12 01:53:35,830 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode10-0-0.py', 'wb')
+2026-02-12 01:53:35,831 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/unicode10-0-0.py', 'wb') -> 00000000
+2026-02-12 01:53:35,831 - DEBUG - Progress: 14496/14496 bytes
+2026-02-12 01:53:35,831 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,835 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/unicode10-0-0.py')
+2026-02-12 01:53:35,837 - INFO - Upload completed successfully: unicode10-0-0.py
+2026-02-12 01:53:35,837 - INFO - Starting upload: lib/rich/_unicode_data/_versions.py -> /home/kevin/test/lib/rich/_unicode_data/_versions.py (298 bytes)
+2026-02-12 01:53:35,839 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/_versions.py', 'wb')
+2026-02-12 01:53:35,840 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/_versions.py', 'wb') -> 00000000
+2026-02-12 01:53:35,840 - DEBUG - Progress: 298/298 bytes
+2026-02-12 01:53:35,840 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,841 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/_versions.py')
+2026-02-12 01:53:35,843 - INFO - Upload completed successfully: _versions.py
+2026-02-12 01:53:35,843 - INFO - Starting upload: lib/rich/_unicode_data/__init__.py -> /home/kevin/test/lib/rich/_unicode_data/__init__.py (2631 bytes)
+2026-02-12 01:53:35,845 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__init__.py', 'wb')
+2026-02-12 01:53:35,845 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:35,846 - DEBUG - Progress: 2631/2631 bytes
+2026-02-12 01:53:35,846 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,847 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__init__.py')
+2026-02-12 01:53:35,849 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:35,849 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__', 511)
+2026-02-12 01:53:35,850 - INFO - Created remote folder: /home/kevin/test/lib/rich/_unicode_data/__pycache__
+2026-02-12 01:53:35,850 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode9-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode9-0-0.cpython-314.pyc (18826 bytes)
+2026-02-12 01:53:35,851 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode9-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,852 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode9-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,852 - DEBUG - Progress: 18826/18826 bytes
+2026-02-12 01:53:35,852 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,855 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode9-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,856 - INFO - Upload completed successfully: unicode9-0-0.cpython-314.pyc
+2026-02-12 01:53:35,856 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode8-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode8-0-0.cpython-314.pyc (15838 bytes)
+2026-02-12 01:53:35,858 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode8-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,858 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode8-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,858 - DEBUG - Progress: 15838/15838 bytes
+2026-02-12 01:53:35,858 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,860 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode8-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,864 - INFO - Upload completed successfully: unicode8-0-0.cpython-314.pyc
+2026-02-12 01:53:35,864 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode7-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode7-0-0.cpython-314.pyc (15550 bytes)
+2026-02-12 01:53:35,867 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode7-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,868 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode7-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,869 - DEBUG - Progress: 15550/15550 bytes
+2026-02-12 01:53:35,869 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,872 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode7-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,875 - INFO - Upload completed successfully: unicode7-0-0.cpython-314.pyc
+2026-02-12 01:53:35,875 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode6-3-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-3-0.cpython-314.pyc (14614 bytes)
+2026-02-12 01:53:35,878 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-3-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,879 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-3-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,880 - DEBUG - Progress: 14614/14614 bytes
+2026-02-12 01:53:35,880 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,882 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-3-0.cpython-314.pyc')
+2026-02-12 01:53:35,885 - INFO - Upload completed successfully: unicode6-3-0.cpython-314.pyc
+2026-02-12 01:53:35,885 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode6-2-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-2-0.cpython-314.pyc (14578 bytes)
+2026-02-12 01:53:35,889 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-2-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,890 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-2-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,890 - DEBUG - Progress: 14578/14578 bytes
+2026-02-12 01:53:35,890 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,894 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-2-0.cpython-314.pyc')
+2026-02-12 01:53:35,896 - INFO - Upload completed successfully: unicode6-2-0.cpython-314.pyc
+2026-02-12 01:53:35,896 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode6-1-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-1-0.cpython-314.pyc (14578 bytes)
+2026-02-12 01:53:35,897 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-1-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,898 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-1-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,898 - DEBUG - Progress: 14578/14578 bytes
+2026-02-12 01:53:35,898 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,902 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-1-0.cpython-314.pyc')
+2026-02-12 01:53:35,904 - INFO - Upload completed successfully: unicode6-1-0.cpython-314.pyc
+2026-02-12 01:53:35,904 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode6-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-0-0.cpython-314.pyc (14182 bytes)
+2026-02-12 01:53:35,906 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,906 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,906 - DEBUG - Progress: 14182/14182 bytes
+2026-02-12 01:53:35,907 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,909 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode6-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,910 - INFO - Upload completed successfully: unicode6-0-0.cpython-314.pyc
+2026-02-12 01:53:35,910 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode5-2-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-2-0.cpython-314.pyc (13894 bytes)
+2026-02-12 01:53:35,912 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-2-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,912 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-2-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,912 - DEBUG - Progress: 13894/13894 bytes
+2026-02-12 01:53:35,912 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,915 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-2-0.cpython-314.pyc')
+2026-02-12 01:53:35,916 - INFO - Upload completed successfully: unicode5-2-0.cpython-314.pyc
+2026-02-12 01:53:35,916 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode5-1-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-1-0.cpython-314.pyc (12886 bytes)
+2026-02-12 01:53:35,918 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-1-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,918 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-1-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,918 - DEBUG - Progress: 12886/12886 bytes
+2026-02-12 01:53:35,918 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,922 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-1-0.cpython-314.pyc')
+2026-02-12 01:53:35,924 - INFO - Upload completed successfully: unicode5-1-0.cpython-314.pyc
+2026-02-12 01:53:35,925 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode5-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-0-0.cpython-314.pyc (12778 bytes)
+2026-02-12 01:53:35,927 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,927 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,928 - DEBUG - Progress: 12778/12778 bytes
+2026-02-12 01:53:35,928 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,930 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode5-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,932 - INFO - Upload completed successfully: unicode5-0-0.cpython-314.pyc
+2026-02-12 01:53:35,932 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode4-1-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode4-1-0.cpython-314.pyc (12598 bytes)
+2026-02-12 01:53:35,934 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode4-1-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,935 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode4-1-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,935 - DEBUG - Progress: 12598/12598 bytes
+2026-02-12 01:53:35,935 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,939 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode4-1-0.cpython-314.pyc')
+2026-02-12 01:53:35,942 - INFO - Upload completed successfully: unicode4-1-0.cpython-314.pyc
+2026-02-12 01:53:35,942 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode17-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode17-0-0.cpython-314.pyc (22176 bytes)
+2026-02-12 01:53:35,944 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode17-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,944 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode17-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,945 - DEBUG - Progress: 22176/22176 bytes
+2026-02-12 01:53:35,945 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,948 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode17-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,950 - INFO - Upload completed successfully: unicode17-0-0.cpython-314.pyc
+2026-02-12 01:53:35,950 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode16-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode16-0-0.cpython-314.pyc (21888 bytes)
+2026-02-12 01:53:35,952 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode16-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,953 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode16-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,953 - DEBUG - Progress: 21888/21888 bytes
+2026-02-12 01:53:35,953 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,956 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode16-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,958 - INFO - Upload completed successfully: unicode16-0-0.cpython-314.pyc
+2026-02-12 01:53:35,958 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode15-1-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-1-0.cpython-314.pyc (21420 bytes)
+2026-02-12 01:53:35,960 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-1-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,961 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-1-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,961 - DEBUG - Progress: 21420/21420 bytes
+2026-02-12 01:53:35,961 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,964 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-1-0.cpython-314.pyc')
+2026-02-12 01:53:35,966 - INFO - Upload completed successfully: unicode15-1-0.cpython-314.pyc
+2026-02-12 01:53:35,966 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode15-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-0-0.cpython-314.pyc (21456 bytes)
+2026-02-12 01:53:35,968 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,969 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,969 - DEBUG - Progress: 21456/21456 bytes
+2026-02-12 01:53:35,969 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,973 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode15-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,975 - INFO - Upload completed successfully: unicode15-0-0.cpython-314.pyc
+2026-02-12 01:53:35,975 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode14-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode14-0-0.cpython-314.pyc (21096 bytes)
+2026-02-12 01:53:35,977 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode14-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,977 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode14-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,978 - DEBUG - Progress: 21096/21096 bytes
+2026-02-12 01:53:35,978 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,981 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode14-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,983 - INFO - Upload completed successfully: unicode14-0-0.cpython-314.pyc
+2026-02-12 01:53:35,983 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode13-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode13-0-0.cpython-314.pyc (20628 bytes)
+2026-02-12 01:53:35,985 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode13-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,985 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode13-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,986 - DEBUG - Progress: 20628/20628 bytes
+2026-02-12 01:53:35,986 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,989 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode13-0-0.cpython-314.pyc')
+2026-02-12 01:53:35,991 - INFO - Upload completed successfully: unicode13-0-0.cpython-314.pyc
+2026-02-12 01:53:35,991 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode12-1-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-1-0.cpython-314.pyc (20196 bytes)
+2026-02-12 01:53:35,993 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-1-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:35,994 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-1-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:35,994 - DEBUG - Progress: 20196/20196 bytes
+2026-02-12 01:53:35,994 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:35,997 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-1-0.cpython-314.pyc')
+2026-02-12 01:53:35,999 - INFO - Upload completed successfully: unicode12-1-0.cpython-314.pyc
+2026-02-12 01:53:35,999 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode12-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-0-0.cpython-314.pyc (20232 bytes)
+2026-02-12 01:53:36,001 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,002 - DEBUG - Progress: 20232/20232 bytes
+2026-02-12 01:53:36,002 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,005 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode12-0-0.cpython-314.pyc')
+2026-02-12 01:53:36,007 - INFO - Upload completed successfully: unicode12-0-0.cpython-314.pyc
+2026-02-12 01:53:36,007 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode11-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode11-0-0.cpython-314.pyc (19800 bytes)
+2026-02-12 01:53:36,009 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode11-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,010 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode11-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,010 - DEBUG - Progress: 19800/19800 bytes
+2026-02-12 01:53:36,010 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,013 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode11-0-0.cpython-314.pyc')
+2026-02-12 01:53:36,015 - INFO - Upload completed successfully: unicode11-0-0.cpython-314.pyc
+2026-02-12 01:53:36,015 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/unicode10-0-0.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode10-0-0.cpython-314.pyc (19296 bytes)
+2026-02-12 01:53:36,017 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode10-0-0.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,018 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode10-0-0.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,018 - DEBUG - Progress: 19296/19296 bytes
+2026-02-12 01:53:36,018 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,021 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/unicode10-0-0.cpython-314.pyc')
+2026-02-12 01:53:36,023 - INFO - Upload completed successfully: unicode10-0-0.cpython-314.pyc
+2026-02-12 01:53:36,023 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/_versions.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/_versions.cpython-314.pyc (348 bytes)
+2026-02-12 01:53:36,025 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/_versions.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,026 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/_versions.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,026 - DEBUG - Progress: 348/348 bytes
+2026-02-12 01:53:36,026 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,027 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/_versions.cpython-314.pyc')
+2026-02-12 01:53:36,029 - INFO - Upload completed successfully: _versions.cpython-314.pyc
+2026-02-12 01:53:36,029 - INFO - Starting upload: lib/rich/_unicode_data/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/rich/_unicode_data/__pycache__/__init__.cpython-314.pyc (3692 bytes)
+2026-02-12 01:53:36,031 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,031 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,032 - DEBUG - Progress: 3692/3692 bytes
+2026-02-12 01:53:36,032 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,033 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/rich/_unicode_data/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,035 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,036 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info', 511)
+2026-02-12 01:53:36,036 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info
+2026-02-12 01:53:36,037 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/RECORD -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/RECORD (10841 bytes)
+2026-02-12 01:53:36,038 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/RECORD', 'wb')
+2026-02-12 01:53:36,039 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:36,039 - DEBUG - Progress: 10841/10841 bytes
+2026-02-12 01:53:36,039 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,042 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/RECORD')
+2026-02-12 01:53:36,044 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:36,044 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/INSTALLER -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:36,046 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:36,047 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:36,047 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:36,047 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,048 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/INSTALLER')
+2026-02-12 01:53:36,050 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:36,050 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/METADATA -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/METADATA (7288 bytes)
+2026-02-12 01:53:36,052 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/METADATA', 'wb')
+2026-02-12 01:53:36,053 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:36,053 - DEBUG - Progress: 7288/7288 bytes
+2026-02-12 01:53:36,053 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,055 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/METADATA')
+2026-02-12 01:53:36,057 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:36,057 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/WHEEL -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/WHEEL (82 bytes)
+2026-02-12 01:53:36,059 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:36,060 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:36,060 - DEBUG - Progress: 82/82 bytes
+2026-02-12 01:53:36,060 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,061 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/WHEEL')
+2026-02-12 01:53:36,063 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:36,063 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/entry_points.txt -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt (58 bytes)
+2026-02-12 01:53:36,066 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt', 'wb')
+2026-02-12 01:53:36,067 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt', 'wb') -> 00000000
+2026-02-12 01:53:36,067 - DEBUG - Progress: 58/58 bytes
+2026-02-12 01:53:36,067 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,068 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/entry_points.txt')
+2026-02-12 01:53:36,071 - INFO - Upload completed successfully: entry_points.txt
+2026-02-12 01:53:36,071 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses', 511)
+2026-02-12 01:53:36,072 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses
+2026-02-12 01:53:36,072 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it (1073 bytes)
+2026-02-12 01:53:36,073 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it', 'wb')
+2026-02-12 01:53:36,074 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it', 'wb') -> 00000000
+2026-02-12 01:53:36,074 - DEBUG - Progress: 1073/1073 bytes
+2026-02-12 01:53:36,074 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,075 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE.markdown-it')
+2026-02-12 01:53:36,077 - INFO - Upload completed successfully: LICENSE.markdown-it
+2026-02-12 01:53:36,077 - INFO - Starting upload: lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE -> /home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE (1078 bytes)
+2026-02-12 01:53:36,079 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE', 'wb')
+2026-02-12 01:53:36,080 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:36,080 - DEBUG - Progress: 1078/1078 bytes
+2026-02-12 01:53:36,080 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,081 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it_py-4.0.0.dist-info/licenses/LICENSE')
+2026-02-12 01:53:36,082 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:36,082 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it', 511)
+2026-02-12 01:53:36,083 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it
+2026-02-12 01:53:36,083 - INFO - Starting upload: lib/markdown_it/utils.py -> /home/kevin/test/lib/markdown_it/utils.py (5687 bytes)
+2026-02-12 01:53:36,084 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/utils.py', 'wb')
+2026-02-12 01:53:36,085 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:36,085 - DEBUG - Progress: 5687/5687 bytes
+2026-02-12 01:53:36,085 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,086 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/utils.py')
+2026-02-12 01:53:36,088 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:36,089 - INFO - Starting upload: lib/markdown_it/tree.py -> /home/kevin/test/lib/markdown_it/tree.py (11111 bytes)
+2026-02-12 01:53:36,091 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/tree.py', 'wb')
+2026-02-12 01:53:36,092 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/tree.py', 'wb') -> 00000000
+2026-02-12 01:53:36,092 - DEBUG - Progress: 11111/11111 bytes
+2026-02-12 01:53:36,092 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,094 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/tree.py')
+2026-02-12 01:53:36,096 - INFO - Upload completed successfully: tree.py
+2026-02-12 01:53:36,096 - INFO - Starting upload: lib/markdown_it/token.py -> /home/kevin/test/lib/markdown_it/token.py (6381 bytes)
+2026-02-12 01:53:36,097 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/token.py', 'wb')
+2026-02-12 01:53:36,098 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/token.py', 'wb') -> 00000000
+2026-02-12 01:53:36,098 - DEBUG - Progress: 6381/6381 bytes
+2026-02-12 01:53:36,098 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,099 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/token.py')
+2026-02-12 01:53:36,101 - INFO - Upload completed successfully: token.py
+2026-02-12 01:53:36,101 - INFO - Starting upload: lib/markdown_it/ruler.py -> /home/kevin/test/lib/markdown_it/ruler.py (9142 bytes)
+2026-02-12 01:53:36,103 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/ruler.py', 'wb')
+2026-02-12 01:53:36,103 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/ruler.py', 'wb') -> 00000000
+2026-02-12 01:53:36,103 - DEBUG - Progress: 9142/9142 bytes
+2026-02-12 01:53:36,103 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,105 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/ruler.py')
+2026-02-12 01:53:36,106 - INFO - Upload completed successfully: ruler.py
+2026-02-12 01:53:36,106 - INFO - Starting upload: lib/markdown_it/renderer.py -> /home/kevin/test/lib/markdown_it/renderer.py (9947 bytes)
+2026-02-12 01:53:36,108 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/renderer.py', 'wb')
+2026-02-12 01:53:36,109 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/renderer.py', 'wb') -> 00000000
+2026-02-12 01:53:36,109 - DEBUG - Progress: 9947/9947 bytes
+2026-02-12 01:53:36,109 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,110 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/renderer.py')
+2026-02-12 01:53:36,112 - INFO - Upload completed successfully: renderer.py
+2026-02-12 01:53:36,112 - INFO - Starting upload: lib/markdown_it/py.typed -> /home/kevin/test/lib/markdown_it/py.typed (26 bytes)
+2026-02-12 01:53:36,114 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/py.typed', 'wb')
+2026-02-12 01:53:36,114 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/py.typed', 'wb') -> 00000000
+2026-02-12 01:53:36,114 - DEBUG - Progress: 26/26 bytes
+2026-02-12 01:53:36,114 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,115 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/py.typed')
+2026-02-12 01:53:36,117 - INFO - Upload completed successfully: py.typed
+2026-02-12 01:53:36,117 - INFO - Starting upload: lib/markdown_it/port.yaml -> /home/kevin/test/lib/markdown_it/port.yaml (2447 bytes)
+2026-02-12 01:53:36,118 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/port.yaml', 'wb')
+2026-02-12 01:53:36,119 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/port.yaml', 'wb') -> 00000000
+2026-02-12 01:53:36,119 - DEBUG - Progress: 2447/2447 bytes
+2026-02-12 01:53:36,119 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,120 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/port.yaml')
+2026-02-12 01:53:36,122 - INFO - Upload completed successfully: port.yaml
+2026-02-12 01:53:36,122 - INFO - Starting upload: lib/markdown_it/parser_inline.py -> /home/kevin/test/lib/markdown_it/parser_inline.py (5024 bytes)
+2026-02-12 01:53:36,123 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/parser_inline.py', 'wb')
+2026-02-12 01:53:36,124 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/parser_inline.py', 'wb') -> 00000000
+2026-02-12 01:53:36,124 - DEBUG - Progress: 5024/5024 bytes
+2026-02-12 01:53:36,124 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,126 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/parser_inline.py')
+2026-02-12 01:53:36,127 - INFO - Upload completed successfully: parser_inline.py
+2026-02-12 01:53:36,128 - INFO - Starting upload: lib/markdown_it/parser_core.py -> /home/kevin/test/lib/markdown_it/parser_core.py (1016 bytes)
+2026-02-12 01:53:36,129 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/parser_core.py', 'wb')
+2026-02-12 01:53:36,130 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/parser_core.py', 'wb') -> 00000000
+2026-02-12 01:53:36,130 - DEBUG - Progress: 1016/1016 bytes
+2026-02-12 01:53:36,130 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,131 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/parser_core.py')
+2026-02-12 01:53:36,132 - INFO - Upload completed successfully: parser_core.py
+2026-02-12 01:53:36,132 - INFO - Starting upload: lib/markdown_it/parser_block.py -> /home/kevin/test/lib/markdown_it/parser_block.py (3939 bytes)
+2026-02-12 01:53:36,134 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/parser_block.py', 'wb')
+2026-02-12 01:53:36,134 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/parser_block.py', 'wb') -> 00000000
+2026-02-12 01:53:36,135 - DEBUG - Progress: 3939/3939 bytes
+2026-02-12 01:53:36,135 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,136 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/parser_block.py')
+2026-02-12 01:53:36,138 - INFO - Upload completed successfully: parser_block.py
+2026-02-12 01:53:36,138 - INFO - Starting upload: lib/markdown_it/main.py -> /home/kevin/test/lib/markdown_it/main.py (12732 bytes)
+2026-02-12 01:53:36,139 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/main.py', 'wb')
+2026-02-12 01:53:36,140 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/main.py', 'wb') -> 00000000
+2026-02-12 01:53:36,140 - DEBUG - Progress: 12732/12732 bytes
+2026-02-12 01:53:36,140 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,144 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/main.py')
+2026-02-12 01:53:36,146 - INFO - Upload completed successfully: main.py
+2026-02-12 01:53:36,146 - INFO - Starting upload: lib/markdown_it/_punycode.py -> /home/kevin/test/lib/markdown_it/_punycode.py (2373 bytes)
+2026-02-12 01:53:36,148 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/_punycode.py', 'wb')
+2026-02-12 01:53:36,148 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/_punycode.py', 'wb') -> 00000000
+2026-02-12 01:53:36,148 - DEBUG - Progress: 2373/2373 bytes
+2026-02-12 01:53:36,149 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,150 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/_punycode.py')
+2026-02-12 01:53:36,152 - INFO - Upload completed successfully: _punycode.py
+2026-02-12 01:53:36,152 - INFO - Starting upload: lib/markdown_it/_compat.py -> /home/kevin/test/lib/markdown_it/_compat.py (35 bytes)
+2026-02-12 01:53:36,153 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/_compat.py', 'wb')
+2026-02-12 01:53:36,154 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/_compat.py', 'wb') -> 00000000
+2026-02-12 01:53:36,154 - DEBUG - Progress: 35/35 bytes
+2026-02-12 01:53:36,154 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,155 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/_compat.py')
+2026-02-12 01:53:36,157 - INFO - Upload completed successfully: _compat.py
+2026-02-12 01:53:36,157 - INFO - Starting upload: lib/markdown_it/__init__.py -> /home/kevin/test/lib/markdown_it/__init__.py (114 bytes)
+2026-02-12 01:53:36,159 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__init__.py', 'wb')
+2026-02-12 01:53:36,159 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,159 - DEBUG - Progress: 114/114 bytes
+2026-02-12 01:53:36,159 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,160 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__init__.py')
+2026-02-12 01:53:36,162 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,162 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/__pycache__', 511)
+2026-02-12 01:53:36,162 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/__pycache__
+2026-02-12 01:53:36,163 - INFO - Starting upload: lib/markdown_it/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/utils.cpython-314.pyc (12168 bytes)
+2026-02-12 01:53:36,164 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,165 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,165 - DEBUG - Progress: 12168/12168 bytes
+2026-02-12 01:53:36,165 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,167 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:36,169 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:36,169 - INFO - Starting upload: lib/markdown_it/__pycache__/tree.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/tree.cpython-314.pyc (20233 bytes)
+2026-02-12 01:53:36,171 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/tree.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,171 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/tree.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,171 - DEBUG - Progress: 20233/20233 bytes
+2026-02-12 01:53:36,171 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,174 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/tree.cpython-314.pyc')
+2026-02-12 01:53:36,176 - INFO - Upload completed successfully: tree.cpython-314.pyc
+2026-02-12 01:53:36,176 - INFO - Starting upload: lib/markdown_it/__pycache__/token.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/token.cpython-314.pyc (9432 bytes)
+2026-02-12 01:53:36,177 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/token.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,178 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/token.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,178 - DEBUG - Progress: 9432/9432 bytes
+2026-02-12 01:53:36,178 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,180 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/token.cpython-314.pyc')
+2026-02-12 01:53:36,181 - INFO - Upload completed successfully: token.cpython-314.pyc
+2026-02-12 01:53:36,181 - INFO - Starting upload: lib/markdown_it/__pycache__/ruler.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/ruler.cpython-314.pyc (14535 bytes)
+2026-02-12 01:53:36,183 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/ruler.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,184 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/ruler.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,184 - DEBUG - Progress: 14535/14535 bytes
+2026-02-12 01:53:36,184 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,186 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/ruler.cpython-314.pyc')
+2026-02-12 01:53:36,188 - INFO - Upload completed successfully: ruler.cpython-314.pyc
+2026-02-12 01:53:36,188 - INFO - Starting upload: lib/markdown_it/__pycache__/renderer.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/renderer.cpython-314.pyc (14304 bytes)
+2026-02-12 01:53:36,189 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/renderer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,190 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/renderer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,190 - DEBUG - Progress: 14304/14304 bytes
+2026-02-12 01:53:36,190 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,192 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/renderer.cpython-314.pyc')
+2026-02-12 01:53:36,194 - INFO - Upload completed successfully: renderer.cpython-314.pyc
+2026-02-12 01:53:36,194 - INFO - Starting upload: lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc (6069 bytes)
+2026-02-12 01:53:36,195 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,196 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,196 - DEBUG - Progress: 6069/6069 bytes
+2026-02-12 01:53:36,196 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,197 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_inline.cpython-314.pyc')
+2026-02-12 01:53:36,199 - INFO - Upload completed successfully: parser_inline.cpython-314.pyc
+2026-02-12 01:53:36,199 - INFO - Starting upload: lib/markdown_it/__pycache__/parser_core.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc (2199 bytes)
+2026-02-12 01:53:36,201 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,201 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,202 - DEBUG - Progress: 2199/2199 bytes
+2026-02-12 01:53:36,202 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,203 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_core.cpython-314.pyc')
+2026-02-12 01:53:36,205 - INFO - Upload completed successfully: parser_core.cpython-314.pyc
+2026-02-12 01:53:36,205 - INFO - Starting upload: lib/markdown_it/__pycache__/parser_block.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc (4677 bytes)
+2026-02-12 01:53:36,206 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,207 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,207 - DEBUG - Progress: 4677/4677 bytes
+2026-02-12 01:53:36,207 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,209 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/parser_block.cpython-314.pyc')
+2026-02-12 01:53:36,210 - INFO - Upload completed successfully: parser_block.cpython-314.pyc
+2026-02-12 01:53:36,211 - INFO - Starting upload: lib/markdown_it/__pycache__/main.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/main.cpython-314.pyc (20032 bytes)
+2026-02-12 01:53:36,212 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/main.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,212 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/main.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,213 - DEBUG - Progress: 20032/20032 bytes
+2026-02-12 01:53:36,213 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,215 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/main.cpython-314.pyc')
+2026-02-12 01:53:36,217 - INFO - Upload completed successfully: main.cpython-314.pyc
+2026-02-12 01:53:36,217 - INFO - Starting upload: lib/markdown_it/__pycache__/_punycode.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc (3654 bytes)
+2026-02-12 01:53:36,219 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,220 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,220 - DEBUG - Progress: 3654/3654 bytes
+2026-02-12 01:53:36,220 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,221 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/_punycode.cpython-314.pyc')
+2026-02-12 01:53:36,223 - INFO - Upload completed successfully: _punycode.cpython-314.pyc
+2026-02-12 01:53:36,223 - INFO - Starting upload: lib/markdown_it/__pycache__/_compat.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/_compat.cpython-314.pyc (206 bytes)
+2026-02-12 01:53:36,225 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/_compat.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,225 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/_compat.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,225 - DEBUG - Progress: 206/206 bytes
+2026-02-12 01:53:36,225 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,226 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/_compat.cpython-314.pyc')
+2026-02-12 01:53:36,227 - INFO - Upload completed successfully: _compat.cpython-314.pyc
+2026-02-12 01:53:36,228 - INFO - Starting upload: lib/markdown_it/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/__pycache__/__init__.cpython-314.pyc (288 bytes)
+2026-02-12 01:53:36,229 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,229 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,230 - DEBUG - Progress: 288/288 bytes
+2026-02-12 01:53:36,230 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,230 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,232 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,232 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/rules_inline', 511)
+2026-02-12 01:53:36,233 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/rules_inline
+2026-02-12 01:53:36,233 - INFO - Starting upload: lib/markdown_it/rules_inline/text.py -> /home/kevin/test/lib/markdown_it/rules_inline/text.py (1119 bytes)
+2026-02-12 01:53:36,235 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/text.py', 'wb')
+2026-02-12 01:53:36,236 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/text.py', 'wb') -> 00000000
+2026-02-12 01:53:36,236 - DEBUG - Progress: 1119/1119 bytes
+2026-02-12 01:53:36,236 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,237 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/text.py')
+2026-02-12 01:53:36,238 - INFO - Upload completed successfully: text.py
+2026-02-12 01:53:36,238 - INFO - Starting upload: lib/markdown_it/rules_inline/strikethrough.py -> /home/kevin/test/lib/markdown_it/rules_inline/strikethrough.py (3214 bytes)
+2026-02-12 01:53:36,240 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/strikethrough.py', 'wb')
+2026-02-12 01:53:36,240 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/strikethrough.py', 'wb') -> 00000000
+2026-02-12 01:53:36,241 - DEBUG - Progress: 3214/3214 bytes
+2026-02-12 01:53:36,241 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,242 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/strikethrough.py')
+2026-02-12 01:53:36,244 - INFO - Upload completed successfully: strikethrough.py
+2026-02-12 01:53:36,244 - INFO - Starting upload: lib/markdown_it/rules_inline/state_inline.py -> /home/kevin/test/lib/markdown_it/rules_inline/state_inline.py (5003 bytes)
+2026-02-12 01:53:36,245 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/state_inline.py', 'wb')
+2026-02-12 01:53:36,246 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/state_inline.py', 'wb') -> 00000000
+2026-02-12 01:53:36,246 - DEBUG - Progress: 5003/5003 bytes
+2026-02-12 01:53:36,246 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,247 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/state_inline.py')
+2026-02-12 01:53:36,251 - INFO - Upload completed successfully: state_inline.py
+2026-02-12 01:53:36,251 - INFO - Starting upload: lib/markdown_it/rules_inline/newline.py -> /home/kevin/test/lib/markdown_it/rules_inline/newline.py (1297 bytes)
+2026-02-12 01:53:36,253 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/newline.py', 'wb')
+2026-02-12 01:53:36,253 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/newline.py', 'wb') -> 00000000
+2026-02-12 01:53:36,253 - DEBUG - Progress: 1297/1297 bytes
+2026-02-12 01:53:36,254 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,254 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/newline.py')
+2026-02-12 01:53:36,256 - INFO - Upload completed successfully: newline.py
+2026-02-12 01:53:36,256 - INFO - Starting upload: lib/markdown_it/rules_inline/linkify.py -> /home/kevin/test/lib/markdown_it/rules_inline/linkify.py (1706 bytes)
+2026-02-12 01:53:36,257 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/linkify.py', 'wb')
+2026-02-12 01:53:36,258 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/linkify.py', 'wb') -> 00000000
+2026-02-12 01:53:36,258 - DEBUG - Progress: 1706/1706 bytes
+2026-02-12 01:53:36,258 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,259 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/linkify.py')
+2026-02-12 01:53:36,261 - INFO - Upload completed successfully: linkify.py
+2026-02-12 01:53:36,261 - INFO - Starting upload: lib/markdown_it/rules_inline/link.py -> /home/kevin/test/lib/markdown_it/rules_inline/link.py (4258 bytes)
+2026-02-12 01:53:36,263 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/link.py', 'wb')
+2026-02-12 01:53:36,263 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/link.py', 'wb') -> 00000000
+2026-02-12 01:53:36,263 - DEBUG - Progress: 4258/4258 bytes
+2026-02-12 01:53:36,263 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,265 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/link.py')
+2026-02-12 01:53:36,266 - INFO - Upload completed successfully: link.py
+2026-02-12 01:53:36,266 - INFO - Starting upload: lib/markdown_it/rules_inline/image.py -> /home/kevin/test/lib/markdown_it/rules_inline/image.py (4141 bytes)
+2026-02-12 01:53:36,268 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/image.py', 'wb')
+2026-02-12 01:53:36,268 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/image.py', 'wb') -> 00000000
+2026-02-12 01:53:36,268 - DEBUG - Progress: 4141/4141 bytes
+2026-02-12 01:53:36,268 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,270 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/image.py')
+2026-02-12 01:53:36,271 - INFO - Upload completed successfully: image.py
+2026-02-12 01:53:36,271 - INFO - Starting upload: lib/markdown_it/rules_inline/html_inline.py -> /home/kevin/test/lib/markdown_it/rules_inline/html_inline.py (1130 bytes)
+2026-02-12 01:53:36,273 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/html_inline.py', 'wb')
+2026-02-12 01:53:36,273 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/html_inline.py', 'wb') -> 00000000
+2026-02-12 01:53:36,273 - DEBUG - Progress: 1130/1130 bytes
+2026-02-12 01:53:36,273 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,274 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/html_inline.py')
+2026-02-12 01:53:36,276 - INFO - Upload completed successfully: html_inline.py
+2026-02-12 01:53:36,276 - INFO - Starting upload: lib/markdown_it/rules_inline/fragments_join.py -> /home/kevin/test/lib/markdown_it/rules_inline/fragments_join.py (1493 bytes)
+2026-02-12 01:53:36,277 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/fragments_join.py', 'wb')
+2026-02-12 01:53:36,278 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/fragments_join.py', 'wb') -> 00000000
+2026-02-12 01:53:36,278 - DEBUG - Progress: 1493/1493 bytes
+2026-02-12 01:53:36,278 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,279 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/fragments_join.py')
+2026-02-12 01:53:36,280 - INFO - Upload completed successfully: fragments_join.py
+2026-02-12 01:53:36,281 - INFO - Starting upload: lib/markdown_it/rules_inline/escape.py -> /home/kevin/test/lib/markdown_it/rules_inline/escape.py (1659 bytes)
+2026-02-12 01:53:36,282 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/escape.py', 'wb')
+2026-02-12 01:53:36,283 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/escape.py', 'wb') -> 00000000
+2026-02-12 01:53:36,283 - DEBUG - Progress: 1659/1659 bytes
+2026-02-12 01:53:36,283 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,284 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/escape.py')
+2026-02-12 01:53:36,285 - INFO - Upload completed successfully: escape.py
+2026-02-12 01:53:36,285 - INFO - Starting upload: lib/markdown_it/rules_inline/entity.py -> /home/kevin/test/lib/markdown_it/rules_inline/entity.py (1651 bytes)
+2026-02-12 01:53:36,286 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/entity.py', 'wb')
+2026-02-12 01:53:36,287 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/entity.py', 'wb') -> 00000000
+2026-02-12 01:53:36,287 - DEBUG - Progress: 1651/1651 bytes
+2026-02-12 01:53:36,287 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,288 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/entity.py')
+2026-02-12 01:53:36,290 - INFO - Upload completed successfully: entity.py
+2026-02-12 01:53:36,290 - INFO - Starting upload: lib/markdown_it/rules_inline/emphasis.py -> /home/kevin/test/lib/markdown_it/rules_inline/emphasis.py (3123 bytes)
+2026-02-12 01:53:36,292 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/emphasis.py', 'wb')
+2026-02-12 01:53:36,292 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/emphasis.py', 'wb') -> 00000000
+2026-02-12 01:53:36,292 - DEBUG - Progress: 3123/3123 bytes
+2026-02-12 01:53:36,292 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,294 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/emphasis.py')
+2026-02-12 01:53:36,295 - INFO - Upload completed successfully: emphasis.py
+2026-02-12 01:53:36,295 - INFO - Starting upload: lib/markdown_it/rules_inline/balance_pairs.py -> /home/kevin/test/lib/markdown_it/rules_inline/balance_pairs.py (4852 bytes)
+2026-02-12 01:53:36,297 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/balance_pairs.py', 'wb')
+2026-02-12 01:53:36,297 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/balance_pairs.py', 'wb') -> 00000000
+2026-02-12 01:53:36,298 - DEBUG - Progress: 4852/4852 bytes
+2026-02-12 01:53:36,298 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,299 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/balance_pairs.py')
+2026-02-12 01:53:36,301 - INFO - Upload completed successfully: balance_pairs.py
+2026-02-12 01:53:36,301 - INFO - Starting upload: lib/markdown_it/rules_inline/backticks.py -> /home/kevin/test/lib/markdown_it/rules_inline/backticks.py (2037 bytes)
+2026-02-12 01:53:36,302 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/backticks.py', 'wb')
+2026-02-12 01:53:36,303 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/backticks.py', 'wb') -> 00000000
+2026-02-12 01:53:36,303 - DEBUG - Progress: 2037/2037 bytes
+2026-02-12 01:53:36,303 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,304 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/backticks.py')
+2026-02-12 01:53:36,306 - INFO - Upload completed successfully: backticks.py
+2026-02-12 01:53:36,306 - INFO - Starting upload: lib/markdown_it/rules_inline/autolink.py -> /home/kevin/test/lib/markdown_it/rules_inline/autolink.py (2065 bytes)
+2026-02-12 01:53:36,307 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/autolink.py', 'wb')
+2026-02-12 01:53:36,308 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/autolink.py', 'wb') -> 00000000
+2026-02-12 01:53:36,308 - DEBUG - Progress: 2065/2065 bytes
+2026-02-12 01:53:36,308 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,309 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/autolink.py')
+2026-02-12 01:53:36,311 - INFO - Upload completed successfully: autolink.py
+2026-02-12 01:53:36,311 - INFO - Starting upload: lib/markdown_it/rules_inline/__init__.py -> /home/kevin/test/lib/markdown_it/rules_inline/__init__.py (696 bytes)
+2026-02-12 01:53:36,313 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__init__.py', 'wb')
+2026-02-12 01:53:36,313 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,313 - DEBUG - Progress: 696/696 bytes
+2026-02-12 01:53:36,313 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,314 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__init__.py')
+2026-02-12 01:53:36,316 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,316 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__', 511)
+2026-02-12 01:53:36,316 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/rules_inline/__pycache__
+2026-02-12 01:53:36,317 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc (1835 bytes)
+2026-02-12 01:53:36,318 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,319 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,319 - DEBUG - Progress: 1835/1835 bytes
+2026-02-12 01:53:36,319 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,319 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/text.cpython-314.pyc')
+2026-02-12 01:53:36,321 - INFO - Upload completed successfully: text.cpython-314.pyc
+2026-02-12 01:53:36,321 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc (4690 bytes)
+2026-02-12 01:53:36,323 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,323 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,323 - DEBUG - Progress: 4690/4690 bytes
+2026-02-12 01:53:36,323 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,325 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/strikethrough.cpython-314.pyc')
+2026-02-12 01:53:36,326 - INFO - Upload completed successfully: strikethrough.cpython-314.pyc
+2026-02-12 01:53:36,326 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc (6994 bytes)
+2026-02-12 01:53:36,328 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,328 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,328 - DEBUG - Progress: 6994/6994 bytes
+2026-02-12 01:53:36,328 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,330 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/state_inline.cpython-314.pyc')
+2026-02-12 01:53:36,331 - INFO - Upload completed successfully: state_inline.cpython-314.pyc
+2026-02-12 01:53:36,332 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc (1832 bytes)
+2026-02-12 01:53:36,333 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,333 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,334 - DEBUG - Progress: 1832/1832 bytes
+2026-02-12 01:53:36,334 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,334 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/newline.cpython-314.pyc')
+2026-02-12 01:53:36,336 - INFO - Upload completed successfully: newline.cpython-314.pyc
+2026-02-12 01:53:36,336 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc (3018 bytes)
+2026-02-12 01:53:36,337 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,338 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,338 - DEBUG - Progress: 3018/3018 bytes
+2026-02-12 01:53:36,338 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,340 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/linkify.cpython-314.pyc')
+2026-02-12 01:53:36,342 - INFO - Upload completed successfully: linkify.cpython-314.pyc
+2026-02-12 01:53:36,342 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc (4452 bytes)
+2026-02-12 01:53:36,343 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,344 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,344 - DEBUG - Progress: 4452/4452 bytes
+2026-02-12 01:53:36,344 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,345 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/link.cpython-314.pyc')
+2026-02-12 01:53:36,347 - INFO - Upload completed successfully: link.cpython-314.pyc
+2026-02-12 01:53:36,347 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc (4648 bytes)
+2026-02-12 01:53:36,348 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,349 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,349 - DEBUG - Progress: 4648/4648 bytes
+2026-02-12 01:53:36,349 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,350 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/image.cpython-314.pyc')
+2026-02-12 01:53:36,352 - INFO - Upload completed successfully: image.cpython-314.pyc
+2026-02-12 01:53:36,352 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc (2418 bytes)
+2026-02-12 01:53:36,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,354 - DEBUG - Progress: 2418/2418 bytes
+2026-02-12 01:53:36,354 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,356 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/html_inline.cpython-314.pyc')
+2026-02-12 01:53:36,357 - INFO - Upload completed successfully: html_inline.cpython-314.pyc
+2026-02-12 01:53:36,357 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc (2158 bytes)
+2026-02-12 01:53:36,359 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,359 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,359 - DEBUG - Progress: 2158/2158 bytes
+2026-02-12 01:53:36,359 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,361 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/fragments_join.cpython-314.pyc')
+2026-02-12 01:53:36,362 - INFO - Upload completed successfully: fragments_join.cpython-314.pyc
+2026-02-12 01:53:36,362 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc (2595 bytes)
+2026-02-12 01:53:36,364 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,365 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,365 - DEBUG - Progress: 2595/2595 bytes
+2026-02-12 01:53:36,365 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,366 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/escape.cpython-314.pyc')
+2026-02-12 01:53:36,370 - INFO - Upload completed successfully: escape.cpython-314.pyc
+2026-02-12 01:53:36,370 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc (2740 bytes)
+2026-02-12 01:53:36,372 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,373 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,373 - DEBUG - Progress: 2740/2740 bytes
+2026-02-12 01:53:36,373 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,374 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/entity.cpython-314.pyc')
+2026-02-12 01:53:36,376 - INFO - Upload completed successfully: entity.cpython-314.pyc
+2026-02-12 01:53:36,376 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc (4674 bytes)
+2026-02-12 01:53:36,378 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,378 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,378 - DEBUG - Progress: 4674/4674 bytes
+2026-02-12 01:53:36,378 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,380 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/emphasis.cpython-314.pyc')
+2026-02-12 01:53:36,383 - INFO - Upload completed successfully: emphasis.cpython-314.pyc
+2026-02-12 01:53:36,383 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc (3883 bytes)
+2026-02-12 01:53:36,384 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,385 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,385 - DEBUG - Progress: 3883/3883 bytes
+2026-02-12 01:53:36,385 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,386 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/balance_pairs.cpython-314.pyc')
+2026-02-12 01:53:36,388 - INFO - Upload completed successfully: balance_pairs.cpython-314.pyc
+2026-02-12 01:53:36,388 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc (2707 bytes)
+2026-02-12 01:53:36,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,390 - DEBUG - Progress: 2707/2707 bytes
+2026-02-12 01:53:36,390 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,392 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/backticks.cpython-314.pyc')
+2026-02-12 01:53:36,394 - INFO - Upload completed successfully: backticks.cpython-314.pyc
+2026-02-12 01:53:36,394 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc (3035 bytes)
+2026-02-12 01:53:36,395 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,396 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,396 - DEBUG - Progress: 3035/3035 bytes
+2026-02-12 01:53:36,396 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,397 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/autolink.cpython-314.pyc')
+2026-02-12 01:53:36,399 - INFO - Upload completed successfully: autolink.cpython-314.pyc
+2026-02-12 01:53:36,399 - INFO - Starting upload: lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc (783 bytes)
+2026-02-12 01:53:36,401 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,401 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,402 - DEBUG - Progress: 783/783 bytes
+2026-02-12 01:53:36,402 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,402 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_inline/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,404 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,404 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/rules_core', 511)
+2026-02-12 01:53:36,405 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/rules_core
+2026-02-12 01:53:36,405 - INFO - Starting upload: lib/markdown_it/rules_core/text_join.py -> /home/kevin/test/lib/markdown_it/rules_core/text_join.py (1173 bytes)
+2026-02-12 01:53:36,406 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/text_join.py', 'wb')
+2026-02-12 01:53:36,407 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/text_join.py', 'wb') -> 00000000
+2026-02-12 01:53:36,407 - DEBUG - Progress: 1173/1173 bytes
+2026-02-12 01:53:36,407 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,408 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/text_join.py')
+2026-02-12 01:53:36,410 - INFO - Upload completed successfully: text_join.py
+2026-02-12 01:53:36,410 - INFO - Starting upload: lib/markdown_it/rules_core/state_core.py -> /home/kevin/test/lib/markdown_it/rules_core/state_core.py (570 bytes)
+2026-02-12 01:53:36,412 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/state_core.py', 'wb')
+2026-02-12 01:53:36,412 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/state_core.py', 'wb') -> 00000000
+2026-02-12 01:53:36,412 - DEBUG - Progress: 570/570 bytes
+2026-02-12 01:53:36,412 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,413 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/state_core.py')
+2026-02-12 01:53:36,415 - INFO - Upload completed successfully: state_core.py
+2026-02-12 01:53:36,415 - INFO - Starting upload: lib/markdown_it/rules_core/smartquotes.py -> /home/kevin/test/lib/markdown_it/rules_core/smartquotes.py (7443 bytes)
+2026-02-12 01:53:36,416 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/smartquotes.py', 'wb')
+2026-02-12 01:53:36,417 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/smartquotes.py', 'wb') -> 00000000
+2026-02-12 01:53:36,417 - DEBUG - Progress: 7443/7443 bytes
+2026-02-12 01:53:36,417 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,418 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/smartquotes.py')
+2026-02-12 01:53:36,420 - INFO - Upload completed successfully: smartquotes.py
+2026-02-12 01:53:36,420 - INFO - Starting upload: lib/markdown_it/rules_core/replacements.py -> /home/kevin/test/lib/markdown_it/rules_core/replacements.py (3471 bytes)
+2026-02-12 01:53:36,422 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/replacements.py', 'wb')
+2026-02-12 01:53:36,422 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/replacements.py', 'wb') -> 00000000
+2026-02-12 01:53:36,422 - DEBUG - Progress: 3471/3471 bytes
+2026-02-12 01:53:36,422 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,424 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/replacements.py')
+2026-02-12 01:53:36,425 - INFO - Upload completed successfully: replacements.py
+2026-02-12 01:53:36,425 - INFO - Starting upload: lib/markdown_it/rules_core/normalize.py -> /home/kevin/test/lib/markdown_it/rules_core/normalize.py (403 bytes)
+2026-02-12 01:53:36,427 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/normalize.py', 'wb')
+2026-02-12 01:53:36,427 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/normalize.py', 'wb') -> 00000000
+2026-02-12 01:53:36,427 - DEBUG - Progress: 403/403 bytes
+2026-02-12 01:53:36,427 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,428 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/normalize.py')
+2026-02-12 01:53:36,430 - INFO - Upload completed successfully: normalize.py
+2026-02-12 01:53:36,430 - INFO - Starting upload: lib/markdown_it/rules_core/linkify.py -> /home/kevin/test/lib/markdown_it/rules_core/linkify.py (5141 bytes)
+2026-02-12 01:53:36,431 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/linkify.py', 'wb')
+2026-02-12 01:53:36,432 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/linkify.py', 'wb') -> 00000000
+2026-02-12 01:53:36,432 - DEBUG - Progress: 5141/5141 bytes
+2026-02-12 01:53:36,432 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,434 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/linkify.py')
+2026-02-12 01:53:36,436 - INFO - Upload completed successfully: linkify.py
+2026-02-12 01:53:36,436 - INFO - Starting upload: lib/markdown_it/rules_core/inline.py -> /home/kevin/test/lib/markdown_it/rules_core/inline.py (325 bytes)
+2026-02-12 01:53:36,437 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/inline.py', 'wb')
+2026-02-12 01:53:36,438 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/inline.py', 'wb') -> 00000000
+2026-02-12 01:53:36,438 - DEBUG - Progress: 325/325 bytes
+2026-02-12 01:53:36,438 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,439 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/inline.py')
+2026-02-12 01:53:36,440 - INFO - Upload completed successfully: inline.py
+2026-02-12 01:53:36,440 - INFO - Starting upload: lib/markdown_it/rules_core/block.py -> /home/kevin/test/lib/markdown_it/rules_core/block.py (372 bytes)
+2026-02-12 01:53:36,442 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/block.py', 'wb')
+2026-02-12 01:53:36,442 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/block.py', 'wb') -> 00000000
+2026-02-12 01:53:36,443 - DEBUG - Progress: 372/372 bytes
+2026-02-12 01:53:36,443 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,443 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/block.py')
+2026-02-12 01:53:36,445 - INFO - Upload completed successfully: block.py
+2026-02-12 01:53:36,445 - INFO - Starting upload: lib/markdown_it/rules_core/__init__.py -> /home/kevin/test/lib/markdown_it/rules_core/__init__.py (394 bytes)
+2026-02-12 01:53:36,446 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__init__.py', 'wb')
+2026-02-12 01:53:36,447 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,447 - DEBUG - Progress: 394/394 bytes
+2026-02-12 01:53:36,447 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,448 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__init__.py')
+2026-02-12 01:53:36,449 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,450 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__', 511)
+2026-02-12 01:53:36,450 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/rules_core/__pycache__
+2026-02-12 01:53:36,451 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc (1659 bytes)
+2026-02-12 01:53:36,452 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,453 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,453 - DEBUG - Progress: 1659/1659 bytes
+2026-02-12 01:53:36,453 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,454 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/text_join.cpython-314.pyc')
+2026-02-12 01:53:36,456 - INFO - Upload completed successfully: text_join.cpython-314.pyc
+2026-02-12 01:53:36,456 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc (1297 bytes)
+2026-02-12 01:53:36,457 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,458 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,458 - DEBUG - Progress: 1297/1297 bytes
+2026-02-12 01:53:36,458 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,459 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/state_core.cpython-314.pyc')
+2026-02-12 01:53:36,461 - INFO - Upload completed successfully: state_core.cpython-314.pyc
+2026-02-12 01:53:36,461 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc (7365 bytes)
+2026-02-12 01:53:36,462 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,463 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,463 - DEBUG - Progress: 7365/7365 bytes
+2026-02-12 01:53:36,463 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,465 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/smartquotes.cpython-314.pyc')
+2026-02-12 01:53:36,467 - INFO - Upload completed successfully: smartquotes.cpython-314.pyc
+2026-02-12 01:53:36,467 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc (5629 bytes)
+2026-02-12 01:53:36,468 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,469 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,469 - DEBUG - Progress: 5629/5629 bytes
+2026-02-12 01:53:36,469 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,470 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/replacements.cpython-314.pyc')
+2026-02-12 01:53:36,472 - INFO - Upload completed successfully: replacements.cpython-314.pyc
+2026-02-12 01:53:36,472 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc (901 bytes)
+2026-02-12 01:53:36,474 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,474 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,475 - DEBUG - Progress: 901/901 bytes
+2026-02-12 01:53:36,475 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,475 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/normalize.cpython-314.pyc')
+2026-02-12 01:53:36,477 - INFO - Upload completed successfully: normalize.cpython-314.pyc
+2026-02-12 01:53:36,478 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc (5684 bytes)
+2026-02-12 01:53:36,479 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,480 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,480 - DEBUG - Progress: 5684/5684 bytes
+2026-02-12 01:53:36,480 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,481 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/linkify.cpython-314.pyc')
+2026-02-12 01:53:36,483 - INFO - Upload completed successfully: linkify.cpython-314.pyc
+2026-02-12 01:53:36,483 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc (946 bytes)
+2026-02-12 01:53:36,485 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,485 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,485 - DEBUG - Progress: 946/946 bytes
+2026-02-12 01:53:36,485 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,486 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/inline.cpython-314.pyc')
+2026-02-12 01:53:36,488 - INFO - Upload completed successfully: inline.cpython-314.pyc
+2026-02-12 01:53:36,489 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc (1095 bytes)
+2026-02-12 01:53:36,490 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,491 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,491 - DEBUG - Progress: 1095/1095 bytes
+2026-02-12 01:53:36,491 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,492 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/block.cpython-314.pyc')
+2026-02-12 01:53:36,493 - INFO - Upload completed successfully: block.cpython-314.pyc
+2026-02-12 01:53:36,493 - INFO - Starting upload: lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc (516 bytes)
+2026-02-12 01:53:36,495 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,495 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,495 - DEBUG - Progress: 516/516 bytes
+2026-02-12 01:53:36,495 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,496 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_core/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,498 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,498 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/rules_block', 511)
+2026-02-12 01:53:36,498 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/rules_block
+2026-02-12 01:53:36,498 - INFO - Starting upload: lib/markdown_it/rules_block/table.py -> /home/kevin/test/lib/markdown_it/rules_block/table.py (7682 bytes)
+2026-02-12 01:53:36,500 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/table.py', 'wb')
+2026-02-12 01:53:36,500 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/table.py', 'wb') -> 00000000
+2026-02-12 01:53:36,500 - DEBUG - Progress: 7682/7682 bytes
+2026-02-12 01:53:36,501 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,502 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/table.py')
+2026-02-12 01:53:36,503 - INFO - Upload completed successfully: table.py
+2026-02-12 01:53:36,503 - INFO - Starting upload: lib/markdown_it/rules_block/state_block.py -> /home/kevin/test/lib/markdown_it/rules_block/state_block.py (8422 bytes)
+2026-02-12 01:53:36,505 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/state_block.py', 'wb')
+2026-02-12 01:53:36,505 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/state_block.py', 'wb') -> 00000000
+2026-02-12 01:53:36,505 - DEBUG - Progress: 8422/8422 bytes
+2026-02-12 01:53:36,505 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,507 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/state_block.py')
+2026-02-12 01:53:36,508 - INFO - Upload completed successfully: state_block.py
+2026-02-12 01:53:36,508 - INFO - Starting upload: lib/markdown_it/rules_block/reference.py -> /home/kevin/test/lib/markdown_it/rules_block/reference.py (6983 bytes)
+2026-02-12 01:53:36,510 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/reference.py', 'wb')
+2026-02-12 01:53:36,510 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/reference.py', 'wb') -> 00000000
+2026-02-12 01:53:36,510 - DEBUG - Progress: 6983/6983 bytes
+2026-02-12 01:53:36,511 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,512 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/reference.py')
+2026-02-12 01:53:36,514 - INFO - Upload completed successfully: reference.py
+2026-02-12 01:53:36,514 - INFO - Starting upload: lib/markdown_it/rules_block/paragraph.py -> /home/kevin/test/lib/markdown_it/rules_block/paragraph.py (1819 bytes)
+2026-02-12 01:53:36,515 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/paragraph.py', 'wb')
+2026-02-12 01:53:36,516 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/paragraph.py', 'wb') -> 00000000
+2026-02-12 01:53:36,516 - DEBUG - Progress: 1819/1819 bytes
+2026-02-12 01:53:36,516 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,517 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/paragraph.py')
+2026-02-12 01:53:36,519 - INFO - Upload completed successfully: paragraph.py
+2026-02-12 01:53:36,519 - INFO - Starting upload: lib/markdown_it/rules_block/list.py -> /home/kevin/test/lib/markdown_it/rules_block/list.py (9668 bytes)
+2026-02-12 01:53:36,520 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/list.py', 'wb')
+2026-02-12 01:53:36,521 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/list.py', 'wb') -> 00000000
+2026-02-12 01:53:36,521 - DEBUG - Progress: 9668/9668 bytes
+2026-02-12 01:53:36,521 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,523 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/list.py')
+2026-02-12 01:53:36,524 - INFO - Upload completed successfully: list.py
+2026-02-12 01:53:36,524 - INFO - Starting upload: lib/markdown_it/rules_block/lheading.py -> /home/kevin/test/lib/markdown_it/rules_block/lheading.py (2625 bytes)
+2026-02-12 01:53:36,526 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/lheading.py', 'wb')
+2026-02-12 01:53:36,527 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/lheading.py', 'wb') -> 00000000
+2026-02-12 01:53:36,527 - DEBUG - Progress: 2625/2625 bytes
+2026-02-12 01:53:36,527 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,528 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/lheading.py')
+2026-02-12 01:53:36,531 - INFO - Upload completed successfully: lheading.py
+2026-02-12 01:53:36,531 - INFO - Starting upload: lib/markdown_it/rules_block/html_block.py -> /home/kevin/test/lib/markdown_it/rules_block/html_block.py (2721 bytes)
+2026-02-12 01:53:36,532 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/html_block.py', 'wb')
+2026-02-12 01:53:36,533 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/html_block.py', 'wb') -> 00000000
+2026-02-12 01:53:36,533 - DEBUG - Progress: 2721/2721 bytes
+2026-02-12 01:53:36,533 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,534 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/html_block.py')
+2026-02-12 01:53:36,536 - INFO - Upload completed successfully: html_block.py
+2026-02-12 01:53:36,536 - INFO - Starting upload: lib/markdown_it/rules_block/hr.py -> /home/kevin/test/lib/markdown_it/rules_block/hr.py (1227 bytes)
+2026-02-12 01:53:36,537 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/hr.py', 'wb')
+2026-02-12 01:53:36,538 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/hr.py', 'wb') -> 00000000
+2026-02-12 01:53:36,538 - DEBUG - Progress: 1227/1227 bytes
+2026-02-12 01:53:36,538 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,539 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/hr.py')
+2026-02-12 01:53:36,540 - INFO - Upload completed successfully: hr.py
+2026-02-12 01:53:36,540 - INFO - Starting upload: lib/markdown_it/rules_block/heading.py -> /home/kevin/test/lib/markdown_it/rules_block/heading.py (1745 bytes)
+2026-02-12 01:53:36,542 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/heading.py', 'wb')
+2026-02-12 01:53:36,542 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/heading.py', 'wb') -> 00000000
+2026-02-12 01:53:36,543 - DEBUG - Progress: 1745/1745 bytes
+2026-02-12 01:53:36,543 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,543 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/heading.py')
+2026-02-12 01:53:36,545 - INFO - Upload completed successfully: heading.py
+2026-02-12 01:53:36,545 - INFO - Starting upload: lib/markdown_it/rules_block/fence.py -> /home/kevin/test/lib/markdown_it/rules_block/fence.py (2537 bytes)
+2026-02-12 01:53:36,547 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/fence.py', 'wb')
+2026-02-12 01:53:36,547 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/fence.py', 'wb') -> 00000000
+2026-02-12 01:53:36,547 - DEBUG - Progress: 2537/2537 bytes
+2026-02-12 01:53:36,548 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,549 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/fence.py')
+2026-02-12 01:53:36,551 - INFO - Upload completed successfully: fence.py
+2026-02-12 01:53:36,551 - INFO - Starting upload: lib/markdown_it/rules_block/code.py -> /home/kevin/test/lib/markdown_it/rules_block/code.py (860 bytes)
+2026-02-12 01:53:36,552 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/code.py', 'wb')
+2026-02-12 01:53:36,553 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/code.py', 'wb') -> 00000000
+2026-02-12 01:53:36,553 - DEBUG - Progress: 860/860 bytes
+2026-02-12 01:53:36,553 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,554 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/code.py')
+2026-02-12 01:53:36,555 - INFO - Upload completed successfully: code.py
+2026-02-12 01:53:36,555 - INFO - Starting upload: lib/markdown_it/rules_block/blockquote.py -> /home/kevin/test/lib/markdown_it/rules_block/blockquote.py (8887 bytes)
+2026-02-12 01:53:36,557 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/blockquote.py', 'wb')
+2026-02-12 01:53:36,557 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/blockquote.py', 'wb') -> 00000000
+2026-02-12 01:53:36,557 - DEBUG - Progress: 8887/8887 bytes
+2026-02-12 01:53:36,557 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,559 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/blockquote.py')
+2026-02-12 01:53:36,561 - INFO - Upload completed successfully: blockquote.py
+2026-02-12 01:53:36,561 - INFO - Starting upload: lib/markdown_it/rules_block/__init__.py -> /home/kevin/test/lib/markdown_it/rules_block/__init__.py (553 bytes)
+2026-02-12 01:53:36,562 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__init__.py', 'wb')
+2026-02-12 01:53:36,563 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,563 - DEBUG - Progress: 553/553 bytes
+2026-02-12 01:53:36,563 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,564 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__init__.py')
+2026-02-12 01:53:36,565 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,565 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__', 511)
+2026-02-12 01:53:36,566 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/rules_block/__pycache__
+2026-02-12 01:53:36,566 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc (8444 bytes)
+2026-02-12 01:53:36,567 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,568 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,568 - DEBUG - Progress: 8444/8444 bytes
+2026-02-12 01:53:36,568 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,569 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/table.cpython-314.pyc')
+2026-02-12 01:53:36,571 - INFO - Upload completed successfully: table.cpython-314.pyc
+2026-02-12 01:53:36,571 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc (11572 bytes)
+2026-02-12 01:53:36,573 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,573 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,573 - DEBUG - Progress: 11572/11572 bytes
+2026-02-12 01:53:36,573 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,575 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/state_block.cpython-314.pyc')
+2026-02-12 01:53:36,577 - INFO - Upload completed successfully: state_block.cpython-314.pyc
+2026-02-12 01:53:36,577 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc (6839 bytes)
+2026-02-12 01:53:36,578 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,579 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,579 - DEBUG - Progress: 6839/6839 bytes
+2026-02-12 01:53:36,579 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,580 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/reference.cpython-314.pyc')
+2026-02-12 01:53:36,582 - INFO - Upload completed successfully: reference.cpython-314.pyc
+2026-02-12 01:53:36,582 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc (2348 bytes)
+2026-02-12 01:53:36,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,584 - DEBUG - Progress: 2348/2348 bytes
+2026-02-12 01:53:36,584 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,586 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/paragraph.cpython-314.pyc')
+2026-02-12 01:53:36,587 - INFO - Upload completed successfully: paragraph.cpython-314.pyc
+2026-02-12 01:53:36,587 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc (9468 bytes)
+2026-02-12 01:53:36,589 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,589 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,589 - DEBUG - Progress: 9468/9468 bytes
+2026-02-12 01:53:36,589 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,591 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/list.cpython-314.pyc')
+2026-02-12 01:53:36,592 - INFO - Upload completed successfully: list.cpython-314.pyc
+2026-02-12 01:53:36,592 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc (3186 bytes)
+2026-02-12 01:53:36,594 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,595 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,595 - DEBUG - Progress: 3186/3186 bytes
+2026-02-12 01:53:36,595 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,596 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/lheading.cpython-314.pyc')
+2026-02-12 01:53:36,597 - INFO - Upload completed successfully: lheading.cpython-314.pyc
+2026-02-12 01:53:36,598 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc (3980 bytes)
+2026-02-12 01:53:36,599 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,600 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,600 - DEBUG - Progress: 3980/3980 bytes
+2026-02-12 01:53:36,600 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,601 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/html_block.cpython-314.pyc')
+2026-02-12 01:53:36,603 - INFO - Upload completed successfully: html_block.cpython-314.pyc
+2026-02-12 01:53:36,603 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc (2001 bytes)
+2026-02-12 01:53:36,604 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,605 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,605 - DEBUG - Progress: 2001/2001 bytes
+2026-02-12 01:53:36,605 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,606 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/hr.cpython-314.pyc')
+2026-02-12 01:53:36,607 - INFO - Upload completed successfully: hr.cpython-314.pyc
+2026-02-12 01:53:36,607 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc (2875 bytes)
+2026-02-12 01:53:36,609 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,609 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,610 - DEBUG - Progress: 2875/2875 bytes
+2026-02-12 01:53:36,610 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,611 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/heading.cpython-314.pyc')
+2026-02-12 01:53:36,612 - INFO - Upload completed successfully: heading.cpython-314.pyc
+2026-02-12 01:53:36,612 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc (2854 bytes)
+2026-02-12 01:53:36,614 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,614 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,614 - DEBUG - Progress: 2854/2854 bytes
+2026-02-12 01:53:36,614 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,616 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/fence.cpython-314.pyc')
+2026-02-12 01:53:36,617 - INFO - Upload completed successfully: fence.cpython-314.pyc
+2026-02-12 01:53:36,617 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc (1566 bytes)
+2026-02-12 01:53:36,619 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,619 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,619 - DEBUG - Progress: 1566/1566 bytes
+2026-02-12 01:53:36,619 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,620 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/code.cpython-314.pyc')
+2026-02-12 01:53:36,622 - INFO - Upload completed successfully: code.cpython-314.pyc
+2026-02-12 01:53:36,622 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc (7923 bytes)
+2026-02-12 01:53:36,624 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,625 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,625 - DEBUG - Progress: 7923/7923 bytes
+2026-02-12 01:53:36,625 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,626 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/blockquote.cpython-314.pyc')
+2026-02-12 01:53:36,628 - INFO - Upload completed successfully: blockquote.cpython-314.pyc
+2026-02-12 01:53:36,628 - INFO - Starting upload: lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc (652 bytes)
+2026-02-12 01:53:36,629 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,630 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,630 - DEBUG - Progress: 652/652 bytes
+2026-02-12 01:53:36,630 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,631 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/rules_block/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,632 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,632 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/presets', 511)
+2026-02-12 01:53:36,633 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/presets
+2026-02-12 01:53:36,633 - INFO - Starting upload: lib/markdown_it/presets/zero.py -> /home/kevin/test/lib/markdown_it/presets/zero.py (2113 bytes)
+2026-02-12 01:53:36,634 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/zero.py', 'wb')
+2026-02-12 01:53:36,635 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/zero.py', 'wb') -> 00000000
+2026-02-12 01:53:36,635 - DEBUG - Progress: 2113/2113 bytes
+2026-02-12 01:53:36,635 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,636 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/zero.py')
+2026-02-12 01:53:36,638 - INFO - Upload completed successfully: zero.py
+2026-02-12 01:53:36,638 - INFO - Starting upload: lib/markdown_it/presets/default.py -> /home/kevin/test/lib/markdown_it/presets/default.py (1811 bytes)
+2026-02-12 01:53:36,639 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/default.py', 'wb')
+2026-02-12 01:53:36,640 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/default.py', 'wb') -> 00000000
+2026-02-12 01:53:36,640 - DEBUG - Progress: 1811/1811 bytes
+2026-02-12 01:53:36,640 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,641 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/default.py')
+2026-02-12 01:53:36,642 - INFO - Upload completed successfully: default.py
+2026-02-12 01:53:36,642 - INFO - Starting upload: lib/markdown_it/presets/commonmark.py -> /home/kevin/test/lib/markdown_it/presets/commonmark.py (2869 bytes)
+2026-02-12 01:53:36,644 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/commonmark.py', 'wb')
+2026-02-12 01:53:36,644 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/commonmark.py', 'wb') -> 00000000
+2026-02-12 01:53:36,644 - DEBUG - Progress: 2869/2869 bytes
+2026-02-12 01:53:36,644 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,646 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/commonmark.py')
+2026-02-12 01:53:36,647 - INFO - Upload completed successfully: commonmark.py
+2026-02-12 01:53:36,647 - INFO - Starting upload: lib/markdown_it/presets/__init__.py -> /home/kevin/test/lib/markdown_it/presets/__init__.py (970 bytes)
+2026-02-12 01:53:36,649 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__init__.py', 'wb')
+2026-02-12 01:53:36,649 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,649 - DEBUG - Progress: 970/970 bytes
+2026-02-12 01:53:36,649 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,650 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/__init__.py')
+2026-02-12 01:53:36,652 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,652 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/presets/__pycache__', 511)
+2026-02-12 01:53:36,652 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/presets/__pycache__
+2026-02-12 01:53:36,652 - INFO - Starting upload: lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc (1078 bytes)
+2026-02-12 01:53:36,654 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,654 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,654 - DEBUG - Progress: 1078/1078 bytes
+2026-02-12 01:53:36,655 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,655 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/zero.cpython-314.pyc')
+2026-02-12 01:53:36,657 - INFO - Upload completed successfully: zero.cpython-314.pyc
+2026-02-12 01:53:36,657 - INFO - Starting upload: lib/markdown_it/presets/__pycache__/default.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc (816 bytes)
+2026-02-12 01:53:36,658 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,659 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,659 - DEBUG - Progress: 816/816 bytes
+2026-02-12 01:53:36,659 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,660 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/default.cpython-314.pyc')
+2026-02-12 01:53:36,661 - INFO - Upload completed successfully: default.cpython-314.pyc
+2026-02-12 01:53:36,662 - INFO - Starting upload: lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc (1305 bytes)
+2026-02-12 01:53:36,663 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,664 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,664 - DEBUG - Progress: 1305/1305 bytes
+2026-02-12 01:53:36,664 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,665 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/commonmark.cpython-314.pyc')
+2026-02-12 01:53:36,666 - INFO - Upload completed successfully: commonmark.cpython-314.pyc
+2026-02-12 01:53:36,666 - INFO - Starting upload: lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc (1931 bytes)
+2026-02-12 01:53:36,667 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,668 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,668 - DEBUG - Progress: 1931/1931 bytes
+2026-02-12 01:53:36,668 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,669 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/presets/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,670 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,671 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/helpers', 511)
+2026-02-12 01:53:36,671 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/helpers
+2026-02-12 01:53:36,671 - INFO - Starting upload: lib/markdown_it/helpers/parse_link_title.py -> /home/kevin/test/lib/markdown_it/helpers/parse_link_title.py (2273 bytes)
+2026-02-12 01:53:36,673 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_title.py', 'wb')
+2026-02-12 01:53:36,673 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_title.py', 'wb') -> 00000000
+2026-02-12 01:53:36,673 - DEBUG - Progress: 2273/2273 bytes
+2026-02-12 01:53:36,673 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,674 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_title.py')
+2026-02-12 01:53:36,676 - INFO - Upload completed successfully: parse_link_title.py
+2026-02-12 01:53:36,676 - INFO - Starting upload: lib/markdown_it/helpers/parse_link_label.py -> /home/kevin/test/lib/markdown_it/helpers/parse_link_label.py (1037 bytes)
+2026-02-12 01:53:36,678 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_label.py', 'wb')
+2026-02-12 01:53:36,678 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_label.py', 'wb') -> 00000000
+2026-02-12 01:53:36,678 - DEBUG - Progress: 1037/1037 bytes
+2026-02-12 01:53:36,678 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,679 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_label.py')
+2026-02-12 01:53:36,681 - INFO - Upload completed successfully: parse_link_label.py
+2026-02-12 01:53:36,681 - INFO - Starting upload: lib/markdown_it/helpers/parse_link_destination.py -> /home/kevin/test/lib/markdown_it/helpers/parse_link_destination.py (1906 bytes)
+2026-02-12 01:53:36,682 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_destination.py', 'wb')
+2026-02-12 01:53:36,682 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_destination.py', 'wb') -> 00000000
+2026-02-12 01:53:36,683 - DEBUG - Progress: 1906/1906 bytes
+2026-02-12 01:53:36,683 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,684 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/parse_link_destination.py')
+2026-02-12 01:53:36,686 - INFO - Upload completed successfully: parse_link_destination.py
+2026-02-12 01:53:36,686 - INFO - Starting upload: lib/markdown_it/helpers/__init__.py -> /home/kevin/test/lib/markdown_it/helpers/__init__.py (253 bytes)
+2026-02-12 01:53:36,688 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__init__.py', 'wb')
+2026-02-12 01:53:36,688 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,688 - DEBUG - Progress: 253/253 bytes
+2026-02-12 01:53:36,689 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,689 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/__init__.py')
+2026-02-12 01:53:36,691 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,691 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__', 511)
+2026-02-12 01:53:36,691 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/helpers/__pycache__
+2026-02-12 01:53:36,692 - INFO - Starting upload: lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc (2966 bytes)
+2026-02-12 01:53:36,693 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,694 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,694 - DEBUG - Progress: 2966/2966 bytes
+2026-02-12 01:53:36,694 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,695 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_title.cpython-314.pyc')
+2026-02-12 01:53:36,696 - INFO - Upload completed successfully: parse_link_title.cpython-314.pyc
+2026-02-12 01:53:36,697 - INFO - Starting upload: lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc (1576 bytes)
+2026-02-12 01:53:36,698 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,699 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,699 - DEBUG - Progress: 1576/1576 bytes
+2026-02-12 01:53:36,699 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,700 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_label.cpython-314.pyc')
+2026-02-12 01:53:36,702 - INFO - Upload completed successfully: parse_link_label.cpython-314.pyc
+2026-02-12 01:53:36,702 - INFO - Starting upload: lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc (2563 bytes)
+2026-02-12 01:53:36,703 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,704 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,704 - DEBUG - Progress: 2563/2563 bytes
+2026-02-12 01:53:36,704 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,705 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/parse_link_destination.cpython-314.pyc')
+2026-02-12 01:53:36,707 - INFO - Upload completed successfully: parse_link_destination.cpython-314.pyc
+2026-02-12 01:53:36,707 - INFO - Starting upload: lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc (423 bytes)
+2026-02-12 01:53:36,710 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,710 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,710 - DEBUG - Progress: 423/423 bytes
+2026-02-12 01:53:36,710 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,711 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/helpers/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,713 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,713 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/common', 511)
+2026-02-12 01:53:36,713 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/common
+2026-02-12 01:53:36,713 - INFO - Starting upload: lib/markdown_it/common/utils.py -> /home/kevin/test/lib/markdown_it/common/utils.py (8734 bytes)
+2026-02-12 01:53:36,715 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/utils.py', 'wb')
+2026-02-12 01:53:36,716 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/utils.py', 'wb') -> 00000000
+2026-02-12 01:53:36,716 - DEBUG - Progress: 8734/8734 bytes
+2026-02-12 01:53:36,716 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,717 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/utils.py')
+2026-02-12 01:53:36,719 - INFO - Upload completed successfully: utils.py
+2026-02-12 01:53:36,719 - INFO - Starting upload: lib/markdown_it/common/normalize_url.py -> /home/kevin/test/lib/markdown_it/common/normalize_url.py (2568 bytes)
+2026-02-12 01:53:36,721 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/normalize_url.py', 'wb')
+2026-02-12 01:53:36,721 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/normalize_url.py', 'wb') -> 00000000
+2026-02-12 01:53:36,721 - DEBUG - Progress: 2568/2568 bytes
+2026-02-12 01:53:36,722 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,723 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/normalize_url.py')
+2026-02-12 01:53:36,724 - INFO - Upload completed successfully: normalize_url.py
+2026-02-12 01:53:36,724 - INFO - Starting upload: lib/markdown_it/common/html_re.py -> /home/kevin/test/lib/markdown_it/common/html_re.py (926 bytes)
+2026-02-12 01:53:36,726 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/html_re.py', 'wb')
+2026-02-12 01:53:36,726 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/html_re.py', 'wb') -> 00000000
+2026-02-12 01:53:36,726 - DEBUG - Progress: 926/926 bytes
+2026-02-12 01:53:36,726 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,727 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/html_re.py')
+2026-02-12 01:53:36,729 - INFO - Upload completed successfully: html_re.py
+2026-02-12 01:53:36,729 - INFO - Starting upload: lib/markdown_it/common/html_blocks.py -> /home/kevin/test/lib/markdown_it/common/html_blocks.py (986 bytes)
+2026-02-12 01:53:36,730 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/html_blocks.py', 'wb')
+2026-02-12 01:53:36,731 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/html_blocks.py', 'wb') -> 00000000
+2026-02-12 01:53:36,731 - DEBUG - Progress: 986/986 bytes
+2026-02-12 01:53:36,731 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,732 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/html_blocks.py')
+2026-02-12 01:53:36,733 - INFO - Upload completed successfully: html_blocks.py
+2026-02-12 01:53:36,733 - INFO - Starting upload: lib/markdown_it/common/entities.py -> /home/kevin/test/lib/markdown_it/common/entities.py (157 bytes)
+2026-02-12 01:53:36,735 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/entities.py', 'wb')
+2026-02-12 01:53:36,735 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/entities.py', 'wb') -> 00000000
+2026-02-12 01:53:36,735 - DEBUG - Progress: 157/157 bytes
+2026-02-12 01:53:36,735 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,736 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/entities.py')
+2026-02-12 01:53:36,737 - INFO - Upload completed successfully: entities.py
+2026-02-12 01:53:36,738 - INFO - Starting upload: lib/markdown_it/common/__init__.py -> /home/kevin/test/lib/markdown_it/common/__init__.py (0 bytes)
+2026-02-12 01:53:36,739 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__init__.py', 'wb')
+2026-02-12 01:53:36,740 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,740 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,740 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__init__.py')
+2026-02-12 01:53:36,742 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,742 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/common/__pycache__', 511)
+2026-02-12 01:53:36,742 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/common/__pycache__
+2026-02-12 01:53:36,742 - INFO - Starting upload: lib/markdown_it/common/__pycache__/utils.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc (11186 bytes)
+2026-02-12 01:53:36,744 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,744 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,745 - DEBUG - Progress: 11186/11186 bytes
+2026-02-12 01:53:36,745 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,747 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__pycache__/utils.cpython-314.pyc')
+2026-02-12 01:53:36,748 - INFO - Upload completed successfully: utils.cpython-314.pyc
+2026-02-12 01:53:36,748 - INFO - Starting upload: lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc (3844 bytes)
+2026-02-12 01:53:36,750 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,750 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,750 - DEBUG - Progress: 3844/3844 bytes
+2026-02-12 01:53:36,750 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,752 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__pycache__/normalize_url.cpython-314.pyc')
+2026-02-12 01:53:36,753 - INFO - Upload completed successfully: normalize_url.cpython-314.pyc
+2026-02-12 01:53:36,753 - INFO - Starting upload: lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc (1520 bytes)
+2026-02-12 01:53:36,755 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,755 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,755 - DEBUG - Progress: 1520/1520 bytes
+2026-02-12 01:53:36,755 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,756 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__pycache__/html_re.cpython-314.pyc')
+2026-02-12 01:53:36,758 - INFO - Upload completed successfully: html_re.cpython-314.pyc
+2026-02-12 01:53:36,758 - INFO - Starting upload: lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc (1669 bytes)
+2026-02-12 01:53:36,759 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,760 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,760 - DEBUG - Progress: 1669/1669 bytes
+2026-02-12 01:53:36,760 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,761 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__pycache__/html_blocks.cpython-314.pyc')
+2026-02-12 01:53:36,762 - INFO - Upload completed successfully: html_blocks.cpython-314.pyc
+2026-02-12 01:53:36,762 - INFO - Starting upload: lib/markdown_it/common/__pycache__/entities.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc (526 bytes)
+2026-02-12 01:53:36,764 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,764 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,764 - DEBUG - Progress: 526/526 bytes
+2026-02-12 01:53:36,764 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,765 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__pycache__/entities.cpython-314.pyc')
+2026-02-12 01:53:36,766 - INFO - Upload completed successfully: entities.cpython-314.pyc
+2026-02-12 01:53:36,767 - INFO - Starting upload: lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc (165 bytes)
+2026-02-12 01:53:36,768 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,769 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,769 - DEBUG - Progress: 165/165 bytes
+2026-02-12 01:53:36,769 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,770 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/common/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,771 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,771 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/cli', 511)
+2026-02-12 01:53:36,772 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/cli
+2026-02-12 01:53:36,772 - INFO - Starting upload: lib/markdown_it/cli/parse.py -> /home/kevin/test/lib/markdown_it/cli/parse.py (2881 bytes)
+2026-02-12 01:53:36,773 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/parse.py', 'wb')
+2026-02-12 01:53:36,774 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/parse.py', 'wb') -> 00000000
+2026-02-12 01:53:36,774 - DEBUG - Progress: 2881/2881 bytes
+2026-02-12 01:53:36,774 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,775 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/cli/parse.py')
+2026-02-12 01:53:36,777 - INFO - Upload completed successfully: parse.py
+2026-02-12 01:53:36,777 - INFO - Starting upload: lib/markdown_it/cli/__init__.py -> /home/kevin/test/lib/markdown_it/cli/__init__.py (0 bytes)
+2026-02-12 01:53:36,778 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/__init__.py', 'wb')
+2026-02-12 01:53:36,779 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,779 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,779 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/cli/__init__.py')
+2026-02-12 01:53:36,781 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,781 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/markdown_it/cli/__pycache__', 511)
+2026-02-12 01:53:36,781 - INFO - Created remote folder: /home/kevin/test/lib/markdown_it/cli/__pycache__
+2026-02-12 01:53:36,782 - INFO - Starting upload: lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc (5338 bytes)
+2026-02-12 01:53:36,783 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,784 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,784 - DEBUG - Progress: 5338/5338 bytes
+2026-02-12 01:53:36,784 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,785 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/cli/__pycache__/parse.cpython-314.pyc')
+2026-02-12 01:53:36,787 - INFO - Upload completed successfully: parse.cpython-314.pyc
+2026-02-12 01:53:36,787 - INFO - Starting upload: lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc (162 bytes)
+2026-02-12 01:53:36,788 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,789 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,789 - DEBUG - Progress: 162/162 bytes
+2026-02-12 01:53:36,789 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,790 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/markdown_it/cli/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,791 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,791 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info', 511)
+2026-02-12 01:53:36,792 - INFO - Created remote folder: /home/kevin/test/lib/mdurl-0.1.2.dist-info
+2026-02-12 01:53:36,792 - INFO - Starting upload: lib/mdurl-0.1.2.dist-info/RECORD -> /home/kevin/test/lib/mdurl-0.1.2.dist-info/RECORD (1152 bytes)
+2026-02-12 01:53:36,793 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/RECORD', 'wb')
+2026-02-12 01:53:36,794 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:36,794 - DEBUG - Progress: 1152/1152 bytes
+2026-02-12 01:53:36,794 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,795 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/RECORD')
+2026-02-12 01:53:36,796 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:36,796 - INFO - Starting upload: lib/mdurl-0.1.2.dist-info/INSTALLER -> /home/kevin/test/lib/mdurl-0.1.2.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:36,798 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:36,798 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:36,798 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:36,798 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,799 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/INSTALLER')
+2026-02-12 01:53:36,800 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:36,801 - INFO - Starting upload: lib/mdurl-0.1.2.dist-info/METADATA -> /home/kevin/test/lib/mdurl-0.1.2.dist-info/METADATA (1638 bytes)
+2026-02-12 01:53:36,802 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/METADATA', 'wb')
+2026-02-12 01:53:36,802 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:36,803 - DEBUG - Progress: 1638/1638 bytes
+2026-02-12 01:53:36,803 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,803 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/METADATA')
+2026-02-12 01:53:36,805 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:36,805 - INFO - Starting upload: lib/mdurl-0.1.2.dist-info/WHEEL -> /home/kevin/test/lib/mdurl-0.1.2.dist-info/WHEEL (81 bytes)
+2026-02-12 01:53:36,806 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:36,807 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:36,807 - DEBUG - Progress: 81/81 bytes
+2026-02-12 01:53:36,807 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,808 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/WHEEL')
+2026-02-12 01:53:36,809 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:36,809 - INFO - Starting upload: lib/mdurl-0.1.2.dist-info/LICENSE -> /home/kevin/test/lib/mdurl-0.1.2.dist-info/LICENSE (2338 bytes)
+2026-02-12 01:53:36,811 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/LICENSE', 'wb')
+2026-02-12 01:53:36,811 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:36,811 - DEBUG - Progress: 2338/2338 bytes
+2026-02-12 01:53:36,812 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,813 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl-0.1.2.dist-info/LICENSE')
+2026-02-12 01:53:36,814 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:36,814 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/mdurl', 511)
+2026-02-12 01:53:36,815 - INFO - Created remote folder: /home/kevin/test/lib/mdurl
+2026-02-12 01:53:36,815 - INFO - Starting upload: lib/mdurl/py.typed -> /home/kevin/test/lib/mdurl/py.typed (26 bytes)
+2026-02-12 01:53:36,816 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/py.typed', 'wb')
+2026-02-12 01:53:36,817 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/py.typed', 'wb') -> 00000000
+2026-02-12 01:53:36,817 - DEBUG - Progress: 26/26 bytes
+2026-02-12 01:53:36,817 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,818 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/py.typed')
+2026-02-12 01:53:36,819 - INFO - Upload completed successfully: py.typed
+2026-02-12 01:53:36,819 - INFO - Starting upload: lib/mdurl/_url.py -> /home/kevin/test/lib/mdurl/_url.py (284 bytes)
+2026-02-12 01:53:36,821 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_url.py', 'wb')
+2026-02-12 01:53:36,821 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_url.py', 'wb') -> 00000000
+2026-02-12 01:53:36,821 - DEBUG - Progress: 284/284 bytes
+2026-02-12 01:53:36,821 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,822 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/_url.py')
+2026-02-12 01:53:36,824 - INFO - Upload completed successfully: _url.py
+2026-02-12 01:53:36,824 - INFO - Starting upload: lib/mdurl/_parse.py -> /home/kevin/test/lib/mdurl/_parse.py (11374 bytes)
+2026-02-12 01:53:36,825 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_parse.py', 'wb')
+2026-02-12 01:53:36,826 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_parse.py', 'wb') -> 00000000
+2026-02-12 01:53:36,826 - DEBUG - Progress: 11374/11374 bytes
+2026-02-12 01:53:36,826 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,827 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/_parse.py')
+2026-02-12 01:53:36,829 - INFO - Upload completed successfully: _parse.py
+2026-02-12 01:53:36,829 - INFO - Starting upload: lib/mdurl/_format.py -> /home/kevin/test/lib/mdurl/_format.py (626 bytes)
+2026-02-12 01:53:36,830 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_format.py', 'wb')
+2026-02-12 01:53:36,831 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_format.py', 'wb') -> 00000000
+2026-02-12 01:53:36,831 - DEBUG - Progress: 626/626 bytes
+2026-02-12 01:53:36,831 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,832 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/_format.py')
+2026-02-12 01:53:36,833 - INFO - Upload completed successfully: _format.py
+2026-02-12 01:53:36,833 - INFO - Starting upload: lib/mdurl/_encode.py -> /home/kevin/test/lib/mdurl/_encode.py (2602 bytes)
+2026-02-12 01:53:36,835 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_encode.py', 'wb')
+2026-02-12 01:53:36,836 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_encode.py', 'wb') -> 00000000
+2026-02-12 01:53:36,836 - DEBUG - Progress: 2602/2602 bytes
+2026-02-12 01:53:36,836 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,837 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/_encode.py')
+2026-02-12 01:53:36,839 - INFO - Upload completed successfully: _encode.py
+2026-02-12 01:53:36,839 - INFO - Starting upload: lib/mdurl/_decode.py -> /home/kevin/test/lib/mdurl/_decode.py (3004 bytes)
+2026-02-12 01:53:36,841 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_decode.py', 'wb')
+2026-02-12 01:53:36,841 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/_decode.py', 'wb') -> 00000000
+2026-02-12 01:53:36,841 - DEBUG - Progress: 3004/3004 bytes
+2026-02-12 01:53:36,841 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,843 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/_decode.py')
+2026-02-12 01:53:36,844 - INFO - Upload completed successfully: _decode.py
+2026-02-12 01:53:36,844 - INFO - Starting upload: lib/mdurl/__init__.py -> /home/kevin/test/lib/mdurl/__init__.py (547 bytes)
+2026-02-12 01:53:36,846 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__init__.py', 'wb')
+2026-02-12 01:53:36,846 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:36,846 - DEBUG - Progress: 547/547 bytes
+2026-02-12 01:53:36,847 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,848 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__init__.py')
+2026-02-12 01:53:36,849 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:36,850 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/mdurl/__pycache__', 511)
+2026-02-12 01:53:36,850 - INFO - Created remote folder: /home/kevin/test/lib/mdurl/__pycache__
+2026-02-12 01:53:36,850 - INFO - Starting upload: lib/mdurl/__pycache__/_url.cpython-314.pyc -> /home/kevin/test/lib/mdurl/__pycache__/_url.cpython-314.pyc (692 bytes)
+2026-02-12 01:53:36,852 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_url.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,852 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_url.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,852 - DEBUG - Progress: 692/692 bytes
+2026-02-12 01:53:36,852 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,853 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__pycache__/_url.cpython-314.pyc')
+2026-02-12 01:53:36,854 - INFO - Upload completed successfully: _url.cpython-314.pyc
+2026-02-12 01:53:36,854 - INFO - Starting upload: lib/mdurl/__pycache__/_parse.cpython-314.pyc -> /home/kevin/test/lib/mdurl/__pycache__/_parse.cpython-314.pyc (8579 bytes)
+2026-02-12 01:53:36,856 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_parse.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,857 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_parse.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,857 - DEBUG - Progress: 8579/8579 bytes
+2026-02-12 01:53:36,857 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,858 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__pycache__/_parse.cpython-314.pyc')
+2026-02-12 01:53:36,860 - INFO - Upload completed successfully: _parse.cpython-314.pyc
+2026-02-12 01:53:36,860 - INFO - Starting upload: lib/mdurl/__pycache__/_format.cpython-314.pyc -> /home/kevin/test/lib/mdurl/__pycache__/_format.cpython-314.pyc (1550 bytes)
+2026-02-12 01:53:36,862 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_format.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,862 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_format.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,863 - DEBUG - Progress: 1550/1550 bytes
+2026-02-12 01:53:36,863 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,863 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__pycache__/_format.cpython-314.pyc')
+2026-02-12 01:53:36,865 - INFO - Upload completed successfully: _format.cpython-314.pyc
+2026-02-12 01:53:36,865 - INFO - Starting upload: lib/mdurl/__pycache__/_encode.cpython-314.pyc -> /home/kevin/test/lib/mdurl/__pycache__/_encode.cpython-314.pyc (3563 bytes)
+2026-02-12 01:53:36,867 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_encode.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,867 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_encode.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,868 - DEBUG - Progress: 3563/3563 bytes
+2026-02-12 01:53:36,868 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,869 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__pycache__/_encode.cpython-314.pyc')
+2026-02-12 01:53:36,870 - INFO - Upload completed successfully: _encode.cpython-314.pyc
+2026-02-12 01:53:36,870 - INFO - Starting upload: lib/mdurl/__pycache__/_decode.cpython-314.pyc -> /home/kevin/test/lib/mdurl/__pycache__/_decode.cpython-314.pyc (4596 bytes)
+2026-02-12 01:53:36,872 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_decode.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,872 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/_decode.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,872 - DEBUG - Progress: 4596/4596 bytes
+2026-02-12 01:53:36,872 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,874 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__pycache__/_decode.cpython-314.pyc')
+2026-02-12 01:53:36,875 - INFO - Upload completed successfully: _decode.cpython-314.pyc
+2026-02-12 01:53:36,876 - INFO - Starting upload: lib/mdurl/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/mdurl/__pycache__/__init__.cpython-314.pyc (614 bytes)
+2026-02-12 01:53:36,877 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:36,878 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/mdurl/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:36,878 - DEBUG - Progress: 614/614 bytes
+2026-02-12 01:53:36,878 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,879 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/mdurl/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:36,880 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:36,880 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments-2.19.2.dist-info', 511)
+2026-02-12 01:53:36,881 - INFO - Created remote folder: /home/kevin/test/lib/pygments-2.19.2.dist-info
+2026-02-12 01:53:36,881 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/RECORD -> /home/kevin/test/lib/pygments-2.19.2.dist-info/RECORD (47425 bytes)
+2026-02-12 01:53:36,882 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/RECORD', 'wb')
+2026-02-12 01:53:36,883 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/RECORD', 'wb') -> 00000000
+2026-02-12 01:53:36,883 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,892 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/RECORD')
+2026-02-12 01:53:36,894 - INFO - Upload completed successfully: RECORD
+2026-02-12 01:53:36,894 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/INSTALLER -> /home/kevin/test/lib/pygments-2.19.2.dist-info/INSTALLER (4 bytes)
+2026-02-12 01:53:36,895 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/INSTALLER', 'wb')
+2026-02-12 01:53:36,896 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/INSTALLER', 'wb') -> 00000000
+2026-02-12 01:53:36,896 - DEBUG - Progress: 4/4 bytes
+2026-02-12 01:53:36,896 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,897 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/INSTALLER')
+2026-02-12 01:53:36,898 - INFO - Upload completed successfully: INSTALLER
+2026-02-12 01:53:36,898 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/entry_points.txt -> /home/kevin/test/lib/pygments-2.19.2.dist-info/entry_points.txt (53 bytes)
+2026-02-12 01:53:36,900 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/entry_points.txt', 'wb')
+2026-02-12 01:53:36,900 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/entry_points.txt', 'wb') -> 00000000
+2026-02-12 01:53:36,900 - DEBUG - Progress: 53/53 bytes
+2026-02-12 01:53:36,900 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,901 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/entry_points.txt')
+2026-02-12 01:53:36,905 - INFO - Upload completed successfully: entry_points.txt
+2026-02-12 01:53:36,905 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/WHEEL -> /home/kevin/test/lib/pygments-2.19.2.dist-info/WHEEL (87 bytes)
+2026-02-12 01:53:36,909 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/WHEEL', 'wb')
+2026-02-12 01:53:36,910 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/WHEEL', 'wb') -> 00000000
+2026-02-12 01:53:36,910 - DEBUG - Progress: 87/87 bytes
+2026-02-12 01:53:36,910 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,911 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/WHEEL')
+2026-02-12 01:53:36,914 - INFO - Upload completed successfully: WHEEL
+2026-02-12 01:53:36,915 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/METADATA -> /home/kevin/test/lib/pygments-2.19.2.dist-info/METADATA (2512 bytes)
+2026-02-12 01:53:36,917 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/METADATA', 'wb')
+2026-02-12 01:53:36,917 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/METADATA', 'wb') -> 00000000
+2026-02-12 01:53:36,917 - DEBUG - Progress: 2512/2512 bytes
+2026-02-12 01:53:36,918 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,918 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/METADATA')
+2026-02-12 01:53:36,920 - INFO - Upload completed successfully: METADATA
+2026-02-12 01:53:36,920 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses', 511)
+2026-02-12 01:53:36,921 - INFO - Created remote folder: /home/kevin/test/lib/pygments-2.19.2.dist-info/licenses
+2026-02-12 01:53:36,921 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/licenses/LICENSE -> /home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/LICENSE (1331 bytes)
+2026-02-12 01:53:36,922 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/LICENSE', 'wb')
+2026-02-12 01:53:36,923 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/LICENSE', 'wb') -> 00000000
+2026-02-12 01:53:36,923 - DEBUG - Progress: 1331/1331 bytes
+2026-02-12 01:53:36,923 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,924 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/LICENSE')
+2026-02-12 01:53:36,925 - INFO - Upload completed successfully: LICENSE
+2026-02-12 01:53:36,926 - INFO - Starting upload: lib/pygments-2.19.2.dist-info/licenses/AUTHORS -> /home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/AUTHORS (10824 bytes)
+2026-02-12 01:53:36,927 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/AUTHORS', 'wb')
+2026-02-12 01:53:36,928 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/AUTHORS', 'wb') -> 00000000
+2026-02-12 01:53:36,928 - DEBUG - Progress: 10824/10824 bytes
+2026-02-12 01:53:36,928 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,930 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments-2.19.2.dist-info/licenses/AUTHORS')
+2026-02-12 01:53:36,932 - INFO - Upload completed successfully: AUTHORS
+2026-02-12 01:53:36,932 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments', 511)
+2026-02-12 01:53:36,932 - INFO - Created remote folder: /home/kevin/test/lib/pygments
+2026-02-12 01:53:36,932 - INFO - Starting upload: lib/pygments/util.py -> /home/kevin/test/lib/pygments/util.py (10031 bytes)
+2026-02-12 01:53:36,934 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/util.py', 'wb')
+2026-02-12 01:53:36,934 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/util.py', 'wb') -> 00000000
+2026-02-12 01:53:36,935 - DEBUG - Progress: 10031/10031 bytes
+2026-02-12 01:53:36,935 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,936 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/util.py')
+2026-02-12 01:53:36,938 - INFO - Upload completed successfully: util.py
+2026-02-12 01:53:36,938 - INFO - Starting upload: lib/pygments/unistring.py -> /home/kevin/test/lib/pygments/unistring.py (63208 bytes)
+2026-02-12 01:53:36,939 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/unistring.py', 'wb')
+2026-02-12 01:53:36,940 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/unistring.py', 'wb') -> 00000000
+2026-02-12 01:53:36,940 - DEBUG - Progress: 32768/63208 bytes
+2026-02-12 01:53:36,940 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,949 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/unistring.py')
+2026-02-12 01:53:36,951 - INFO - Upload completed successfully: unistring.py
+2026-02-12 01:53:36,951 - INFO - Starting upload: lib/pygments/token.py -> /home/kevin/test/lib/pygments/token.py (6226 bytes)
+2026-02-12 01:53:36,953 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/token.py', 'wb')
+2026-02-12 01:53:36,953 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/token.py', 'wb') -> 00000000
+2026-02-12 01:53:36,954 - DEBUG - Progress: 6226/6226 bytes
+2026-02-12 01:53:36,954 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,955 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/token.py')
+2026-02-12 01:53:36,957 - INFO - Upload completed successfully: token.py
+2026-02-12 01:53:36,957 - INFO - Starting upload: lib/pygments/style.py -> /home/kevin/test/lib/pygments/style.py (6408 bytes)
+2026-02-12 01:53:36,958 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/style.py', 'wb')
+2026-02-12 01:53:36,959 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/style.py', 'wb') -> 00000000
+2026-02-12 01:53:36,959 - DEBUG - Progress: 6408/6408 bytes
+2026-02-12 01:53:36,959 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,960 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/style.py')
+2026-02-12 01:53:36,962 - INFO - Upload completed successfully: style.py
+2026-02-12 01:53:36,962 - INFO - Starting upload: lib/pygments/sphinxext.py -> /home/kevin/test/lib/pygments/sphinxext.py (7898 bytes)
+2026-02-12 01:53:36,963 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/sphinxext.py', 'wb')
+2026-02-12 01:53:36,964 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/sphinxext.py', 'wb') -> 00000000
+2026-02-12 01:53:36,964 - DEBUG - Progress: 7898/7898 bytes
+2026-02-12 01:53:36,964 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,965 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/sphinxext.py')
+2026-02-12 01:53:36,967 - INFO - Upload completed successfully: sphinxext.py
+2026-02-12 01:53:36,967 - INFO - Starting upload: lib/pygments/scanner.py -> /home/kevin/test/lib/pygments/scanner.py (3092 bytes)
+2026-02-12 01:53:36,968 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/scanner.py', 'wb')
+2026-02-12 01:53:36,969 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/scanner.py', 'wb') -> 00000000
+2026-02-12 01:53:36,969 - DEBUG - Progress: 3092/3092 bytes
+2026-02-12 01:53:36,969 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,970 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/scanner.py')
+2026-02-12 01:53:36,972 - INFO - Upload completed successfully: scanner.py
+2026-02-12 01:53:36,972 - INFO - Starting upload: lib/pygments/regexopt.py -> /home/kevin/test/lib/pygments/regexopt.py (3072 bytes)
+2026-02-12 01:53:36,973 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/regexopt.py', 'wb')
+2026-02-12 01:53:36,974 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/regexopt.py', 'wb') -> 00000000
+2026-02-12 01:53:36,974 - DEBUG - Progress: 3072/3072 bytes
+2026-02-12 01:53:36,974 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,975 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/regexopt.py')
+2026-02-12 01:53:36,977 - INFO - Upload completed successfully: regexopt.py
+2026-02-12 01:53:36,977 - INFO - Starting upload: lib/pygments/plugin.py -> /home/kevin/test/lib/pygments/plugin.py (1891 bytes)
+2026-02-12 01:53:36,979 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/plugin.py', 'wb')
+2026-02-12 01:53:36,979 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/plugin.py', 'wb') -> 00000000
+2026-02-12 01:53:36,979 - DEBUG - Progress: 1891/1891 bytes
+2026-02-12 01:53:36,979 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,980 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/plugin.py')
+2026-02-12 01:53:36,983 - INFO - Upload completed successfully: plugin.py
+2026-02-12 01:53:36,983 - INFO - Starting upload: lib/pygments/modeline.py -> /home/kevin/test/lib/pygments/modeline.py (1005 bytes)
+2026-02-12 01:53:36,985 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/modeline.py', 'wb')
+2026-02-12 01:53:36,985 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/modeline.py', 'wb') -> 00000000
+2026-02-12 01:53:36,985 - DEBUG - Progress: 1005/1005 bytes
+2026-02-12 01:53:36,985 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,986 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/modeline.py')
+2026-02-12 01:53:36,988 - INFO - Upload completed successfully: modeline.py
+2026-02-12 01:53:36,988 - INFO - Starting upload: lib/pygments/lexer.py -> /home/kevin/test/lib/pygments/lexer.py (35109 bytes)
+2026-02-12 01:53:36,989 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexer.py', 'wb')
+2026-02-12 01:53:36,990 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexer.py', 'wb') -> 00000000
+2026-02-12 01:53:36,990 - DEBUG - Progress: 32768/35109 bytes
+2026-02-12 01:53:36,990 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,994 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexer.py')
+2026-02-12 01:53:36,995 - INFO - Upload completed successfully: lexer.py
+2026-02-12 01:53:36,996 - INFO - Starting upload: lib/pygments/formatter.py -> /home/kevin/test/lib/pygments/formatter.py (4366 bytes)
+2026-02-12 01:53:36,997 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/formatter.py', 'wb')
+2026-02-12 01:53:36,998 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/formatter.py', 'wb') -> 00000000
+2026-02-12 01:53:36,998 - DEBUG - Progress: 4366/4366 bytes
+2026-02-12 01:53:36,998 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:36,999 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/formatter.py')
+2026-02-12 01:53:37,000 - INFO - Upload completed successfully: formatter.py
+2026-02-12 01:53:37,000 - INFO - Starting upload: lib/pygments/filter.py -> /home/kevin/test/lib/pygments/filter.py (1910 bytes)
+2026-02-12 01:53:37,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/filter.py', 'wb')
+2026-02-12 01:53:37,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/filter.py', 'wb') -> 00000000
+2026-02-12 01:53:37,002 - DEBUG - Progress: 1910/1910 bytes
+2026-02-12 01:53:37,003 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,003 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/filter.py')
+2026-02-12 01:53:37,005 - INFO - Upload completed successfully: filter.py
+2026-02-12 01:53:37,005 - INFO - Starting upload: lib/pygments/console.py -> /home/kevin/test/lib/pygments/console.py (1718 bytes)
+2026-02-12 01:53:37,006 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/console.py', 'wb')
+2026-02-12 01:53:37,007 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/console.py', 'wb') -> 00000000
+2026-02-12 01:53:37,007 - DEBUG - Progress: 1718/1718 bytes
+2026-02-12 01:53:37,007 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,008 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/console.py')
+2026-02-12 01:53:37,010 - INFO - Upload completed successfully: console.py
+2026-02-12 01:53:37,010 - INFO - Starting upload: lib/pygments/cmdline.py -> /home/kevin/test/lib/pygments/cmdline.py (23536 bytes)
+2026-02-12 01:53:37,012 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/cmdline.py', 'wb')
+2026-02-12 01:53:37,012 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/cmdline.py', 'wb') -> 00000000
+2026-02-12 01:53:37,012 - DEBUG - Progress: 23536/23536 bytes
+2026-02-12 01:53:37,012 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,015 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/cmdline.py')
+2026-02-12 01:53:37,017 - INFO - Upload completed successfully: cmdline.py
+2026-02-12 01:53:37,017 - INFO - Starting upload: lib/pygments/__main__.py -> /home/kevin/test/lib/pygments/__main__.py (348 bytes)
+2026-02-12 01:53:37,018 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__main__.py', 'wb')
+2026-02-12 01:53:37,019 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__main__.py', 'wb') -> 00000000
+2026-02-12 01:53:37,019 - DEBUG - Progress: 348/348 bytes
+2026-02-12 01:53:37,019 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,020 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__main__.py')
+2026-02-12 01:53:37,021 - INFO - Upload completed successfully: __main__.py
+2026-02-12 01:53:37,021 - INFO - Starting upload: lib/pygments/__init__.py -> /home/kevin/test/lib/pygments/__init__.py (2959 bytes)
+2026-02-12 01:53:37,023 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__init__.py', 'wb')
+2026-02-12 01:53:37,023 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:37,023 - DEBUG - Progress: 2959/2959 bytes
+2026-02-12 01:53:37,024 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,025 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__init__.py')
+2026-02-12 01:53:37,027 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:37,027 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments/__pycache__', 511)
+2026-02-12 01:53:37,028 - INFO - Created remote folder: /home/kevin/test/lib/pygments/__pycache__
+2026-02-12 01:53:37,028 - INFO - Starting upload: lib/pygments/__pycache__/util.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/util.cpython-314.pyc (14496 bytes)
+2026-02-12 01:53:37,029 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/util.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,030 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/util.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,030 - DEBUG - Progress: 14496/14496 bytes
+2026-02-12 01:53:37,030 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,032 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/util.cpython-314.pyc')
+2026-02-12 01:53:37,034 - INFO - Upload completed successfully: util.cpython-314.pyc
+2026-02-12 01:53:37,034 - INFO - Starting upload: lib/pygments/__pycache__/unistring.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/unistring.cpython-314.pyc (33184 bytes)
+2026-02-12 01:53:37,035 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/unistring.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,036 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/unistring.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,036 - DEBUG - Progress: 32768/33184 bytes
+2026-02-12 01:53:37,036 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,040 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/unistring.cpython-314.pyc')
+2026-02-12 01:53:37,042 - INFO - Upload completed successfully: unistring.cpython-314.pyc
+2026-02-12 01:53:37,042 - INFO - Starting upload: lib/pygments/__pycache__/token.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/token.cpython-314.pyc (8303 bytes)
+2026-02-12 01:53:37,043 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/token.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,044 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/token.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,044 - DEBUG - Progress: 8303/8303 bytes
+2026-02-12 01:53:37,044 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,045 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/token.cpython-314.pyc')
+2026-02-12 01:53:37,047 - INFO - Upload completed successfully: token.cpython-314.pyc
+2026-02-12 01:53:37,047 - INFO - Starting upload: lib/pygments/__pycache__/style.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/style.cpython-314.pyc (7241 bytes)
+2026-02-12 01:53:37,049 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/style.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,049 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/style.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,050 - DEBUG - Progress: 7241/7241 bytes
+2026-02-12 01:53:37,050 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,051 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/style.cpython-314.pyc')
+2026-02-12 01:53:37,053 - INFO - Upload completed successfully: style.cpython-314.pyc
+2026-02-12 01:53:37,053 - INFO - Starting upload: lib/pygments/__pycache__/sphinxext.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/sphinxext.cpython-314.pyc (12665 bytes)
+2026-02-12 01:53:37,055 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/sphinxext.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,055 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/sphinxext.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,055 - DEBUG - Progress: 12665/12665 bytes
+2026-02-12 01:53:37,055 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,058 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/sphinxext.cpython-314.pyc')
+2026-02-12 01:53:37,059 - INFO - Upload completed successfully: sphinxext.cpython-314.pyc
+2026-02-12 01:53:37,060 - INFO - Starting upload: lib/pygments/__pycache__/scanner.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/scanner.cpython-314.pyc (4726 bytes)
+2026-02-12 01:53:37,061 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/scanner.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,062 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/scanner.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,062 - DEBUG - Progress: 4726/4726 bytes
+2026-02-12 01:53:37,062 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,063 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/scanner.cpython-314.pyc')
+2026-02-12 01:53:37,065 - INFO - Upload completed successfully: scanner.cpython-314.pyc
+2026-02-12 01:53:37,065 - INFO - Starting upload: lib/pygments/__pycache__/regexopt.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/regexopt.cpython-314.pyc (4332 bytes)
+2026-02-12 01:53:37,066 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/regexopt.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,067 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/regexopt.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,067 - DEBUG - Progress: 4332/4332 bytes
+2026-02-12 01:53:37,067 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,068 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/regexopt.cpython-314.pyc')
+2026-02-12 01:53:37,071 - INFO - Upload completed successfully: regexopt.cpython-314.pyc
+2026-02-12 01:53:37,071 - INFO - Starting upload: lib/pygments/__pycache__/plugin.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/plugin.cpython-314.pyc (2521 bytes)
+2026-02-12 01:53:37,072 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/plugin.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,073 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/plugin.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,073 - DEBUG - Progress: 2521/2521 bytes
+2026-02-12 01:53:37,073 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,074 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/plugin.cpython-314.pyc')
+2026-02-12 01:53:37,076 - INFO - Upload completed successfully: plugin.cpython-314.pyc
+2026-02-12 01:53:37,076 - INFO - Starting upload: lib/pygments/__pycache__/modeline.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/modeline.cpython-314.pyc (1566 bytes)
+2026-02-12 01:53:37,078 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/modeline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,078 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/modeline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,079 - DEBUG - Progress: 1566/1566 bytes
+2026-02-12 01:53:37,079 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,080 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/modeline.cpython-314.pyc')
+2026-02-12 01:53:37,081 - INFO - Upload completed successfully: modeline.cpython-314.pyc
+2026-02-12 01:53:37,081 - INFO - Starting upload: lib/pygments/__pycache__/lexer.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/lexer.cpython-314.pyc (39960 bytes)
+2026-02-12 01:53:37,083 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/lexer.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,083 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/lexer.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,084 - DEBUG - Progress: 32768/39960 bytes
+2026-02-12 01:53:37,084 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,088 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/lexer.cpython-314.pyc')
+2026-02-12 01:53:37,090 - INFO - Upload completed successfully: lexer.cpython-314.pyc
+2026-02-12 01:53:37,090 - INFO - Starting upload: lib/pygments/__pycache__/formatter.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/formatter.cpython-314.pyc (4564 bytes)
+2026-02-12 01:53:37,092 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/formatter.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,092 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/formatter.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,092 - DEBUG - Progress: 4564/4564 bytes
+2026-02-12 01:53:37,092 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,094 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/formatter.cpython-314.pyc')
+2026-02-12 01:53:37,096 - INFO - Upload completed successfully: formatter.cpython-314.pyc
+2026-02-12 01:53:37,096 - INFO - Starting upload: lib/pygments/__pycache__/filter.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/filter.cpython-314.pyc (3271 bytes)
+2026-02-12 01:53:37,097 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/filter.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,098 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/filter.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,098 - DEBUG - Progress: 3271/3271 bytes
+2026-02-12 01:53:37,098 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,099 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/filter.cpython-314.pyc')
+2026-02-12 01:53:37,101 - INFO - Upload completed successfully: filter.cpython-314.pyc
+2026-02-12 01:53:37,101 - INFO - Starting upload: lib/pygments/__pycache__/console.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/console.cpython-314.pyc (2773 bytes)
+2026-02-12 01:53:37,103 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/console.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,103 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/console.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,104 - DEBUG - Progress: 2773/2773 bytes
+2026-02-12 01:53:37,104 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,105 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/console.cpython-314.pyc')
+2026-02-12 01:53:37,107 - INFO - Upload completed successfully: console.cpython-314.pyc
+2026-02-12 01:53:37,107 - INFO - Starting upload: lib/pygments/__pycache__/cmdline.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/cmdline.cpython-314.pyc (28156 bytes)
+2026-02-12 01:53:37,108 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/cmdline.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,109 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/cmdline.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,109 - DEBUG - Progress: 28156/28156 bytes
+2026-02-12 01:53:37,109 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,113 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/cmdline.cpython-314.pyc')
+2026-02-12 01:53:37,115 - INFO - Upload completed successfully: cmdline.cpython-314.pyc
+2026-02-12 01:53:37,115 - INFO - Starting upload: lib/pygments/__pycache__/__main__.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/__main__.cpython-314.pyc (719 bytes)
+2026-02-12 01:53:37,116 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/__main__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,117 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/__main__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,117 - DEBUG - Progress: 719/719 bytes
+2026-02-12 01:53:37,117 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,118 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/__main__.cpython-314.pyc')
+2026-02-12 01:53:37,119 - INFO - Upload completed successfully: __main__.cpython-314.pyc
+2026-02-12 01:53:37,119 - INFO - Starting upload: lib/pygments/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/pygments/__pycache__/__init__.cpython-314.pyc (3397 bytes)
+2026-02-12 01:53:37,121 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,121 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,122 - DEBUG - Progress: 3397/3397 bytes
+2026-02-12 01:53:37,122 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,123 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:37,124 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:37,125 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments/styles', 511)
+2026-02-12 01:53:37,125 - INFO - Created remote folder: /home/kevin/test/lib/pygments/styles
+2026-02-12 01:53:37,125 - INFO - Starting upload: lib/pygments/styles/zenburn.py -> /home/kevin/test/lib/pygments/styles/zenburn.py (2203 bytes)
+2026-02-12 01:53:37,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/zenburn.py', 'wb')
+2026-02-12 01:53:37,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/zenburn.py', 'wb') -> 00000000
+2026-02-12 01:53:37,128 - DEBUG - Progress: 2203/2203 bytes
+2026-02-12 01:53:37,128 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,129 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/zenburn.py')
+2026-02-12 01:53:37,131 - INFO - Upload completed successfully: zenburn.py
+2026-02-12 01:53:37,131 - INFO - Starting upload: lib/pygments/styles/xcode.py -> /home/kevin/test/lib/pygments/styles/xcode.py (1504 bytes)
+2026-02-12 01:53:37,132 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/xcode.py', 'wb')
+2026-02-12 01:53:37,133 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/xcode.py', 'wb') -> 00000000
+2026-02-12 01:53:37,133 - DEBUG - Progress: 1504/1504 bytes
+2026-02-12 01:53:37,133 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,134 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/xcode.py')
+2026-02-12 01:53:37,136 - INFO - Upload completed successfully: xcode.py
+2026-02-12 01:53:37,136 - INFO - Starting upload: lib/pygments/styles/vs.py -> /home/kevin/test/lib/pygments/styles/vs.py (1130 bytes)
+2026-02-12 01:53:37,137 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/vs.py', 'wb')
+2026-02-12 01:53:37,138 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/vs.py', 'wb') -> 00000000
+2026-02-12 01:53:37,138 - DEBUG - Progress: 1130/1130 bytes
+2026-02-12 01:53:37,138 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,139 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/vs.py')
+2026-02-12 01:53:37,142 - INFO - Upload completed successfully: vs.py
+2026-02-12 01:53:37,142 - INFO - Starting upload: lib/pygments/styles/vim.py -> /home/kevin/test/lib/pygments/styles/vim.py (2019 bytes)
+2026-02-12 01:53:37,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/vim.py', 'wb')
+2026-02-12 01:53:37,145 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/vim.py', 'wb') -> 00000000
+2026-02-12 01:53:37,145 - DEBUG - Progress: 2019/2019 bytes
+2026-02-12 01:53:37,145 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,146 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/vim.py')
+2026-02-12 01:53:37,148 - INFO - Upload completed successfully: vim.py
+2026-02-12 01:53:37,148 - INFO - Starting upload: lib/pygments/styles/trac.py -> /home/kevin/test/lib/pygments/styles/trac.py (1981 bytes)
+2026-02-12 01:53:37,149 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/trac.py', 'wb')
+2026-02-12 01:53:37,150 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/trac.py', 'wb') -> 00000000
+2026-02-12 01:53:37,150 - DEBUG - Progress: 1981/1981 bytes
+2026-02-12 01:53:37,150 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,151 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/trac.py')
+2026-02-12 01:53:37,153 - INFO - Upload completed successfully: trac.py
+2026-02-12 01:53:37,153 - INFO - Starting upload: lib/pygments/styles/tango.py -> /home/kevin/test/lib/pygments/styles/tango.py (7137 bytes)
+2026-02-12 01:53:37,155 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/tango.py', 'wb')
+2026-02-12 01:53:37,155 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/tango.py', 'wb') -> 00000000
+2026-02-12 01:53:37,155 - DEBUG - Progress: 7137/7137 bytes
+2026-02-12 01:53:37,155 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,157 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/tango.py')
+2026-02-12 01:53:37,158 - INFO - Upload completed successfully: tango.py
+2026-02-12 01:53:37,159 - INFO - Starting upload: lib/pygments/styles/stata_light.py -> /home/kevin/test/lib/pygments/styles/stata_light.py (1289 bytes)
+2026-02-12 01:53:37,160 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/stata_light.py', 'wb')
+2026-02-12 01:53:37,160 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/stata_light.py', 'wb') -> 00000000
+2026-02-12 01:53:37,161 - DEBUG - Progress: 1289/1289 bytes
+2026-02-12 01:53:37,161 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,161 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/stata_light.py')
+2026-02-12 01:53:37,163 - INFO - Upload completed successfully: stata_light.py
+2026-02-12 01:53:37,163 - INFO - Starting upload: lib/pygments/styles/stata_dark.py -> /home/kevin/test/lib/pygments/styles/stata_dark.py (1257 bytes)
+2026-02-12 01:53:37,164 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/stata_dark.py', 'wb')
+2026-02-12 01:53:37,165 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/stata_dark.py', 'wb') -> 00000000
+2026-02-12 01:53:37,165 - DEBUG - Progress: 1257/1257 bytes
+2026-02-12 01:53:37,165 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,166 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/stata_dark.py')
+2026-02-12 01:53:37,167 - INFO - Upload completed successfully: stata_dark.py
+2026-02-12 01:53:37,167 - INFO - Starting upload: lib/pygments/styles/staroffice.py -> /home/kevin/test/lib/pygments/styles/staroffice.py (831 bytes)
+2026-02-12 01:53:37,169 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/staroffice.py', 'wb')
+2026-02-12 01:53:37,170 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/staroffice.py', 'wb') -> 00000000
+2026-02-12 01:53:37,170 - DEBUG - Progress: 831/831 bytes
+2026-02-12 01:53:37,170 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,171 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/staroffice.py')
+2026-02-12 01:53:37,172 - INFO - Upload completed successfully: staroffice.py
+2026-02-12 01:53:37,172 - INFO - Starting upload: lib/pygments/styles/solarized.py -> /home/kevin/test/lib/pygments/styles/solarized.py (4247 bytes)
+2026-02-12 01:53:37,174 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/solarized.py', 'wb')
+2026-02-12 01:53:37,174 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/solarized.py', 'wb') -> 00000000
+2026-02-12 01:53:37,174 - DEBUG - Progress: 4247/4247 bytes
+2026-02-12 01:53:37,175 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,176 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/solarized.py')
+2026-02-12 01:53:37,178 - INFO - Upload completed successfully: solarized.py
+2026-02-12 01:53:37,178 - INFO - Starting upload: lib/pygments/styles/sas.py -> /home/kevin/test/lib/pygments/styles/sas.py (1440 bytes)
+2026-02-12 01:53:37,179 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/sas.py', 'wb')
+2026-02-12 01:53:37,180 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/sas.py', 'wb') -> 00000000
+2026-02-12 01:53:37,180 - DEBUG - Progress: 1440/1440 bytes
+2026-02-12 01:53:37,180 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,181 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/sas.py')
+2026-02-12 01:53:37,182 - INFO - Upload completed successfully: sas.py
+2026-02-12 01:53:37,182 - INFO - Starting upload: lib/pygments/styles/rrt.py -> /home/kevin/test/lib/pygments/styles/rrt.py (1006 bytes)
+2026-02-12 01:53:37,184 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/rrt.py', 'wb')
+2026-02-12 01:53:37,184 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/rrt.py', 'wb') -> 00000000
+2026-02-12 01:53:37,184 - DEBUG - Progress: 1006/1006 bytes
+2026-02-12 01:53:37,184 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,185 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/rrt.py')
+2026-02-12 01:53:37,187 - INFO - Upload completed successfully: rrt.py
+2026-02-12 01:53:37,187 - INFO - Starting upload: lib/pygments/styles/rainbow_dash.py -> /home/kevin/test/lib/pygments/styles/rainbow_dash.py (2390 bytes)
+2026-02-12 01:53:37,189 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/rainbow_dash.py', 'wb')
+2026-02-12 01:53:37,190 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/rainbow_dash.py', 'wb') -> 00000000
+2026-02-12 01:53:37,190 - DEBUG - Progress: 2390/2390 bytes
+2026-02-12 01:53:37,190 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,191 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/rainbow_dash.py')
+2026-02-12 01:53:37,193 - INFO - Upload completed successfully: rainbow_dash.py
+2026-02-12 01:53:37,193 - INFO - Starting upload: lib/pygments/styles/perldoc.py -> /home/kevin/test/lib/pygments/styles/perldoc.py (2230 bytes)
+2026-02-12 01:53:37,194 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/perldoc.py', 'wb')
+2026-02-12 01:53:37,195 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/perldoc.py', 'wb') -> 00000000
+2026-02-12 01:53:37,195 - DEBUG - Progress: 2230/2230 bytes
+2026-02-12 01:53:37,195 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,197 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/perldoc.py')
+2026-02-12 01:53:37,198 - INFO - Upload completed successfully: perldoc.py
+2026-02-12 01:53:37,198 - INFO - Starting upload: lib/pygments/styles/pastie.py -> /home/kevin/test/lib/pygments/styles/pastie.py (2525 bytes)
+2026-02-12 01:53:37,200 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/pastie.py', 'wb')
+2026-02-12 01:53:37,201 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/pastie.py', 'wb') -> 00000000
+2026-02-12 01:53:37,201 - DEBUG - Progress: 2525/2525 bytes
+2026-02-12 01:53:37,201 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,203 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/pastie.py')
+2026-02-12 01:53:37,204 - INFO - Upload completed successfully: pastie.py
+2026-02-12 01:53:37,204 - INFO - Starting upload: lib/pygments/styles/paraiso_light.py -> /home/kevin/test/lib/pygments/styles/paraiso_light.py (5668 bytes)
+2026-02-12 01:53:37,206 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/paraiso_light.py', 'wb')
+2026-02-12 01:53:37,207 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/paraiso_light.py', 'wb') -> 00000000
+2026-02-12 01:53:37,207 - DEBUG - Progress: 5668/5668 bytes
+2026-02-12 01:53:37,207 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,208 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/paraiso_light.py')
+2026-02-12 01:53:37,210 - INFO - Upload completed successfully: paraiso_light.py
+2026-02-12 01:53:37,210 - INFO - Starting upload: lib/pygments/styles/paraiso_dark.py -> /home/kevin/test/lib/pygments/styles/paraiso_dark.py (5662 bytes)
+2026-02-12 01:53:37,211 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/paraiso_dark.py', 'wb')
+2026-02-12 01:53:37,212 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/paraiso_dark.py', 'wb') -> 00000000
+2026-02-12 01:53:37,212 - DEBUG - Progress: 5662/5662 bytes
+2026-02-12 01:53:37,212 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,213 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/paraiso_dark.py')
+2026-02-12 01:53:37,216 - INFO - Upload completed successfully: paraiso_dark.py
+2026-02-12 01:53:37,216 - INFO - Starting upload: lib/pygments/styles/onedark.py -> /home/kevin/test/lib/pygments/styles/onedark.py (1719 bytes)
+2026-02-12 01:53:37,218 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/onedark.py', 'wb')
+2026-02-12 01:53:37,219 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/onedark.py', 'wb') -> 00000000
+2026-02-12 01:53:37,220 - DEBUG - Progress: 1719/1719 bytes
+2026-02-12 01:53:37,220 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,221 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/onedark.py')
+2026-02-12 01:53:37,222 - INFO - Upload completed successfully: onedark.py
+2026-02-12 01:53:37,223 - INFO - Starting upload: lib/pygments/styles/nord.py -> /home/kevin/test/lib/pygments/styles/nord.py (5391 bytes)
+2026-02-12 01:53:37,224 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/nord.py', 'wb')
+2026-02-12 01:53:37,225 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/nord.py', 'wb') -> 00000000
+2026-02-12 01:53:37,225 - DEBUG - Progress: 5391/5391 bytes
+2026-02-12 01:53:37,225 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,226 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/nord.py')
+2026-02-12 01:53:37,228 - INFO - Upload completed successfully: nord.py
+2026-02-12 01:53:37,228 - INFO - Starting upload: lib/pygments/styles/native.py -> /home/kevin/test/lib/pygments/styles/native.py (2043 bytes)
+2026-02-12 01:53:37,229 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/native.py', 'wb')
+2026-02-12 01:53:37,230 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/native.py', 'wb') -> 00000000
+2026-02-12 01:53:37,230 - DEBUG - Progress: 2043/2043 bytes
+2026-02-12 01:53:37,230 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,231 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/native.py')
+2026-02-12 01:53:37,233 - INFO - Upload completed successfully: native.py
+2026-02-12 01:53:37,233 - INFO - Starting upload: lib/pygments/styles/murphy.py -> /home/kevin/test/lib/pygments/styles/murphy.py (2805 bytes)
+2026-02-12 01:53:37,235 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/murphy.py', 'wb')
+2026-02-12 01:53:37,235 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/murphy.py', 'wb') -> 00000000
+2026-02-12 01:53:37,235 - DEBUG - Progress: 2805/2805 bytes
+2026-02-12 01:53:37,235 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,236 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/murphy.py')
+2026-02-12 01:53:37,238 - INFO - Upload completed successfully: murphy.py
+2026-02-12 01:53:37,238 - INFO - Starting upload: lib/pygments/styles/monokai.py -> /home/kevin/test/lib/pygments/styles/monokai.py (5184 bytes)
+2026-02-12 01:53:37,239 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/monokai.py', 'wb')
+2026-02-12 01:53:37,240 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/monokai.py', 'wb') -> 00000000
+2026-02-12 01:53:37,240 - DEBUG - Progress: 5184/5184 bytes
+2026-02-12 01:53:37,240 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,241 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/monokai.py')
+2026-02-12 01:53:37,243 - INFO - Upload completed successfully: monokai.py
+2026-02-12 01:53:37,243 - INFO - Starting upload: lib/pygments/styles/material.py -> /home/kevin/test/lib/pygments/styles/material.py (4201 bytes)
+2026-02-12 01:53:37,244 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/material.py', 'wb')
+2026-02-12 01:53:37,245 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/material.py', 'wb') -> 00000000
+2026-02-12 01:53:37,245 - DEBUG - Progress: 4201/4201 bytes
+2026-02-12 01:53:37,245 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,246 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/material.py')
+2026-02-12 01:53:37,248 - INFO - Upload completed successfully: material.py
+2026-02-12 01:53:37,248 - INFO - Starting upload: lib/pygments/styles/manni.py -> /home/kevin/test/lib/pygments/styles/manni.py (2443 bytes)
+2026-02-12 01:53:37,250 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/manni.py', 'wb')
+2026-02-12 01:53:37,250 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/manni.py', 'wb') -> 00000000
+2026-02-12 01:53:37,250 - DEBUG - Progress: 2443/2443 bytes
+2026-02-12 01:53:37,250 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,252 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/manni.py')
+2026-02-12 01:53:37,253 - INFO - Upload completed successfully: manni.py
+2026-02-12 01:53:37,253 - INFO - Starting upload: lib/pygments/styles/lovelace.py -> /home/kevin/test/lib/pygments/styles/lovelace.py (3178 bytes)
+2026-02-12 01:53:37,255 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/lovelace.py', 'wb')
+2026-02-12 01:53:37,255 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/lovelace.py', 'wb') -> 00000000
+2026-02-12 01:53:37,256 - DEBUG - Progress: 3178/3178 bytes
+2026-02-12 01:53:37,256 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,257 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/lovelace.py')
+2026-02-12 01:53:37,259 - INFO - Upload completed successfully: lovelace.py
+2026-02-12 01:53:37,259 - INFO - Starting upload: lib/pygments/styles/lilypond.py -> /home/kevin/test/lib/pygments/styles/lilypond.py (2066 bytes)
+2026-02-12 01:53:37,260 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/lilypond.py', 'wb')
+2026-02-12 01:53:37,261 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/lilypond.py', 'wb') -> 00000000
+2026-02-12 01:53:37,261 - DEBUG - Progress: 2066/2066 bytes
+2026-02-12 01:53:37,261 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,262 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/lilypond.py')
+2026-02-12 01:53:37,264 - INFO - Upload completed successfully: lilypond.py
+2026-02-12 01:53:37,264 - INFO - Starting upload: lib/pygments/styles/lightbulb.py -> /home/kevin/test/lib/pygments/styles/lightbulb.py (3172 bytes)
+2026-02-12 01:53:37,266 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/lightbulb.py', 'wb')
+2026-02-12 01:53:37,266 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/lightbulb.py', 'wb') -> 00000000
+2026-02-12 01:53:37,267 - DEBUG - Progress: 3172/3172 bytes
+2026-02-12 01:53:37,267 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,268 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/lightbulb.py')
+2026-02-12 01:53:37,270 - INFO - Upload completed successfully: lightbulb.py
+2026-02-12 01:53:37,270 - INFO - Starting upload: lib/pygments/styles/inkpot.py -> /home/kevin/test/lib/pygments/styles/inkpot.py (2404 bytes)
+2026-02-12 01:53:37,271 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/inkpot.py', 'wb')
+2026-02-12 01:53:37,272 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/inkpot.py', 'wb') -> 00000000
+2026-02-12 01:53:37,272 - DEBUG - Progress: 2404/2404 bytes
+2026-02-12 01:53:37,272 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,273 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/inkpot.py')
+2026-02-12 01:53:37,275 - INFO - Upload completed successfully: inkpot.py
+2026-02-12 01:53:37,275 - INFO - Starting upload: lib/pygments/styles/igor.py -> /home/kevin/test/lib/pygments/styles/igor.py (737 bytes)
+2026-02-12 01:53:37,277 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/igor.py', 'wb')
+2026-02-12 01:53:37,277 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/igor.py', 'wb') -> 00000000
+2026-02-12 01:53:37,277 - DEBUG - Progress: 737/737 bytes
+2026-02-12 01:53:37,277 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,278 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/igor.py')
+2026-02-12 01:53:37,280 - INFO - Upload completed successfully: igor.py
+2026-02-12 01:53:37,280 - INFO - Starting upload: lib/pygments/styles/gruvbox.py -> /home/kevin/test/lib/pygments/styles/gruvbox.py (3387 bytes)
+2026-02-12 01:53:37,281 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/gruvbox.py', 'wb')
+2026-02-12 01:53:37,282 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/gruvbox.py', 'wb') -> 00000000
+2026-02-12 01:53:37,282 - DEBUG - Progress: 3387/3387 bytes
+2026-02-12 01:53:37,282 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,283 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/gruvbox.py')
+2026-02-12 01:53:37,285 - INFO - Upload completed successfully: gruvbox.py
+2026-02-12 01:53:37,285 - INFO - Starting upload: lib/pygments/styles/gh_dark.py -> /home/kevin/test/lib/pygments/styles/gh_dark.py (3590 bytes)
+2026-02-12 01:53:37,286 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/gh_dark.py', 'wb')
+2026-02-12 01:53:37,287 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/gh_dark.py', 'wb') -> 00000000
+2026-02-12 01:53:37,287 - DEBUG - Progress: 3590/3590 bytes
+2026-02-12 01:53:37,287 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,288 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/gh_dark.py')
+2026-02-12 01:53:37,291 - INFO - Upload completed successfully: gh_dark.py
+2026-02-12 01:53:37,291 - INFO - Starting upload: lib/pygments/styles/fruity.py -> /home/kevin/test/lib/pygments/styles/fruity.py (1324 bytes)
+2026-02-12 01:53:37,292 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/fruity.py', 'wb')
+2026-02-12 01:53:37,293 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/fruity.py', 'wb') -> 00000000
+2026-02-12 01:53:37,293 - DEBUG - Progress: 1324/1324 bytes
+2026-02-12 01:53:37,293 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,294 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/fruity.py')
+2026-02-12 01:53:37,296 - INFO - Upload completed successfully: fruity.py
+2026-02-12 01:53:37,296 - INFO - Starting upload: lib/pygments/styles/friendly_grayscale.py -> /home/kevin/test/lib/pygments/styles/friendly_grayscale.py (2828 bytes)
+2026-02-12 01:53:37,298 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/friendly_grayscale.py', 'wb')
+2026-02-12 01:53:37,299 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/friendly_grayscale.py', 'wb') -> 00000000
+2026-02-12 01:53:37,299 - DEBUG - Progress: 2828/2828 bytes
+2026-02-12 01:53:37,299 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,300 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/friendly_grayscale.py')
+2026-02-12 01:53:37,302 - INFO - Upload completed successfully: friendly_grayscale.py
+2026-02-12 01:53:37,302 - INFO - Starting upload: lib/pygments/styles/friendly.py -> /home/kevin/test/lib/pygments/styles/friendly.py (2604 bytes)
+2026-02-12 01:53:37,304 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/friendly.py', 'wb')
+2026-02-12 01:53:37,305 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/friendly.py', 'wb') -> 00000000
+2026-02-12 01:53:37,305 - DEBUG - Progress: 2604/2604 bytes
+2026-02-12 01:53:37,305 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,306 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/friendly.py')
+2026-02-12 01:53:37,308 - INFO - Upload completed successfully: friendly.py
+2026-02-12 01:53:37,309 - INFO - Starting upload: lib/pygments/styles/emacs.py -> /home/kevin/test/lib/pygments/styles/emacs.py (2535 bytes)
+2026-02-12 01:53:37,310 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/emacs.py', 'wb')
+2026-02-12 01:53:37,311 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/emacs.py', 'wb') -> 00000000
+2026-02-12 01:53:37,311 - DEBUG - Progress: 2535/2535 bytes
+2026-02-12 01:53:37,311 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,312 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/emacs.py')
+2026-02-12 01:53:37,314 - INFO - Upload completed successfully: emacs.py
+2026-02-12 01:53:37,314 - INFO - Starting upload: lib/pygments/styles/dracula.py -> /home/kevin/test/lib/pygments/styles/dracula.py (2182 bytes)
+2026-02-12 01:53:37,316 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/dracula.py', 'wb')
+2026-02-12 01:53:37,316 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/dracula.py', 'wb') -> 00000000
+2026-02-12 01:53:37,317 - DEBUG - Progress: 2182/2182 bytes
+2026-02-12 01:53:37,317 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,318 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/dracula.py')
+2026-02-12 01:53:37,320 - INFO - Upload completed successfully: dracula.py
+2026-02-12 01:53:37,320 - INFO - Starting upload: lib/pygments/styles/default.py -> /home/kevin/test/lib/pygments/styles/default.py (2588 bytes)
+2026-02-12 01:53:37,321 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/default.py', 'wb')
+2026-02-12 01:53:37,322 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/default.py', 'wb') -> 00000000
+2026-02-12 01:53:37,322 - DEBUG - Progress: 2588/2588 bytes
+2026-02-12 01:53:37,322 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,323 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/default.py')
+2026-02-12 01:53:37,325 - INFO - Upload completed successfully: default.py
+2026-02-12 01:53:37,325 - INFO - Starting upload: lib/pygments/styles/colorful.py -> /home/kevin/test/lib/pygments/styles/colorful.py (2832 bytes)
+2026-02-12 01:53:37,326 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/colorful.py', 'wb')
+2026-02-12 01:53:37,327 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/colorful.py', 'wb') -> 00000000
+2026-02-12 01:53:37,327 - DEBUG - Progress: 2832/2832 bytes
+2026-02-12 01:53:37,327 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,328 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/colorful.py')
+2026-02-12 01:53:37,331 - INFO - Upload completed successfully: colorful.py
+2026-02-12 01:53:37,331 - INFO - Starting upload: lib/pygments/styles/coffee.py -> /home/kevin/test/lib/pygments/styles/coffee.py (2308 bytes)
+2026-02-12 01:53:37,333 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/coffee.py', 'wb')
+2026-02-12 01:53:37,333 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/coffee.py', 'wb') -> 00000000
+2026-02-12 01:53:37,334 - DEBUG - Progress: 2308/2308 bytes
+2026-02-12 01:53:37,334 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,335 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/coffee.py')
+2026-02-12 01:53:37,337 - INFO - Upload completed successfully: coffee.py
+2026-02-12 01:53:37,337 - INFO - Starting upload: lib/pygments/styles/bw.py -> /home/kevin/test/lib/pygments/styles/bw.py (1406 bytes)
+2026-02-12 01:53:37,338 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/bw.py', 'wb')
+2026-02-12 01:53:37,339 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/bw.py', 'wb') -> 00000000
+2026-02-12 01:53:37,339 - DEBUG - Progress: 1406/1406 bytes
+2026-02-12 01:53:37,339 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,340 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/bw.py')
+2026-02-12 01:53:37,343 - INFO - Upload completed successfully: bw.py
+2026-02-12 01:53:37,343 - INFO - Starting upload: lib/pygments/styles/borland.py -> /home/kevin/test/lib/pygments/styles/borland.py (1611 bytes)
+2026-02-12 01:53:37,344 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/borland.py', 'wb')
+2026-02-12 01:53:37,345 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/borland.py', 'wb') -> 00000000
+2026-02-12 01:53:37,346 - DEBUG - Progress: 1611/1611 bytes
+2026-02-12 01:53:37,346 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,347 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/borland.py')
+2026-02-12 01:53:37,348 - INFO - Upload completed successfully: borland.py
+2026-02-12 01:53:37,348 - INFO - Starting upload: lib/pygments/styles/autumn.py -> /home/kevin/test/lib/pygments/styles/autumn.py (2195 bytes)
+2026-02-12 01:53:37,350 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/autumn.py', 'wb')
+2026-02-12 01:53:37,350 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/autumn.py', 'wb') -> 00000000
+2026-02-12 01:53:37,351 - DEBUG - Progress: 2195/2195 bytes
+2026-02-12 01:53:37,351 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,351 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/autumn.py')
+2026-02-12 01:53:37,353 - INFO - Upload completed successfully: autumn.py
+2026-02-12 01:53:37,353 - INFO - Starting upload: lib/pygments/styles/arduino.py -> /home/kevin/test/lib/pygments/styles/arduino.py (4557 bytes)
+2026-02-12 01:53:37,355 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/arduino.py', 'wb')
+2026-02-12 01:53:37,356 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/arduino.py', 'wb') -> 00000000
+2026-02-12 01:53:37,356 - DEBUG - Progress: 4557/4557 bytes
+2026-02-12 01:53:37,356 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,357 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/arduino.py')
+2026-02-12 01:53:37,359 - INFO - Upload completed successfully: arduino.py
+2026-02-12 01:53:37,359 - INFO - Starting upload: lib/pygments/styles/algol_nu.py -> /home/kevin/test/lib/pygments/styles/algol_nu.py (2283 bytes)
+2026-02-12 01:53:37,361 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/algol_nu.py', 'wb')
+2026-02-12 01:53:37,362 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/algol_nu.py', 'wb') -> 00000000
+2026-02-12 01:53:37,362 - DEBUG - Progress: 2283/2283 bytes
+2026-02-12 01:53:37,362 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,363 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/algol_nu.py')
+2026-02-12 01:53:37,364 - INFO - Upload completed successfully: algol_nu.py
+2026-02-12 01:53:37,365 - INFO - Starting upload: lib/pygments/styles/algol.py -> /home/kevin/test/lib/pygments/styles/algol.py (2262 bytes)
+2026-02-12 01:53:37,366 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/algol.py', 'wb')
+2026-02-12 01:53:37,367 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/algol.py', 'wb') -> 00000000
+2026-02-12 01:53:37,367 - DEBUG - Progress: 2262/2262 bytes
+2026-02-12 01:53:37,367 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,368 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/algol.py')
+2026-02-12 01:53:37,370 - INFO - Upload completed successfully: algol.py
+2026-02-12 01:53:37,371 - INFO - Starting upload: lib/pygments/styles/abap.py -> /home/kevin/test/lib/pygments/styles/abap.py (749 bytes)
+2026-02-12 01:53:37,372 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/abap.py', 'wb')
+2026-02-12 01:53:37,373 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/abap.py', 'wb') -> 00000000
+2026-02-12 01:53:37,373 - DEBUG - Progress: 749/749 bytes
+2026-02-12 01:53:37,373 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,374 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/abap.py')
+2026-02-12 01:53:37,376 - INFO - Upload completed successfully: abap.py
+2026-02-12 01:53:37,376 - INFO - Starting upload: lib/pygments/styles/_mapping.py -> /home/kevin/test/lib/pygments/styles/_mapping.py (3312 bytes)
+2026-02-12 01:53:37,377 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/_mapping.py', 'wb')
+2026-02-12 01:53:37,378 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/_mapping.py', 'wb') -> 00000000
+2026-02-12 01:53:37,378 - DEBUG - Progress: 3312/3312 bytes
+2026-02-12 01:53:37,378 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,379 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/_mapping.py')
+2026-02-12 01:53:37,381 - INFO - Upload completed successfully: _mapping.py
+2026-02-12 01:53:37,381 - INFO - Starting upload: lib/pygments/styles/__init__.py -> /home/kevin/test/lib/pygments/styles/__init__.py (2006 bytes)
+2026-02-12 01:53:37,383 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__init__.py', 'wb')
+2026-02-12 01:53:37,384 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__init__.py', 'wb') -> 00000000
+2026-02-12 01:53:37,384 - DEBUG - Progress: 2006/2006 bytes
+2026-02-12 01:53:37,384 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,385 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__init__.py')
+2026-02-12 01:53:37,387 - INFO - Upload completed successfully: __init__.py
+2026-02-12 01:53:37,387 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments/styles/__pycache__', 511)
+2026-02-12 01:53:37,388 - INFO - Created remote folder: /home/kevin/test/lib/pygments/styles/__pycache__
+2026-02-12 01:53:37,388 - INFO - Starting upload: lib/pygments/styles/__pycache__/zenburn.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/zenburn.cpython-314.pyc (3312 bytes)
+2026-02-12 01:53:37,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/zenburn.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/zenburn.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,390 - DEBUG - Progress: 3312/3312 bytes
+2026-02-12 01:53:37,390 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,392 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/zenburn.cpython-314.pyc')
+2026-02-12 01:53:37,393 - INFO - Upload completed successfully: zenburn.cpython-314.pyc
+2026-02-12 01:53:37,394 - INFO - Starting upload: lib/pygments/styles/__pycache__/xcode.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/xcode.cpython-314.pyc (1811 bytes)
+2026-02-12 01:53:37,395 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/xcode.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,396 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/xcode.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,396 - DEBUG - Progress: 1811/1811 bytes
+2026-02-12 01:53:37,396 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,397 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/xcode.cpython-314.pyc')
+2026-02-12 01:53:37,399 - INFO - Upload completed successfully: xcode.cpython-314.pyc
+2026-02-12 01:53:37,400 - INFO - Starting upload: lib/pygments/styles/__pycache__/vs.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/vs.cpython-314.pyc (1481 bytes)
+2026-02-12 01:53:37,401 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/vs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,402 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/vs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,402 - DEBUG - Progress: 1481/1481 bytes
+2026-02-12 01:53:37,402 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,403 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/vs.cpython-314.pyc')
+2026-02-12 01:53:37,404 - INFO - Upload completed successfully: vs.cpython-314.pyc
+2026-02-12 01:53:37,404 - INFO - Starting upload: lib/pygments/styles/__pycache__/vim.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/vim.cpython-314.pyc (2468 bytes)
+2026-02-12 01:53:37,406 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/vim.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,406 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/vim.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,407 - DEBUG - Progress: 2468/2468 bytes
+2026-02-12 01:53:37,407 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,408 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/vim.cpython-314.pyc')
+2026-02-12 01:53:37,410 - INFO - Upload completed successfully: vim.cpython-314.pyc
+2026-02-12 01:53:37,410 - INFO - Starting upload: lib/pygments/styles/__pycache__/trac.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/trac.cpython-314.pyc (2585 bytes)
+2026-02-12 01:53:37,412 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/trac.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,413 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/trac.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,413 - DEBUG - Progress: 2585/2585 bytes
+2026-02-12 01:53:37,413 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,414 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/trac.cpython-314.pyc')
+2026-02-12 01:53:37,416 - INFO - Upload completed successfully: trac.cpython-314.pyc
+2026-02-12 01:53:37,416 - INFO - Starting upload: lib/pygments/styles/__pycache__/tango.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/tango.cpython-314.pyc (6021 bytes)
+2026-02-12 01:53:37,418 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/tango.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,418 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/tango.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,418 - DEBUG - Progress: 6021/6021 bytes
+2026-02-12 01:53:37,419 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,420 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/tango.cpython-314.pyc')
+2026-02-12 01:53:37,422 - INFO - Upload completed successfully: tango.cpython-314.pyc
+2026-02-12 01:53:37,422 - INFO - Starting upload: lib/pygments/styles/__pycache__/stata_light.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/stata_light.cpython-314.pyc (1652 bytes)
+2026-02-12 01:53:37,424 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/stata_light.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,425 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/stata_light.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,425 - DEBUG - Progress: 1652/1652 bytes
+2026-02-12 01:53:37,425 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,426 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/stata_light.cpython-314.pyc')
+2026-02-12 01:53:37,428 - INFO - Upload completed successfully: stata_light.cpython-314.pyc
+2026-02-12 01:53:37,428 - INFO - Starting upload: lib/pygments/styles/__pycache__/stata_dark.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/stata_dark.cpython-314.pyc (1663 bytes)
+2026-02-12 01:53:37,429 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/stata_dark.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,430 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/stata_dark.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,430 - DEBUG - Progress: 1663/1663 bytes
+2026-02-12 01:53:37,430 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,431 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/stata_dark.cpython-314.pyc')
+2026-02-12 01:53:37,433 - INFO - Upload completed successfully: stata_dark.cpython-314.pyc
+2026-02-12 01:53:37,433 - INFO - Starting upload: lib/pygments/styles/__pycache__/staroffice.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/staroffice.cpython-314.pyc (1092 bytes)
+2026-02-12 01:53:37,434 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/staroffice.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,435 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/staroffice.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,435 - DEBUG - Progress: 1092/1092 bytes
+2026-02-12 01:53:37,435 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,436 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/staroffice.cpython-314.pyc')
+2026-02-12 01:53:37,438 - INFO - Upload completed successfully: staroffice.cpython-314.pyc
+2026-02-12 01:53:37,438 - INFO - Starting upload: lib/pygments/styles/__pycache__/solarized.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/solarized.cpython-314.pyc (6173 bytes)
+2026-02-12 01:53:37,440 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/solarized.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,441 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/solarized.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,441 - DEBUG - Progress: 6173/6173 bytes
+2026-02-12 01:53:37,441 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,442 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/solarized.cpython-314.pyc')
+2026-02-12 01:53:37,444 - INFO - Upload completed successfully: solarized.cpython-314.pyc
+2026-02-12 01:53:37,444 - INFO - Starting upload: lib/pygments/styles/__pycache__/sas.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/sas.cpython-314.pyc (1773 bytes)
+2026-02-12 01:53:37,446 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/sas.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,446 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/sas.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,446 - DEBUG - Progress: 1773/1773 bytes
+2026-02-12 01:53:37,446 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,447 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/sas.cpython-314.pyc')
+2026-02-12 01:53:37,450 - INFO - Upload completed successfully: sas.cpython-314.pyc
+2026-02-12 01:53:37,450 - INFO - Starting upload: lib/pygments/styles/__pycache__/rrt.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/rrt.cpython-314.pyc (1444 bytes)
+2026-02-12 01:53:37,452 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/rrt.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,452 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/rrt.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,453 - DEBUG - Progress: 1444/1444 bytes
+2026-02-12 01:53:37,453 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,453 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/rrt.cpython-314.pyc')
+2026-02-12 01:53:37,455 - INFO - Upload completed successfully: rrt.cpython-314.pyc
+2026-02-12 01:53:37,455 - INFO - Starting upload: lib/pygments/styles/__pycache__/rainbow_dash.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/rainbow_dash.cpython-314.pyc (3678 bytes)
+2026-02-12 01:53:37,457 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/rainbow_dash.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,457 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/rainbow_dash.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,458 - DEBUG - Progress: 3678/3678 bytes
+2026-02-12 01:53:37,458 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,459 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/rainbow_dash.cpython-314.pyc')
+2026-02-12 01:53:37,460 - INFO - Upload completed successfully: rainbow_dash.cpython-314.pyc
+2026-02-12 01:53:37,460 - INFO - Starting upload: lib/pygments/styles/__pycache__/perldoc.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/perldoc.cpython-314.pyc (3032 bytes)
+2026-02-12 01:53:37,462 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/perldoc.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,463 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/perldoc.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,463 - DEBUG - Progress: 3032/3032 bytes
+2026-02-12 01:53:37,463 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,464 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/perldoc.cpython-314.pyc')
+2026-02-12 01:53:37,466 - INFO - Upload completed successfully: perldoc.cpython-314.pyc
+2026-02-12 01:53:37,466 - INFO - Starting upload: lib/pygments/styles/__pycache__/pastie.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/pastie.cpython-314.pyc (3443 bytes)
+2026-02-12 01:53:37,468 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/pastie.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,468 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/pastie.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,469 - DEBUG - Progress: 3443/3443 bytes
+2026-02-12 01:53:37,469 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,470 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/pastie.cpython-314.pyc')
+2026-02-12 01:53:37,471 - INFO - Upload completed successfully: pastie.cpython-314.pyc
+2026-02-12 01:53:37,471 - INFO - Starting upload: lib/pygments/styles/__pycache__/paraiso_light.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/paraiso_light.cpython-314.pyc (5136 bytes)
+2026-02-12 01:53:37,473 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/paraiso_light.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,473 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/paraiso_light.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,473 - DEBUG - Progress: 5136/5136 bytes
+2026-02-12 01:53:37,474 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,475 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/paraiso_light.cpython-314.pyc')
+2026-02-12 01:53:37,477 - INFO - Upload completed successfully: paraiso_light.cpython-314.pyc
+2026-02-12 01:53:37,477 - INFO - Starting upload: lib/pygments/styles/__pycache__/paraiso_dark.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/paraiso_dark.cpython-314.pyc (5130 bytes)
+2026-02-12 01:53:37,479 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/paraiso_dark.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,480 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/paraiso_dark.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,480 - DEBUG - Progress: 5130/5130 bytes
+2026-02-12 01:53:37,480 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,481 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/paraiso_dark.cpython-314.pyc')
+2026-02-12 01:53:37,483 - INFO - Upload completed successfully: paraiso_dark.cpython-314.pyc
+2026-02-12 01:53:37,483 - INFO - Starting upload: lib/pygments/styles/__pycache__/onedark.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/onedark.cpython-314.pyc (2232 bytes)
+2026-02-12 01:53:37,484 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/onedark.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,485 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/onedark.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,485 - DEBUG - Progress: 2232/2232 bytes
+2026-02-12 01:53:37,485 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,486 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/onedark.cpython-314.pyc')
+2026-02-12 01:53:37,488 - INFO - Upload completed successfully: onedark.cpython-314.pyc
+2026-02-12 01:53:37,488 - INFO - Starting upload: lib/pygments/styles/__pycache__/nord.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/nord.cpython-314.pyc (5577 bytes)
+2026-02-12 01:53:37,490 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/nord.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,491 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/nord.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,491 - DEBUG - Progress: 5577/5577 bytes
+2026-02-12 01:53:37,491 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,493 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/nord.cpython-314.pyc')
+2026-02-12 01:53:37,495 - INFO - Upload completed successfully: nord.cpython-314.pyc
+2026-02-12 01:53:37,495 - INFO - Starting upload: lib/pygments/styles/__pycache__/native.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/native.cpython-314.pyc (2938 bytes)
+2026-02-12 01:53:37,496 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/native.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,497 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/native.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,497 - DEBUG - Progress: 2938/2938 bytes
+2026-02-12 01:53:37,497 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,498 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/native.cpython-314.pyc')
+2026-02-12 01:53:37,500 - INFO - Upload completed successfully: native.cpython-314.pyc
+2026-02-12 01:53:37,500 - INFO - Starting upload: lib/pygments/styles/__pycache__/murphy.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/murphy.cpython-314.pyc (3661 bytes)
+2026-02-12 01:53:37,502 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/murphy.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,503 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/murphy.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,503 - DEBUG - Progress: 3661/3661 bytes
+2026-02-12 01:53:37,503 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,504 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/murphy.cpython-314.pyc')
+2026-02-12 01:53:37,507 - INFO - Upload completed successfully: murphy.cpython-314.pyc
+2026-02-12 01:53:37,507 - INFO - Starting upload: lib/pygments/styles/__pycache__/monokai.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/monokai.cpython-314.pyc (4766 bytes)
+2026-02-12 01:53:37,508 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/monokai.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,509 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/monokai.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,509 - DEBUG - Progress: 4766/4766 bytes
+2026-02-12 01:53:37,509 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,510 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/monokai.cpython-314.pyc')
+2026-02-12 01:53:37,512 - INFO - Upload completed successfully: monokai.cpython-314.pyc
+2026-02-12 01:53:37,512 - INFO - Starting upload: lib/pygments/styles/__pycache__/material.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/material.cpython-314.pyc (4814 bytes)
+2026-02-12 01:53:37,514 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/material.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,514 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/material.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,514 - DEBUG - Progress: 4814/4814 bytes
+2026-02-12 01:53:37,514 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,516 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/material.cpython-314.pyc')
+2026-02-12 01:53:37,518 - INFO - Upload completed successfully: material.cpython-314.pyc
+2026-02-12 01:53:37,518 - INFO - Starting upload: lib/pygments/styles/__pycache__/manni.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/manni.cpython-314.pyc (3468 bytes)
+2026-02-12 01:53:37,520 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/manni.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,520 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/manni.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,520 - DEBUG - Progress: 3468/3468 bytes
+2026-02-12 01:53:37,520 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,522 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/manni.cpython-314.pyc')
+2026-02-12 01:53:37,523 - INFO - Upload completed successfully: manni.cpython-314.pyc
+2026-02-12 01:53:37,524 - INFO - Starting upload: lib/pygments/styles/__pycache__/lovelace.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/lovelace.cpython-314.pyc (4249 bytes)
+2026-02-12 01:53:37,525 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/lovelace.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,526 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/lovelace.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,526 - DEBUG - Progress: 4249/4249 bytes
+2026-02-12 01:53:37,526 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,527 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/lovelace.cpython-314.pyc')
+2026-02-12 01:53:37,529 - INFO - Upload completed successfully: lovelace.cpython-314.pyc
+2026-02-12 01:53:37,530 - INFO - Starting upload: lib/pygments/styles/__pycache__/lilypond.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/lilypond.cpython-314.pyc (3484 bytes)
+2026-02-12 01:53:37,531 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/lilypond.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,532 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/lilypond.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,532 - DEBUG - Progress: 3484/3484 bytes
+2026-02-12 01:53:37,532 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,534 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/lilypond.cpython-314.pyc')
+2026-02-12 01:53:37,535 - INFO - Upload completed successfully: lilypond.cpython-314.pyc
+2026-02-12 01:53:37,536 - INFO - Starting upload: lib/pygments/styles/__pycache__/lightbulb.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/lightbulb.cpython-314.pyc (4803 bytes)
+2026-02-12 01:53:37,537 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/lightbulb.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,538 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/lightbulb.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,538 - DEBUG - Progress: 4803/4803 bytes
+2026-02-12 01:53:37,538 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,539 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/lightbulb.cpython-314.pyc')
+2026-02-12 01:53:37,541 - INFO - Upload completed successfully: lightbulb.cpython-314.pyc
+2026-02-12 01:53:37,541 - INFO - Starting upload: lib/pygments/styles/__pycache__/inkpot.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/inkpot.cpython-314.pyc (2984 bytes)
+2026-02-12 01:53:37,543 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/inkpot.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,543 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/inkpot.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,544 - DEBUG - Progress: 2984/2984 bytes
+2026-02-12 01:53:37,544 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,545 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/inkpot.cpython-314.pyc')
+2026-02-12 01:53:37,547 - INFO - Upload completed successfully: inkpot.cpython-314.pyc
+2026-02-12 01:53:37,547 - INFO - Starting upload: lib/pygments/styles/__pycache__/igor.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/igor.cpython-314.pyc (1114 bytes)
+2026-02-12 01:53:37,549 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/igor.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,549 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/igor.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,549 - DEBUG - Progress: 1114/1114 bytes
+2026-02-12 01:53:37,549 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,550 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/igor.cpython-314.pyc')
+2026-02-12 01:53:37,552 - INFO - Upload completed successfully: igor.cpython-314.pyc
+2026-02-12 01:53:37,552 - INFO - Starting upload: lib/pygments/styles/__pycache__/gruvbox.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/gruvbox.cpython-314.pyc (4367 bytes)
+2026-02-12 01:53:37,554 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/gruvbox.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,554 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/gruvbox.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,555 - DEBUG - Progress: 4367/4367 bytes
+2026-02-12 01:53:37,555 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,556 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/gruvbox.cpython-314.pyc')
+2026-02-12 01:53:37,558 - INFO - Upload completed successfully: gruvbox.cpython-314.pyc
+2026-02-12 01:53:37,558 - INFO - Starting upload: lib/pygments/styles/__pycache__/gh_dark.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/gh_dark.cpython-314.pyc (4093 bytes)
+2026-02-12 01:53:37,560 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/gh_dark.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,561 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/gh_dark.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,561 - DEBUG - Progress: 4093/4093 bytes
+2026-02-12 01:53:37,561 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,562 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/gh_dark.cpython-314.pyc')
+2026-02-12 01:53:37,563 - INFO - Upload completed successfully: gh_dark.cpython-314.pyc
+2026-02-12 01:53:37,563 - INFO - Starting upload: lib/pygments/styles/__pycache__/fruity.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/fruity.cpython-314.pyc (1920 bytes)
+2026-02-12 01:53:37,565 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/fruity.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,566 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/fruity.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,566 - DEBUG - Progress: 1920/1920 bytes
+2026-02-12 01:53:37,566 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,567 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/fruity.cpython-314.pyc')
+2026-02-12 01:53:37,569 - INFO - Upload completed successfully: fruity.cpython-314.pyc
+2026-02-12 01:53:37,569 - INFO - Starting upload: lib/pygments/styles/__pycache__/friendly_grayscale.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/friendly_grayscale.cpython-314.pyc (3535 bytes)
+2026-02-12 01:53:37,571 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/friendly_grayscale.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,572 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/friendly_grayscale.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,572 - DEBUG - Progress: 3535/3535 bytes
+2026-02-12 01:53:37,572 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,573 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/friendly_grayscale.cpython-314.pyc')
+2026-02-12 01:53:37,575 - INFO - Upload completed successfully: friendly_grayscale.cpython-314.pyc
+2026-02-12 01:53:37,575 - INFO - Starting upload: lib/pygments/styles/__pycache__/friendly.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/friendly.cpython-314.pyc (3341 bytes)
+2026-02-12 01:53:37,577 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/friendly.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,577 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/friendly.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,577 - DEBUG - Progress: 3341/3341 bytes
+2026-02-12 01:53:37,578 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,579 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/friendly.cpython-314.pyc')
+2026-02-12 01:53:37,581 - INFO - Upload completed successfully: friendly.cpython-314.pyc
+2026-02-12 01:53:37,581 - INFO - Starting upload: lib/pygments/styles/__pycache__/emacs.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/emacs.cpython-314.pyc (3245 bytes)
+2026-02-12 01:53:37,583 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/emacs.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/emacs.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,584 - DEBUG - Progress: 3245/3245 bytes
+2026-02-12 01:53:37,584 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,585 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/emacs.cpython-314.pyc')
+2026-02-12 01:53:37,587 - INFO - Upload completed successfully: emacs.cpython-314.pyc
+2026-02-12 01:53:37,587 - INFO - Starting upload: lib/pygments/styles/__pycache__/dracula.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/dracula.cpython-314.pyc (3050 bytes)
+2026-02-12 01:53:37,589 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/dracula.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,590 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/dracula.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,590 - DEBUG - Progress: 3050/3050 bytes
+2026-02-12 01:53:37,590 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,591 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/dracula.cpython-314.pyc')
+2026-02-12 01:53:37,593 - INFO - Upload completed successfully: dracula.cpython-314.pyc
+2026-02-12 01:53:37,593 - INFO - Starting upload: lib/pygments/styles/__pycache__/default.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/default.cpython-314.pyc (3205 bytes)
+2026-02-12 01:53:37,594 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/default.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,595 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/default.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,595 - DEBUG - Progress: 3205/3205 bytes
+2026-02-12 01:53:37,595 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,597 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/default.cpython-314.pyc')
+2026-02-12 01:53:37,599 - INFO - Upload completed successfully: default.cpython-314.pyc
+2026-02-12 01:53:37,599 - INFO - Starting upload: lib/pygments/styles/__pycache__/colorful.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/colorful.cpython-314.pyc (3703 bytes)
+2026-02-12 01:53:37,600 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/colorful.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,601 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/colorful.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,601 - DEBUG - Progress: 3703/3703 bytes
+2026-02-12 01:53:37,601 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,602 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/colorful.cpython-314.pyc')
+2026-02-12 01:53:37,604 - INFO - Upload completed successfully: colorful.cpython-314.pyc
+2026-02-12 01:53:37,604 - INFO - Starting upload: lib/pygments/styles/__pycache__/coffee.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/coffee.cpython-314.pyc (3435 bytes)
+2026-02-12 01:53:37,605 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/coffee.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,606 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/coffee.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,606 - DEBUG - Progress: 3435/3435 bytes
+2026-02-12 01:53:37,606 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,607 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/coffee.cpython-314.pyc')
+2026-02-12 01:53:37,610 - INFO - Upload completed successfully: coffee.cpython-314.pyc
+2026-02-12 01:53:37,610 - INFO - Starting upload: lib/pygments/styles/__pycache__/bw.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/bw.cpython-314.pyc (1899 bytes)
+2026-02-12 01:53:37,612 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/bw.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,612 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/bw.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,613 - DEBUG - Progress: 1899/1899 bytes
+2026-02-12 01:53:37,613 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,614 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/bw.cpython-314.pyc')
+2026-02-12 01:53:37,616 - INFO - Upload completed successfully: bw.cpython-314.pyc
+2026-02-12 01:53:37,616 - INFO - Starting upload: lib/pygments/styles/__pycache__/borland.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/borland.cpython-314.pyc (2214 bytes)
+2026-02-12 01:53:37,617 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/borland.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,618 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/borland.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,618 - DEBUG - Progress: 2214/2214 bytes
+2026-02-12 01:53:37,618 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,619 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/borland.cpython-314.pyc')
+2026-02-12 01:53:37,621 - INFO - Upload completed successfully: borland.cpython-314.pyc
+2026-02-12 01:53:37,622 - INFO - Starting upload: lib/pygments/styles/__pycache__/autumn.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/autumn.cpython-314.pyc (2845 bytes)
+2026-02-12 01:53:37,623 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/autumn.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,624 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/autumn.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,624 - DEBUG - Progress: 2845/2845 bytes
+2026-02-12 01:53:37,624 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,626 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/autumn.cpython-314.pyc')
+2026-02-12 01:53:37,628 - INFO - Upload completed successfully: autumn.cpython-314.pyc
+2026-02-12 01:53:37,628 - INFO - Starting upload: lib/pygments/styles/__pycache__/arduino.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/arduino.cpython-314.pyc (4259 bytes)
+2026-02-12 01:53:37,629 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/arduino.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,630 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/arduino.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,630 - DEBUG - Progress: 4259/4259 bytes
+2026-02-12 01:53:37,630 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,631 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/arduino.cpython-314.pyc')
+2026-02-12 01:53:37,633 - INFO - Upload completed successfully: arduino.cpython-314.pyc
+2026-02-12 01:53:37,633 - INFO - Starting upload: lib/pygments/styles/__pycache__/algol_nu.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/algol_nu.cpython-314.pyc (2552 bytes)
+2026-02-12 01:53:37,635 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/algol_nu.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,635 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/algol_nu.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,635 - DEBUG - Progress: 2552/2552 bytes
+2026-02-12 01:53:37,636 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,637 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/algol_nu.cpython-314.pyc')
+2026-02-12 01:53:37,639 - INFO - Upload completed successfully: algol_nu.cpython-314.pyc
+2026-02-12 01:53:37,639 - INFO - Starting upload: lib/pygments/styles/__pycache__/algol.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/algol.cpython-314.pyc (2537 bytes)
+2026-02-12 01:53:37,641 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/algol.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,641 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/algol.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,642 - DEBUG - Progress: 2537/2537 bytes
+2026-02-12 01:53:37,642 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,643 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/algol.cpython-314.pyc')
+2026-02-12 01:53:37,645 - INFO - Upload completed successfully: algol.cpython-314.pyc
+2026-02-12 01:53:37,645 - INFO - Starting upload: lib/pygments/styles/__pycache__/abap.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/abap.cpython-314.pyc (1077 bytes)
+2026-02-12 01:53:37,646 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/abap.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,647 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/abap.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,647 - DEBUG - Progress: 1077/1077 bytes
+2026-02-12 01:53:37,647 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,648 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/abap.cpython-314.pyc')
+2026-02-12 01:53:37,650 - INFO - Upload completed successfully: abap.cpython-314.pyc
+2026-02-12 01:53:37,650 - INFO - Starting upload: lib/pygments/styles/__pycache__/_mapping.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/_mapping.cpython-314.pyc (3683 bytes)
+2026-02-12 01:53:37,652 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/_mapping.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,653 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/_mapping.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,653 - DEBUG - Progress: 3683/3683 bytes
+2026-02-12 01:53:37,653 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,654 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/_mapping.cpython-314.pyc')
+2026-02-12 01:53:37,656 - INFO - Upload completed successfully: _mapping.cpython-314.pyc
+2026-02-12 01:53:37,656 - INFO - Starting upload: lib/pygments/styles/__pycache__/__init__.cpython-314.pyc -> /home/kevin/test/lib/pygments/styles/__pycache__/__init__.cpython-314.pyc (2678 bytes)
+2026-02-12 01:53:37,657 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/__init__.cpython-314.pyc', 'wb')
+2026-02-12 01:53:37,658 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/styles/__pycache__/__init__.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 01:53:37,658 - DEBUG - Progress: 2678/2678 bytes
+2026-02-12 01:53:37,658 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,660 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/styles/__pycache__/__init__.cpython-314.pyc')
+2026-02-12 01:53:37,662 - INFO - Upload completed successfully: __init__.cpython-314.pyc
+2026-02-12 01:53:37,662 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/lib/pygments/lexers', 511)
+2026-02-12 01:53:37,663 - INFO - Created remote folder: /home/kevin/test/lib/pygments/lexers
+2026-02-12 01:53:37,663 - INFO - Starting upload: lib/pygments/lexers/zig.py -> /home/kevin/test/lib/pygments/lexers/zig.py (3976 bytes)
+2026-02-12 01:53:37,665 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/zig.py', 'wb')
+2026-02-12 01:53:37,666 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/zig.py', 'wb') -> 00000000
+2026-02-12 01:53:37,666 - DEBUG - Progress: 3976/3976 bytes
+2026-02-12 01:53:37,666 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,667 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/zig.py')
+2026-02-12 01:53:37,669 - INFO - Upload completed successfully: zig.py
+2026-02-12 01:53:37,669 - INFO - Starting upload: lib/pygments/lexers/yara.py -> /home/kevin/test/lib/pygments/lexers/yara.py (2427 bytes)
+2026-02-12 01:53:37,671 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/yara.py', 'wb')
+2026-02-12 01:53:37,671 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/yara.py', 'wb') -> 00000000
+2026-02-12 01:53:37,671 - DEBUG - Progress: 2427/2427 bytes
+2026-02-12 01:53:37,671 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,673 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/yara.py')
+2026-02-12 01:53:37,674 - INFO - Upload completed successfully: yara.py
+2026-02-12 01:53:37,674 - INFO - Starting upload: lib/pygments/lexers/yang.py -> /home/kevin/test/lib/pygments/lexers/yang.py (4499 bytes)
+2026-02-12 01:53:37,676 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/yang.py', 'wb')
+2026-02-12 01:53:37,677 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/yang.py', 'wb') -> 00000000
+2026-02-12 01:53:37,677 - DEBUG - Progress: 4499/4499 bytes
+2026-02-12 01:53:37,677 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,678 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/yang.py')
+2026-02-12 01:53:37,680 - INFO - Upload completed successfully: yang.py
+2026-02-12 01:53:37,680 - INFO - Starting upload: lib/pygments/lexers/xorg.py -> /home/kevin/test/lib/pygments/lexers/xorg.py (925 bytes)
+2026-02-12 01:53:37,681 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/xorg.py', 'wb')
+2026-02-12 01:53:37,682 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/xorg.py', 'wb') -> 00000000
+2026-02-12 01:53:37,682 - DEBUG - Progress: 925/925 bytes
+2026-02-12 01:53:37,682 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,683 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/xorg.py')
+2026-02-12 01:53:37,685 - INFO - Upload completed successfully: xorg.py
+2026-02-12 01:53:37,685 - INFO - Starting upload: lib/pygments/lexers/x10.py -> /home/kevin/test/lib/pygments/lexers/x10.py (1943 bytes)
+2026-02-12 01:53:37,686 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/x10.py', 'wb')
+2026-02-12 01:53:37,687 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/x10.py', 'wb') -> 00000000
+2026-02-12 01:53:37,687 - DEBUG - Progress: 1943/1943 bytes
+2026-02-12 01:53:37,687 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,688 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/x10.py')
+2026-02-12 01:53:37,690 - INFO - Upload completed successfully: x10.py
+2026-02-12 01:53:37,690 - INFO - Starting upload: lib/pygments/lexers/wren.py -> /home/kevin/test/lib/pygments/lexers/wren.py (3229 bytes)
+2026-02-12 01:53:37,692 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/wren.py', 'wb')
+2026-02-12 01:53:37,693 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/wren.py', 'wb') -> 00000000
+2026-02-12 01:53:37,693 - DEBUG - Progress: 3229/3229 bytes
+2026-02-12 01:53:37,693 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,694 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/wren.py')
+2026-02-12 01:53:37,696 - INFO - Upload completed successfully: wren.py
+2026-02-12 01:53:37,696 - INFO - Starting upload: lib/pygments/lexers/wowtoc.py -> /home/kevin/test/lib/pygments/lexers/wowtoc.py (4076 bytes)
+2026-02-12 01:53:37,698 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/wowtoc.py', 'wb')
+2026-02-12 01:53:37,699 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/wowtoc.py', 'wb') -> 00000000
+2026-02-12 01:53:37,699 - DEBUG - Progress: 4076/4076 bytes
+2026-02-12 01:53:37,699 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,700 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/wowtoc.py')
+2026-02-12 01:53:37,702 - INFO - Upload completed successfully: wowtoc.py
+2026-02-12 01:53:37,702 - INFO - Starting upload: lib/pygments/lexers/whiley.py -> /home/kevin/test/lib/pygments/lexers/whiley.py (4017 bytes)
+2026-02-12 01:53:37,704 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/whiley.py', 'wb')
+2026-02-12 01:53:37,705 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/whiley.py', 'wb') -> 00000000
+2026-02-12 01:53:37,705 - DEBUG - Progress: 4017/4017 bytes
+2026-02-12 01:53:37,705 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,707 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/whiley.py')
+2026-02-12 01:53:37,708 - INFO - Upload completed successfully: whiley.py
+2026-02-12 01:53:37,709 - INFO - Starting upload: lib/pygments/lexers/wgsl.py -> /home/kevin/test/lib/pygments/lexers/wgsl.py (11880 bytes)
+2026-02-12 01:53:37,710 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/wgsl.py', 'wb')
+2026-02-12 01:53:37,711 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/wgsl.py', 'wb') -> 00000000
+2026-02-12 01:53:37,711 - DEBUG - Progress: 11880/11880 bytes
+2026-02-12 01:53:37,711 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,713 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/wgsl.py')
+2026-02-12 01:53:37,715 - INFO - Upload completed successfully: wgsl.py
+2026-02-12 01:53:37,715 - INFO - Starting upload: lib/pygments/lexers/webmisc.py -> /home/kevin/test/lib/pygments/lexers/webmisc.py (40564 bytes)
+2026-02-12 01:53:37,717 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/webmisc.py', 'wb')
+2026-02-12 01:53:37,718 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/webmisc.py', 'wb') -> 00000000
+2026-02-12 01:53:37,718 - DEBUG - Progress: 32768/40564 bytes
+2026-02-12 01:53:37,718 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,723 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/webmisc.py')
+2026-02-12 01:53:37,725 - INFO - Upload completed successfully: webmisc.py
+2026-02-12 01:53:37,725 - INFO - Starting upload: lib/pygments/lexers/webidl.py -> /home/kevin/test/lib/pygments/lexers/webidl.py (10516 bytes)
+2026-02-12 01:53:37,727 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/webidl.py', 'wb')
+2026-02-12 01:53:37,728 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/webidl.py', 'wb') -> 00000000
+2026-02-12 01:53:37,728 - DEBUG - Progress: 10516/10516 bytes
+2026-02-12 01:53:37,728 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,730 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/webidl.py')
+2026-02-12 01:53:37,732 - INFO - Upload completed successfully: webidl.py
+2026-02-12 01:53:37,732 - INFO - Starting upload: lib/pygments/lexers/webassembly.py -> /home/kevin/test/lib/pygments/lexers/webassembly.py (5698 bytes)
+2026-02-12 01:53:37,734 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/webassembly.py', 'wb')
+2026-02-12 01:53:37,734 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/webassembly.py', 'wb') -> 00000000
+2026-02-12 01:53:37,734 - DEBUG - Progress: 5698/5698 bytes
+2026-02-12 01:53:37,735 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,736 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/webassembly.py')
+2026-02-12 01:53:37,737 - INFO - Upload completed successfully: webassembly.py
+2026-02-12 01:53:37,737 - INFO - Starting upload: lib/pygments/lexers/web.py -> /home/kevin/test/lib/pygments/lexers/web.py (913 bytes)
+2026-02-12 01:53:37,739 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/web.py', 'wb')
+2026-02-12 01:53:37,739 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/web.py', 'wb') -> 00000000
+2026-02-12 01:53:37,739 - DEBUG - Progress: 913/913 bytes
+2026-02-12 01:53:37,739 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,740 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/web.py')
+2026-02-12 01:53:37,743 - INFO - Upload completed successfully: web.py
+2026-02-12 01:53:37,743 - INFO - Starting upload: lib/pygments/lexers/vyper.py -> /home/kevin/test/lib/pygments/lexers/vyper.py (5615 bytes)
+2026-02-12 01:53:37,745 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/vyper.py', 'wb')
+2026-02-12 01:53:37,746 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/vyper.py', 'wb') -> 00000000
+2026-02-12 01:53:37,746 - DEBUG - Progress: 5615/5615 bytes
+2026-02-12 01:53:37,746 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,747 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/vyper.py')
+2026-02-12 01:53:37,750 - INFO - Upload completed successfully: vyper.py
+2026-02-12 01:53:37,750 - INFO - Starting upload: lib/pygments/lexers/vip.py -> /home/kevin/test/lib/pygments/lexers/vip.py (5711 bytes)
+2026-02-12 01:53:37,752 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/vip.py', 'wb')
+2026-02-12 01:53:37,753 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/vip.py', 'wb') -> 00000000
+2026-02-12 01:53:37,753 - DEBUG - Progress: 5711/5711 bytes
+2026-02-12 01:53:37,753 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,754 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/vip.py')
+2026-02-12 01:53:37,756 - INFO - Upload completed successfully: vip.py
+2026-02-12 01:53:37,756 - INFO - Starting upload: lib/pygments/lexers/verifpal.py -> /home/kevin/test/lib/pygments/lexers/verifpal.py (2661 bytes)
+2026-02-12 01:53:37,759 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/verifpal.py', 'wb')
+2026-02-12 01:53:37,759 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/verifpal.py', 'wb') -> 00000000
+2026-02-12 01:53:37,760 - DEBUG - Progress: 2661/2661 bytes
+2026-02-12 01:53:37,760 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,761 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/verifpal.py')
+2026-02-12 01:53:37,763 - INFO - Upload completed successfully: verifpal.py
+2026-02-12 01:53:37,763 - INFO - Starting upload: lib/pygments/lexers/verification.py -> /home/kevin/test/lib/pygments/lexers/verification.py (3934 bytes)
+2026-02-12 01:53:37,765 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/verification.py', 'wb')
+2026-02-12 01:53:37,766 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/verification.py', 'wb') -> 00000000
+2026-02-12 01:53:37,766 - DEBUG - Progress: 3934/3934 bytes
+2026-02-12 01:53:37,766 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,768 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/verification.py')
+2026-02-12 01:53:37,770 - INFO - Upload completed successfully: verification.py
+2026-02-12 01:53:37,770 - INFO - Starting upload: lib/pygments/lexers/varnish.py -> /home/kevin/test/lib/pygments/lexers/varnish.py (7473 bytes)
+2026-02-12 01:53:37,772 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/varnish.py', 'wb')
+2026-02-12 01:53:37,773 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/varnish.py', 'wb') -> 00000000
+2026-02-12 01:53:37,773 - DEBUG - Progress: 7473/7473 bytes
+2026-02-12 01:53:37,773 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,775 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/varnish.py')
+2026-02-12 01:53:37,776 - INFO - Upload completed successfully: varnish.py
+2026-02-12 01:53:37,776 - INFO - Starting upload: lib/pygments/lexers/usd.py -> /home/kevin/test/lib/pygments/lexers/usd.py (3304 bytes)
+2026-02-12 01:53:37,778 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/usd.py', 'wb')
+2026-02-12 01:53:37,778 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/usd.py', 'wb') -> 00000000
+2026-02-12 01:53:37,778 - DEBUG - Progress: 3304/3304 bytes
+2026-02-12 01:53:37,778 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,780 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/usd.py')
+2026-02-12 01:53:37,782 - INFO - Upload completed successfully: usd.py
+2026-02-12 01:53:37,783 - INFO - Starting upload: lib/pygments/lexers/urbi.py -> /home/kevin/test/lib/pygments/lexers/urbi.py (6082 bytes)
+2026-02-12 01:53:37,785 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/urbi.py', 'wb')
+2026-02-12 01:53:37,785 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/urbi.py', 'wb') -> 00000000
+2026-02-12 01:53:37,786 - DEBUG - Progress: 6082/6082 bytes
+2026-02-12 01:53:37,786 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,787 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/urbi.py')
+2026-02-12 01:53:37,789 - INFO - Upload completed successfully: urbi.py
+2026-02-12 01:53:37,789 - INFO - Starting upload: lib/pygments/lexers/unicon.py -> /home/kevin/test/lib/pygments/lexers/unicon.py (18625 bytes)
+2026-02-12 01:53:37,791 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/unicon.py', 'wb')
+2026-02-12 01:53:37,792 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/unicon.py', 'wb') -> 00000000
+2026-02-12 01:53:37,792 - DEBUG - Progress: 18625/18625 bytes
+2026-02-12 01:53:37,792 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,795 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/unicon.py')
+2026-02-12 01:53:37,797 - INFO - Upload completed successfully: unicon.py
+2026-02-12 01:53:37,798 - INFO - Starting upload: lib/pygments/lexers/ul4.py -> /home/kevin/test/lib/pygments/lexers/ul4.py (10499 bytes)
+2026-02-12 01:53:37,799 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ul4.py', 'wb')
+2026-02-12 01:53:37,800 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ul4.py', 'wb') -> 00000000
+2026-02-12 01:53:37,800 - DEBUG - Progress: 10499/10499 bytes
+2026-02-12 01:53:37,800 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,802 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ul4.py')
+2026-02-12 01:53:37,804 - INFO - Upload completed successfully: ul4.py
+2026-02-12 01:53:37,805 - INFO - Starting upload: lib/pygments/lexers/typst.py -> /home/kevin/test/lib/pygments/lexers/typst.py (7167 bytes)
+2026-02-12 01:53:37,807 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/typst.py', 'wb')
+2026-02-12 01:53:37,807 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/typst.py', 'wb') -> 00000000
+2026-02-12 01:53:37,808 - DEBUG - Progress: 7167/7167 bytes
+2026-02-12 01:53:37,808 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,809 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/typst.py')
+2026-02-12 01:53:37,812 - INFO - Upload completed successfully: typst.py
+2026-02-12 01:53:37,812 - INFO - Starting upload: lib/pygments/lexers/typoscript.py -> /home/kevin/test/lib/pygments/lexers/typoscript.py (8332 bytes)
+2026-02-12 01:53:37,814 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/typoscript.py', 'wb')
+2026-02-12 01:53:37,815 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/typoscript.py', 'wb') -> 00000000
+2026-02-12 01:53:37,815 - DEBUG - Progress: 8332/8332 bytes
+2026-02-12 01:53:37,815 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,817 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/typoscript.py')
+2026-02-12 01:53:37,819 - INFO - Upload completed successfully: typoscript.py
+2026-02-12 01:53:37,819 - INFO - Starting upload: lib/pygments/lexers/trafficscript.py -> /home/kevin/test/lib/pygments/lexers/trafficscript.py (1506 bytes)
+2026-02-12 01:53:37,821 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/trafficscript.py', 'wb')
+2026-02-12 01:53:37,822 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/trafficscript.py', 'wb') -> 00000000
+2026-02-12 01:53:37,822 - DEBUG - Progress: 1506/1506 bytes
+2026-02-12 01:53:37,822 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,823 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/trafficscript.py')
+2026-02-12 01:53:37,825 - INFO - Upload completed successfully: trafficscript.py
+2026-02-12 01:53:37,825 - INFO - Starting upload: lib/pygments/lexers/tnt.py -> /home/kevin/test/lib/pygments/lexers/tnt.py (10456 bytes)
+2026-02-12 01:53:37,827 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tnt.py', 'wb')
+2026-02-12 01:53:37,828 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tnt.py', 'wb') -> 00000000
+2026-02-12 01:53:37,828 - DEBUG - Progress: 10456/10456 bytes
+2026-02-12 01:53:37,828 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,830 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tnt.py')
+2026-02-12 01:53:37,832 - INFO - Upload completed successfully: tnt.py
+2026-02-12 01:53:37,832 - INFO - Starting upload: lib/pygments/lexers/tls.py -> /home/kevin/test/lib/pygments/lexers/tls.py (1540 bytes)
+2026-02-12 01:53:37,833 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tls.py', 'wb')
+2026-02-12 01:53:37,834 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tls.py', 'wb') -> 00000000
+2026-02-12 01:53:37,834 - DEBUG - Progress: 1540/1540 bytes
+2026-02-12 01:53:37,834 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,835 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tls.py')
+2026-02-12 01:53:37,837 - INFO - Upload completed successfully: tls.py
+2026-02-12 01:53:37,837 - INFO - Starting upload: lib/pygments/lexers/tlb.py -> /home/kevin/test/lib/pygments/lexers/tlb.py (1450 bytes)
+2026-02-12 01:53:37,839 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tlb.py', 'wb')
+2026-02-12 01:53:37,839 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tlb.py', 'wb') -> 00000000
+2026-02-12 01:53:37,839 - DEBUG - Progress: 1450/1450 bytes
+2026-02-12 01:53:37,840 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,840 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tlb.py')
+2026-02-12 01:53:37,842 - INFO - Upload completed successfully: tlb.py
+2026-02-12 01:53:37,843 - INFO - Starting upload: lib/pygments/lexers/thingsdb.py -> /home/kevin/test/lib/pygments/lexers/thingsdb.py (6017 bytes)
+2026-02-12 01:53:37,844 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/thingsdb.py', 'wb')
+2026-02-12 01:53:37,845 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/thingsdb.py', 'wb') -> 00000000
+2026-02-12 01:53:37,845 - DEBUG - Progress: 6017/6017 bytes
+2026-02-12 01:53:37,845 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,846 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/thingsdb.py')
+2026-02-12 01:53:37,848 - INFO - Upload completed successfully: thingsdb.py
+2026-02-12 01:53:37,848 - INFO - Starting upload: lib/pygments/lexers/theorem.py -> /home/kevin/test/lib/pygments/lexers/theorem.py (17855 bytes)
+2026-02-12 01:53:37,850 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/theorem.py', 'wb')
+2026-02-12 01:53:37,851 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/theorem.py', 'wb') -> 00000000
+2026-02-12 01:53:37,851 - DEBUG - Progress: 17855/17855 bytes
+2026-02-12 01:53:37,851 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,854 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/theorem.py')
+2026-02-12 01:53:37,856 - INFO - Upload completed successfully: theorem.py
+2026-02-12 01:53:37,856 - INFO - Starting upload: lib/pygments/lexers/textfmts.py -> /home/kevin/test/lib/pygments/lexers/textfmts.py (15524 bytes)
+2026-02-12 01:53:37,857 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/textfmts.py', 'wb')
+2026-02-12 01:53:37,858 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/textfmts.py', 'wb') -> 00000000
+2026-02-12 01:53:37,858 - DEBUG - Progress: 15524/15524 bytes
+2026-02-12 01:53:37,858 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,860 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/textfmts.py')
+2026-02-12 01:53:37,862 - INFO - Upload completed successfully: textfmts.py
+2026-02-12 01:53:37,862 - INFO - Starting upload: lib/pygments/lexers/textedit.py -> /home/kevin/test/lib/pygments/lexers/textedit.py (7760 bytes)
+2026-02-12 01:53:37,863 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/textedit.py', 'wb')
+2026-02-12 01:53:37,864 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/textedit.py', 'wb') -> 00000000
+2026-02-12 01:53:37,864 - DEBUG - Progress: 7760/7760 bytes
+2026-02-12 01:53:37,864 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,866 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/textedit.py')
+2026-02-12 01:53:37,868 - INFO - Upload completed successfully: textedit.py
+2026-02-12 01:53:37,868 - INFO - Starting upload: lib/pygments/lexers/text.py -> /home/kevin/test/lib/pygments/lexers/text.py (1068 bytes)
+2026-02-12 01:53:37,870 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/text.py', 'wb')
+2026-02-12 01:53:37,870 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/text.py', 'wb') -> 00000000
+2026-02-12 01:53:37,870 - DEBUG - Progress: 1068/1068 bytes
+2026-02-12 01:53:37,870 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,871 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/text.py')
+2026-02-12 01:53:37,873 - INFO - Upload completed successfully: text.py
+2026-02-12 01:53:37,873 - INFO - Starting upload: lib/pygments/lexers/testing.py -> /home/kevin/test/lib/pygments/lexers/testing.py (10810 bytes)
+2026-02-12 01:53:37,875 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/testing.py', 'wb')
+2026-02-12 01:53:37,875 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/testing.py', 'wb') -> 00000000
+2026-02-12 01:53:37,875 - DEBUG - Progress: 10810/10810 bytes
+2026-02-12 01:53:37,875 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,877 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/testing.py')
+2026-02-12 01:53:37,879 - INFO - Upload completed successfully: testing.py
+2026-02-12 01:53:37,879 - INFO - Starting upload: lib/pygments/lexers/teraterm.py -> /home/kevin/test/lib/pygments/lexers/teraterm.py (9718 bytes)
+2026-02-12 01:53:37,881 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/teraterm.py', 'wb')
+2026-02-12 01:53:37,882 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/teraterm.py', 'wb') -> 00000000
+2026-02-12 01:53:37,882 - DEBUG - Progress: 9718/9718 bytes
+2026-02-12 01:53:37,882 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,883 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/teraterm.py')
+2026-02-12 01:53:37,885 - INFO - Upload completed successfully: teraterm.py
+2026-02-12 01:53:37,885 - INFO - Starting upload: lib/pygments/lexers/templates.py -> /home/kevin/test/lib/pygments/lexers/templates.py (75731 bytes)
+2026-02-12 01:53:37,886 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/templates.py', 'wb')
+2026-02-12 01:53:37,887 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/templates.py', 'wb') -> 00000000
+2026-02-12 01:53:37,887 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,897 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/templates.py')
+2026-02-12 01:53:37,899 - INFO - Upload completed successfully: templates.py
+2026-02-12 01:53:37,899 - INFO - Starting upload: lib/pygments/lexers/teal.py -> /home/kevin/test/lib/pygments/lexers/teal.py (3522 bytes)
+2026-02-12 01:53:37,901 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/teal.py', 'wb')
+2026-02-12 01:53:37,902 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/teal.py', 'wb') -> 00000000
+2026-02-12 01:53:37,902 - DEBUG - Progress: 3522/3522 bytes
+2026-02-12 01:53:37,902 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,904 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/teal.py')
+2026-02-12 01:53:37,906 - INFO - Upload completed successfully: teal.py
+2026-02-12 01:53:37,906 - INFO - Starting upload: lib/pygments/lexers/tcl.py -> /home/kevin/test/lib/pygments/lexers/tcl.py (5512 bytes)
+2026-02-12 01:53:37,908 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tcl.py', 'wb')
+2026-02-12 01:53:37,909 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tcl.py', 'wb') -> 00000000
+2026-02-12 01:53:37,909 - DEBUG - Progress: 5512/5512 bytes
+2026-02-12 01:53:37,909 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,910 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tcl.py')
+2026-02-12 01:53:37,913 - INFO - Upload completed successfully: tcl.py
+2026-02-12 01:53:37,913 - INFO - Starting upload: lib/pygments/lexers/tal.py -> /home/kevin/test/lib/pygments/lexers/tal.py (2904 bytes)
+2026-02-12 01:53:37,915 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tal.py', 'wb')
+2026-02-12 01:53:37,916 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tal.py', 'wb') -> 00000000
+2026-02-12 01:53:37,916 - DEBUG - Progress: 2904/2904 bytes
+2026-02-12 01:53:37,916 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,918 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tal.py')
+2026-02-12 01:53:37,920 - INFO - Upload completed successfully: tal.py
+2026-02-12 01:53:37,920 - INFO - Starting upload: lib/pygments/lexers/tact.py -> /home/kevin/test/lib/pygments/lexers/tact.py (10809 bytes)
+2026-02-12 01:53:37,922 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tact.py', 'wb')
+2026-02-12 01:53:37,923 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tact.py', 'wb') -> 00000000
+2026-02-12 01:53:37,923 - DEBUG - Progress: 10809/10809 bytes
+2026-02-12 01:53:37,923 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,925 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tact.py')
+2026-02-12 01:53:37,928 - INFO - Upload completed successfully: tact.py
+2026-02-12 01:53:37,928 - INFO - Starting upload: lib/pygments/lexers/tablegen.py -> /home/kevin/test/lib/pygments/lexers/tablegen.py (3987 bytes)
+2026-02-12 01:53:37,930 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tablegen.py', 'wb')
+2026-02-12 01:53:37,931 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/tablegen.py', 'wb') -> 00000000
+2026-02-12 01:53:37,931 - DEBUG - Progress: 3987/3987 bytes
+2026-02-12 01:53:37,931 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,932 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/tablegen.py')
+2026-02-12 01:53:37,935 - INFO - Upload completed successfully: tablegen.py
+2026-02-12 01:53:37,935 - INFO - Starting upload: lib/pygments/lexers/supercollider.py -> /home/kevin/test/lib/pygments/lexers/supercollider.py (3697 bytes)
+2026-02-12 01:53:37,937 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/supercollider.py', 'wb')
+2026-02-12 01:53:37,938 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/supercollider.py', 'wb') -> 00000000
+2026-02-12 01:53:37,938 - DEBUG - Progress: 3697/3697 bytes
+2026-02-12 01:53:37,938 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,939 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/supercollider.py')
+2026-02-12 01:53:37,942 - INFO - Upload completed successfully: supercollider.py
+2026-02-12 01:53:37,942 - INFO - Starting upload: lib/pygments/lexers/stata.py -> /home/kevin/test/lib/pygments/lexers/stata.py (6415 bytes)
+2026-02-12 01:53:37,943 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/stata.py', 'wb')
+2026-02-12 01:53:37,944 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/stata.py', 'wb') -> 00000000
+2026-02-12 01:53:37,944 - DEBUG - Progress: 6415/6415 bytes
+2026-02-12 01:53:37,944 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,945 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/stata.py')
+2026-02-12 01:53:37,948 - INFO - Upload completed successfully: stata.py
+2026-02-12 01:53:37,948 - INFO - Starting upload: lib/pygments/lexers/srcinfo.py -> /home/kevin/test/lib/pygments/lexers/srcinfo.py (1746 bytes)
+2026-02-12 01:53:37,949 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/srcinfo.py', 'wb')
+2026-02-12 01:53:37,950 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/srcinfo.py', 'wb') -> 00000000
+2026-02-12 01:53:37,950 - DEBUG - Progress: 1746/1746 bytes
+2026-02-12 01:53:37,950 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,951 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/srcinfo.py')
+2026-02-12 01:53:37,953 - INFO - Upload completed successfully: srcinfo.py
+2026-02-12 01:53:37,953 - INFO - Starting upload: lib/pygments/lexers/sql.py -> /home/kevin/test/lib/pygments/lexers/sql.py (41476 bytes)
+2026-02-12 01:53:37,956 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sql.py', 'wb')
+2026-02-12 01:53:37,957 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sql.py', 'wb') -> 00000000
+2026-02-12 01:53:37,957 - DEBUG - Progress: 32768/41476 bytes
+2026-02-12 01:53:37,957 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,964 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/sql.py')
+2026-02-12 01:53:37,967 - INFO - Upload completed successfully: sql.py
+2026-02-12 01:53:37,967 - INFO - Starting upload: lib/pygments/lexers/spice.py -> /home/kevin/test/lib/pygments/lexers/spice.py (2790 bytes)
+2026-02-12 01:53:37,969 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/spice.py', 'wb')
+2026-02-12 01:53:37,970 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/spice.py', 'wb') -> 00000000
+2026-02-12 01:53:37,970 - DEBUG - Progress: 2790/2790 bytes
+2026-02-12 01:53:37,970 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,971 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/spice.py')
+2026-02-12 01:53:37,974 - INFO - Upload completed successfully: spice.py
+2026-02-12 01:53:37,974 - INFO - Starting upload: lib/pygments/lexers/special.py -> /home/kevin/test/lib/pygments/lexers/special.py (3585 bytes)
+2026-02-12 01:53:37,976 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/special.py', 'wb')
+2026-02-12 01:53:37,977 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/special.py', 'wb') -> 00000000
+2026-02-12 01:53:37,977 - DEBUG - Progress: 3585/3585 bytes
+2026-02-12 01:53:37,977 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,978 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/special.py')
+2026-02-12 01:53:37,981 - INFO - Upload completed successfully: special.py
+2026-02-12 01:53:37,981 - INFO - Starting upload: lib/pygments/lexers/sophia.py -> /home/kevin/test/lib/pygments/lexers/sophia.py (3376 bytes)
+2026-02-12 01:53:37,983 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sophia.py', 'wb')
+2026-02-12 01:53:37,984 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sophia.py', 'wb') -> 00000000
+2026-02-12 01:53:37,984 - DEBUG - Progress: 3376/3376 bytes
+2026-02-12 01:53:37,984 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,985 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/sophia.py')
+2026-02-12 01:53:37,988 - INFO - Upload completed successfully: sophia.py
+2026-02-12 01:53:37,988 - INFO - Starting upload: lib/pygments/lexers/soong.py -> /home/kevin/test/lib/pygments/lexers/soong.py (2339 bytes)
+2026-02-12 01:53:37,989 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/soong.py', 'wb')
+2026-02-12 01:53:37,990 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/soong.py', 'wb') -> 00000000
+2026-02-12 01:53:37,990 - DEBUG - Progress: 2339/2339 bytes
+2026-02-12 01:53:37,990 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,991 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/soong.py')
+2026-02-12 01:53:37,994 - INFO - Upload completed successfully: soong.py
+2026-02-12 01:53:37,994 - INFO - Starting upload: lib/pygments/lexers/solidity.py -> /home/kevin/test/lib/pygments/lexers/solidity.py (3163 bytes)
+2026-02-12 01:53:37,996 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/solidity.py', 'wb')
+2026-02-12 01:53:37,996 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/solidity.py', 'wb') -> 00000000
+2026-02-12 01:53:37,996 - DEBUG - Progress: 3163/3163 bytes
+2026-02-12 01:53:37,997 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:37,998 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/solidity.py')
+2026-02-12 01:53:38,000 - INFO - Upload completed successfully: solidity.py
+2026-02-12 01:53:38,000 - INFO - Starting upload: lib/pygments/lexers/snobol.py -> /home/kevin/test/lib/pygments/lexers/snobol.py (2778 bytes)
+2026-02-12 01:53:38,002 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/snobol.py', 'wb')
+2026-02-12 01:53:38,003 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/snobol.py', 'wb') -> 00000000
+2026-02-12 01:53:38,003 - DEBUG - Progress: 2778/2778 bytes
+2026-02-12 01:53:38,003 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,004 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/snobol.py')
+2026-02-12 01:53:38,007 - INFO - Upload completed successfully: snobol.py
+2026-02-12 01:53:38,007 - INFO - Starting upload: lib/pygments/lexers/smv.py -> /home/kevin/test/lib/pygments/lexers/smv.py (2805 bytes)
+2026-02-12 01:53:38,009 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/smv.py', 'wb')
+2026-02-12 01:53:38,010 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/smv.py', 'wb') -> 00000000
+2026-02-12 01:53:38,010 - DEBUG - Progress: 2805/2805 bytes
+2026-02-12 01:53:38,010 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,011 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/smv.py')
+2026-02-12 01:53:38,013 - INFO - Upload completed successfully: smv.py
+2026-02-12 01:53:38,014 - INFO - Starting upload: lib/pygments/lexers/smithy.py -> /home/kevin/test/lib/pygments/lexers/smithy.py (2659 bytes)
+2026-02-12 01:53:38,015 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/smithy.py', 'wb')
+2026-02-12 01:53:38,016 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/smithy.py', 'wb') -> 00000000
+2026-02-12 01:53:38,016 - DEBUG - Progress: 2659/2659 bytes
+2026-02-12 01:53:38,016 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,018 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/smithy.py')
+2026-02-12 01:53:38,020 - INFO - Upload completed successfully: smithy.py
+2026-02-12 01:53:38,020 - INFO - Starting upload: lib/pygments/lexers/smalltalk.py -> /home/kevin/test/lib/pygments/lexers/smalltalk.py (7204 bytes)
+2026-02-12 01:53:38,022 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/smalltalk.py', 'wb')
+2026-02-12 01:53:38,023 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/smalltalk.py', 'wb') -> 00000000
+2026-02-12 01:53:38,023 - DEBUG - Progress: 7204/7204 bytes
+2026-02-12 01:53:38,023 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,025 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/smalltalk.py')
+2026-02-12 01:53:38,028 - INFO - Upload completed successfully: smalltalk.py
+2026-02-12 01:53:38,028 - INFO - Starting upload: lib/pygments/lexers/slash.py -> /home/kevin/test/lib/pygments/lexers/slash.py (8484 bytes)
+2026-02-12 01:53:38,030 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/slash.py', 'wb')
+2026-02-12 01:53:38,030 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/slash.py', 'wb') -> 00000000
+2026-02-12 01:53:38,030 - DEBUG - Progress: 8484/8484 bytes
+2026-02-12 01:53:38,031 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,032 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/slash.py')
+2026-02-12 01:53:38,035 - INFO - Upload completed successfully: slash.py
+2026-02-12 01:53:38,035 - INFO - Starting upload: lib/pygments/lexers/sieve.py -> /home/kevin/test/lib/pygments/lexers/sieve.py (2514 bytes)
+2026-02-12 01:53:38,037 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sieve.py', 'wb')
+2026-02-12 01:53:38,038 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sieve.py', 'wb') -> 00000000
+2026-02-12 01:53:38,038 - DEBUG - Progress: 2514/2514 bytes
+2026-02-12 01:53:38,038 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,039 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/sieve.py')
+2026-02-12 01:53:38,041 - INFO - Upload completed successfully: sieve.py
+2026-02-12 01:53:38,041 - INFO - Starting upload: lib/pygments/lexers/shell.py -> /home/kevin/test/lib/pygments/lexers/shell.py (36381 bytes)
+2026-02-12 01:53:38,043 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/shell.py', 'wb')
+2026-02-12 01:53:38,044 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/shell.py', 'wb') -> 00000000
+2026-02-12 01:53:38,044 - DEBUG - Progress: 32768/36381 bytes
+2026-02-12 01:53:38,045 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,049 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/shell.py')
+2026-02-12 01:53:38,051 - INFO - Upload completed successfully: shell.py
+2026-02-12 01:53:38,051 - INFO - Starting upload: lib/pygments/lexers/sgf.py -> /home/kevin/test/lib/pygments/lexers/sgf.py (1985 bytes)
+2026-02-12 01:53:38,052 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sgf.py', 'wb')
+2026-02-12 01:53:38,053 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sgf.py', 'wb') -> 00000000
+2026-02-12 01:53:38,053 - DEBUG - Progress: 1985/1985 bytes
+2026-02-12 01:53:38,053 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,055 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/sgf.py')
+2026-02-12 01:53:38,056 - INFO - Upload completed successfully: sgf.py
+2026-02-12 01:53:38,056 - INFO - Starting upload: lib/pygments/lexers/scripting.py -> /home/kevin/test/lib/pygments/lexers/scripting.py (81814 bytes)
+2026-02-12 01:53:38,057 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/scripting.py', 'wb')
+2026-02-12 01:53:38,058 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/scripting.py', 'wb') -> 00000000
+2026-02-12 01:53:38,058 - DEBUG - Progress: 65536/81814 bytes
+2026-02-12 01:53:38,059 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,068 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/scripting.py')
+2026-02-12 01:53:38,070 - INFO - Upload completed successfully: scripting.py
+2026-02-12 01:53:38,070 - INFO - Starting upload: lib/pygments/lexers/scdoc.py -> /home/kevin/test/lib/pygments/lexers/scdoc.py (2524 bytes)
+2026-02-12 01:53:38,071 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/scdoc.py', 'wb')
+2026-02-12 01:53:38,072 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/scdoc.py', 'wb') -> 00000000
+2026-02-12 01:53:38,072 - DEBUG - Progress: 2524/2524 bytes
+2026-02-12 01:53:38,072 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,074 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/scdoc.py')
+2026-02-12 01:53:38,076 - INFO - Upload completed successfully: scdoc.py
+2026-02-12 01:53:38,076 - INFO - Starting upload: lib/pygments/lexers/savi.py -> /home/kevin/test/lib/pygments/lexers/savi.py (4878 bytes)
+2026-02-12 01:53:38,077 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/savi.py', 'wb')
+2026-02-12 01:53:38,078 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/savi.py', 'wb') -> 00000000
+2026-02-12 01:53:38,078 - DEBUG - Progress: 4878/4878 bytes
+2026-02-12 01:53:38,078 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,080 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/savi.py')
+2026-02-12 01:53:38,082 - INFO - Upload completed successfully: savi.py
+2026-02-12 01:53:38,082 - INFO - Starting upload: lib/pygments/lexers/sas.py -> /home/kevin/test/lib/pygments/lexers/sas.py (9456 bytes)
+2026-02-12 01:53:38,083 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sas.py', 'wb')
+2026-02-12 01:53:38,084 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/sas.py', 'wb') -> 00000000
+2026-02-12 01:53:38,084 - DEBUG - Progress: 9456/9456 bytes
+2026-02-12 01:53:38,084 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,086 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/sas.py')
+2026-02-12 01:53:38,088 - INFO - Upload completed successfully: sas.py
+2026-02-12 01:53:38,088 - INFO - Starting upload: lib/pygments/lexers/rust.py -> /home/kevin/test/lib/pygments/lexers/rust.py (8260 bytes)
+2026-02-12 01:53:38,090 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rust.py', 'wb')
+2026-02-12 01:53:38,091 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rust.py', 'wb') -> 00000000
+2026-02-12 01:53:38,091 - DEBUG - Progress: 8260/8260 bytes
+2026-02-12 01:53:38,091 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,092 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/rust.py')
+2026-02-12 01:53:38,094 - INFO - Upload completed successfully: rust.py
+2026-02-12 01:53:38,094 - INFO - Starting upload: lib/pygments/lexers/ruby.py -> /home/kevin/test/lib/pygments/lexers/ruby.py (22753 bytes)
+2026-02-12 01:53:38,096 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ruby.py', 'wb')
+2026-02-12 01:53:38,097 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ruby.py', 'wb') -> 00000000
+2026-02-12 01:53:38,097 - DEBUG - Progress: 22753/22753 bytes
+2026-02-12 01:53:38,097 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,100 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ruby.py')
+2026-02-12 01:53:38,102 - INFO - Upload completed successfully: ruby.py
+2026-02-12 01:53:38,102 - INFO - Starting upload: lib/pygments/lexers/robotframework.py -> /home/kevin/test/lib/pygments/lexers/robotframework.py (18448 bytes)
+2026-02-12 01:53:38,104 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/robotframework.py', 'wb')
+2026-02-12 01:53:38,104 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/robotframework.py', 'wb') -> 00000000
+2026-02-12 01:53:38,105 - DEBUG - Progress: 18448/18448 bytes
+2026-02-12 01:53:38,105 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,107 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/robotframework.py')
+2026-02-12 01:53:38,109 - INFO - Upload completed successfully: robotframework.py
+2026-02-12 01:53:38,109 - INFO - Starting upload: lib/pygments/lexers/roboconf.py -> /home/kevin/test/lib/pygments/lexers/roboconf.py (2074 bytes)
+2026-02-12 01:53:38,111 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/roboconf.py', 'wb')
+2026-02-12 01:53:38,112 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/roboconf.py', 'wb') -> 00000000
+2026-02-12 01:53:38,112 - DEBUG - Progress: 2074/2074 bytes
+2026-02-12 01:53:38,112 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,113 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/roboconf.py')
+2026-02-12 01:53:38,115 - INFO - Upload completed successfully: roboconf.py
+2026-02-12 01:53:38,115 - INFO - Starting upload: lib/pygments/lexers/rnc.py -> /home/kevin/test/lib/pygments/lexers/rnc.py (1972 bytes)
+2026-02-12 01:53:38,117 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rnc.py', 'wb')
+2026-02-12 01:53:38,118 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rnc.py', 'wb') -> 00000000
+2026-02-12 01:53:38,118 - DEBUG - Progress: 1972/1972 bytes
+2026-02-12 01:53:38,118 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,119 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/rnc.py')
+2026-02-12 01:53:38,121 - INFO - Upload completed successfully: rnc.py
+2026-02-12 01:53:38,121 - INFO - Starting upload: lib/pygments/lexers/rita.py -> /home/kevin/test/lib/pygments/lexers/rita.py (1127 bytes)
+2026-02-12 01:53:38,122 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rita.py', 'wb')
+2026-02-12 01:53:38,123 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rita.py', 'wb') -> 00000000
+2026-02-12 01:53:38,123 - DEBUG - Progress: 1127/1127 bytes
+2026-02-12 01:53:38,123 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,124 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/rita.py')
+2026-02-12 01:53:38,125 - INFO - Upload completed successfully: rita.py
+2026-02-12 01:53:38,125 - INFO - Starting upload: lib/pygments/lexers/ride.py -> /home/kevin/test/lib/pygments/lexers/ride.py (5035 bytes)
+2026-02-12 01:53:38,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ride.py', 'wb')
+2026-02-12 01:53:38,127 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ride.py', 'wb') -> 00000000
+2026-02-12 01:53:38,127 - DEBUG - Progress: 5035/5035 bytes
+2026-02-12 01:53:38,128 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,129 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ride.py')
+2026-02-12 01:53:38,131 - INFO - Upload completed successfully: ride.py
+2026-02-12 01:53:38,131 - INFO - Starting upload: lib/pygments/lexers/resource.py -> /home/kevin/test/lib/pygments/lexers/resource.py (2927 bytes)
+2026-02-12 01:53:38,133 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/resource.py', 'wb')
+2026-02-12 01:53:38,134 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/resource.py', 'wb') -> 00000000
+2026-02-12 01:53:38,134 - DEBUG - Progress: 2927/2927 bytes
+2026-02-12 01:53:38,134 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,135 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/resource.py')
+2026-02-12 01:53:38,137 - INFO - Upload completed successfully: resource.py
+2026-02-12 01:53:38,137 - INFO - Starting upload: lib/pygments/lexers/rego.py -> /home/kevin/test/lib/pygments/lexers/rego.py (1748 bytes)
+2026-02-12 01:53:38,139 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rego.py', 'wb')
+2026-02-12 01:53:38,139 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rego.py', 'wb') -> 00000000
+2026-02-12 01:53:38,139 - DEBUG - Progress: 1748/1748 bytes
+2026-02-12 01:53:38,139 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,140 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/rego.py')
+2026-02-12 01:53:38,142 - INFO - Upload completed successfully: rego.py
+2026-02-12 01:53:38,142 - INFO - Starting upload: lib/pygments/lexers/rebol.py -> /home/kevin/test/lib/pygments/lexers/rebol.py (18259 bytes)
+2026-02-12 01:53:38,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rebol.py', 'wb')
+2026-02-12 01:53:38,144 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rebol.py', 'wb') -> 00000000
+2026-02-12 01:53:38,145 - DEBUG - Progress: 18259/18259 bytes
+2026-02-12 01:53:38,145 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,147 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/rebol.py')
+2026-02-12 01:53:38,149 - INFO - Upload completed successfully: rebol.py
+2026-02-12 01:53:38,149 - INFO - Starting upload: lib/pygments/lexers/rdf.py -> /home/kevin/test/lib/pygments/lexers/rdf.py (16056 bytes)
+2026-02-12 01:53:38,151 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rdf.py', 'wb')
+2026-02-12 01:53:38,151 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/rdf.py', 'wb') -> 00000000
+2026-02-12 01:53:38,151 - DEBUG - Progress: 16056/16056 bytes
+2026-02-12 01:53:38,151 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,154 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/rdf.py')
+2026-02-12 01:53:38,156 - INFO - Upload completed successfully: rdf.py
+2026-02-12 01:53:38,156 - INFO - Starting upload: lib/pygments/lexers/r.py -> /home/kevin/test/lib/pygments/lexers/r.py (6474 bytes)
+2026-02-12 01:53:38,157 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/r.py', 'wb')
+2026-02-12 01:53:38,158 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/r.py', 'wb') -> 00000000
+2026-02-12 01:53:38,158 - DEBUG - Progress: 6474/6474 bytes
+2026-02-12 01:53:38,158 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,160 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/r.py')
+2026-02-12 01:53:38,161 - INFO - Upload completed successfully: r.py
+2026-02-12 01:53:38,161 - INFO - Starting upload: lib/pygments/lexers/qvt.py -> /home/kevin/test/lib/pygments/lexers/qvt.py (6103 bytes)
+2026-02-12 01:53:38,163 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/qvt.py', 'wb')
+2026-02-12 01:53:38,163 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/qvt.py', 'wb') -> 00000000
+2026-02-12 01:53:38,163 - DEBUG - Progress: 6103/6103 bytes
+2026-02-12 01:53:38,163 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,165 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/qvt.py')
+2026-02-12 01:53:38,166 - INFO - Upload completed successfully: qvt.py
+2026-02-12 01:53:38,166 - INFO - Starting upload: lib/pygments/lexers/qlik.py -> /home/kevin/test/lib/pygments/lexers/qlik.py (3693 bytes)
+2026-02-12 01:53:38,168 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/qlik.py', 'wb')
+2026-02-12 01:53:38,168 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/qlik.py', 'wb') -> 00000000
+2026-02-12 01:53:38,169 - DEBUG - Progress: 3693/3693 bytes
+2026-02-12 01:53:38,169 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,170 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/qlik.py')
+2026-02-12 01:53:38,172 - INFO - Upload completed successfully: qlik.py
+2026-02-12 01:53:38,172 - INFO - Starting upload: lib/pygments/lexers/q.py -> /home/kevin/test/lib/pygments/lexers/q.py (6936 bytes)
+2026-02-12 01:53:38,174 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/q.py', 'wb')
+2026-02-12 01:53:38,175 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/q.py', 'wb') -> 00000000
+2026-02-12 01:53:38,175 - DEBUG - Progress: 6936/6936 bytes
+2026-02-12 01:53:38,175 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,177 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/q.py')
+2026-02-12 01:53:38,189 - INFO - Upload completed successfully: q.py
+2026-02-12 01:53:38,189 - INFO - Starting upload: lib/pygments/lexers/python.py -> /home/kevin/test/lib/pygments/lexers/python.py (53805 bytes)
+2026-02-12 01:53:38,193 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/python.py', 'wb')
+2026-02-12 01:53:38,194 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/python.py', 'wb') -> 00000000
+2026-02-12 01:53:38,194 - DEBUG - Progress: 32768/53805 bytes
+2026-02-12 01:53:38,194 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,202 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/python.py')
+2026-02-12 01:53:38,204 - INFO - Upload completed successfully: python.py
+2026-02-12 01:53:38,204 - INFO - Starting upload: lib/pygments/lexers/ptx.py -> /home/kevin/test/lib/pygments/lexers/ptx.py (4501 bytes)
+2026-02-12 01:53:38,206 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ptx.py', 'wb')
+2026-02-12 01:53:38,207 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ptx.py', 'wb') -> 00000000
+2026-02-12 01:53:38,207 - DEBUG - Progress: 4501/4501 bytes
+2026-02-12 01:53:38,207 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,209 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ptx.py')
+2026-02-12 01:53:38,211 - INFO - Upload completed successfully: ptx.py
+2026-02-12 01:53:38,212 - INFO - Starting upload: lib/pygments/lexers/prql.py -> /home/kevin/test/lib/pygments/lexers/prql.py (8747 bytes)
+2026-02-12 01:53:38,214 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/prql.py', 'wb')
+2026-02-12 01:53:38,215 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/prql.py', 'wb') -> 00000000
+2026-02-12 01:53:38,215 - DEBUG - Progress: 8747/8747 bytes
+2026-02-12 01:53:38,215 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,217 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/prql.py')
+2026-02-12 01:53:38,219 - INFO - Upload completed successfully: prql.py
+2026-02-12 01:53:38,219 - INFO - Starting upload: lib/pygments/lexers/promql.py -> /home/kevin/test/lib/pygments/lexers/promql.py (4738 bytes)
+2026-02-12 01:53:38,221 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/promql.py', 'wb')
+2026-02-12 01:53:38,222 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/promql.py', 'wb') -> 00000000
+2026-02-12 01:53:38,222 - DEBUG - Progress: 4738/4738 bytes
+2026-02-12 01:53:38,222 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,223 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/promql.py')
+2026-02-12 01:53:38,225 - INFO - Upload completed successfully: promql.py
+2026-02-12 01:53:38,225 - INFO - Starting upload: lib/pygments/lexers/prolog.py -> /home/kevin/test/lib/pygments/lexers/prolog.py (12866 bytes)
+2026-02-12 01:53:38,228 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/prolog.py', 'wb')
+2026-02-12 01:53:38,228 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/prolog.py', 'wb') -> 00000000
+2026-02-12 01:53:38,229 - DEBUG - Progress: 12866/12866 bytes
+2026-02-12 01:53:38,229 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,231 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/prolog.py')
+2026-02-12 01:53:38,233 - INFO - Upload completed successfully: prolog.py
+2026-02-12 01:53:38,233 - INFO - Starting upload: lib/pygments/lexers/procfile.py -> /home/kevin/test/lib/pygments/lexers/procfile.py (1155 bytes)
+2026-02-12 01:53:38,235 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/procfile.py', 'wb')
+2026-02-12 01:53:38,236 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/procfile.py', 'wb') -> 00000000
+2026-02-12 01:53:38,236 - DEBUG - Progress: 1155/1155 bytes
+2026-02-12 01:53:38,237 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,237 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/procfile.py')
+2026-02-12 01:53:38,239 - INFO - Upload completed successfully: procfile.py
+2026-02-12 01:53:38,240 - INFO - Starting upload: lib/pygments/lexers/praat.py -> /home/kevin/test/lib/pygments/lexers/praat.py (12676 bytes)
+2026-02-12 01:53:38,242 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/praat.py', 'wb')
+2026-02-12 01:53:38,242 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/praat.py', 'wb') -> 00000000
+2026-02-12 01:53:38,243 - DEBUG - Progress: 12676/12676 bytes
+2026-02-12 01:53:38,243 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,245 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/praat.py')
+2026-02-12 01:53:38,247 - INFO - Upload completed successfully: praat.py
+2026-02-12 01:53:38,247 - INFO - Starting upload: lib/pygments/lexers/pony.py -> /home/kevin/test/lib/pygments/lexers/pony.py (3279 bytes)
+2026-02-12 01:53:38,249 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pony.py', 'wb')
+2026-02-12 01:53:38,250 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pony.py', 'wb') -> 00000000
+2026-02-12 01:53:38,251 - DEBUG - Progress: 3279/3279 bytes
+2026-02-12 01:53:38,251 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,252 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/pony.py')
+2026-02-12 01:53:38,254 - INFO - Upload completed successfully: pony.py
+2026-02-12 01:53:38,255 - INFO - Starting upload: lib/pygments/lexers/pointless.py -> /home/kevin/test/lib/pygments/lexers/pointless.py (1974 bytes)
+2026-02-12 01:53:38,257 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pointless.py', 'wb')
+2026-02-12 01:53:38,257 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pointless.py', 'wb') -> 00000000
+2026-02-12 01:53:38,258 - DEBUG - Progress: 1974/1974 bytes
+2026-02-12 01:53:38,258 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,259 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/pointless.py')
+2026-02-12 01:53:38,261 - INFO - Upload completed successfully: pointless.py
+2026-02-12 01:53:38,262 - INFO - Starting upload: lib/pygments/lexers/php.py -> /home/kevin/test/lib/pygments/lexers/php.py (13061 bytes)
+2026-02-12 01:53:38,264 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/php.py', 'wb')
+2026-02-12 01:53:38,264 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/php.py', 'wb') -> 00000000
+2026-02-12 01:53:38,265 - DEBUG - Progress: 13061/13061 bytes
+2026-02-12 01:53:38,265 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,267 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/php.py')
+2026-02-12 01:53:38,269 - INFO - Upload completed successfully: php.py
+2026-02-12 01:53:38,269 - INFO - Starting upload: lib/pygments/lexers/phix.py -> /home/kevin/test/lib/pygments/lexers/phix.py (23249 bytes)
+2026-02-12 01:53:38,271 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/phix.py', 'wb')
+2026-02-12 01:53:38,272 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/phix.py', 'wb') -> 00000000
+2026-02-12 01:53:38,272 - DEBUG - Progress: 23249/23249 bytes
+2026-02-12 01:53:38,272 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,276 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/phix.py')
+2026-02-12 01:53:38,278 - INFO - Upload completed successfully: phix.py
+2026-02-12 01:53:38,278 - INFO - Starting upload: lib/pygments/lexers/perl.py -> /home/kevin/test/lib/pygments/lexers/perl.py (39192 bytes)
+2026-02-12 01:53:38,280 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/perl.py', 'wb')
+2026-02-12 01:53:38,281 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/perl.py', 'wb') -> 00000000
+2026-02-12 01:53:38,281 - DEBUG - Progress: 32768/39192 bytes
+2026-02-12 01:53:38,282 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,286 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/perl.py')
+2026-02-12 01:53:38,289 - INFO - Upload completed successfully: perl.py
+2026-02-12 01:53:38,289 - INFO - Starting upload: lib/pygments/lexers/pddl.py -> /home/kevin/test/lib/pygments/lexers/pddl.py (2989 bytes)
+2026-02-12 01:53:38,291 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pddl.py', 'wb')
+2026-02-12 01:53:38,292 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pddl.py', 'wb') -> 00000000
+2026-02-12 01:53:38,292 - DEBUG - Progress: 2989/2989 bytes
+2026-02-12 01:53:38,292 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,293 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/pddl.py')
+2026-02-12 01:53:38,296 - INFO - Upload completed successfully: pddl.py
+2026-02-12 01:53:38,296 - INFO - Starting upload: lib/pygments/lexers/pawn.py -> /home/kevin/test/lib/pygments/lexers/pawn.py (8253 bytes)
+2026-02-12 01:53:38,298 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pawn.py', 'wb')
+2026-02-12 01:53:38,299 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pawn.py', 'wb') -> 00000000
+2026-02-12 01:53:38,299 - DEBUG - Progress: 8253/8253 bytes
+2026-02-12 01:53:38,300 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,301 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/pawn.py')
+2026-02-12 01:53:38,303 - INFO - Upload completed successfully: pawn.py
+2026-02-12 01:53:38,303 - INFO - Starting upload: lib/pygments/lexers/pascal.py -> /home/kevin/test/lib/pygments/lexers/pascal.py (30989 bytes)
+2026-02-12 01:53:38,305 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pascal.py', 'wb')
+2026-02-12 01:53:38,306 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/pascal.py', 'wb') -> 00000000
+2026-02-12 01:53:38,307 - DEBUG - Progress: 30989/30989 bytes
+2026-02-12 01:53:38,307 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,311 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/pascal.py')
+2026-02-12 01:53:38,313 - INFO - Upload completed successfully: pascal.py
+2026-02-12 01:53:38,313 - INFO - Starting upload: lib/pygments/lexers/parsers.py -> /home/kevin/test/lib/pygments/lexers/parsers.py (26596 bytes)
+2026-02-12 01:53:38,315 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/parsers.py', 'wb')
+2026-02-12 01:53:38,315 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/parsers.py', 'wb') -> 00000000
+2026-02-12 01:53:38,316 - DEBUG - Progress: 26596/26596 bytes
+2026-02-12 01:53:38,316 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,320 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/parsers.py')
+2026-02-12 01:53:38,322 - INFO - Upload completed successfully: parsers.py
+2026-02-12 01:53:38,323 - INFO - Starting upload: lib/pygments/lexers/parasail.py -> /home/kevin/test/lib/pygments/lexers/parasail.py (2719 bytes)
+2026-02-12 01:53:38,325 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/parasail.py', 'wb')
+2026-02-12 01:53:38,325 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/parasail.py', 'wb') -> 00000000
+2026-02-12 01:53:38,325 - DEBUG - Progress: 2719/2719 bytes
+2026-02-12 01:53:38,326 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,327 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/parasail.py')
+2026-02-12 01:53:38,329 - INFO - Upload completed successfully: parasail.py
+2026-02-12 01:53:38,329 - INFO - Starting upload: lib/pygments/lexers/other.py -> /home/kevin/test/lib/pygments/lexers/other.py (1763 bytes)
+2026-02-12 01:53:38,331 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/other.py', 'wb')
+2026-02-12 01:53:38,332 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/other.py', 'wb') -> 00000000
+2026-02-12 01:53:38,332 - DEBUG - Progress: 1763/1763 bytes
+2026-02-12 01:53:38,332 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,333 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/other.py')
+2026-02-12 01:53:38,335 - INFO - Upload completed successfully: other.py
+2026-02-12 01:53:38,336 - INFO - Starting upload: lib/pygments/lexers/openscad.py -> /home/kevin/test/lib/pygments/lexers/openscad.py (3700 bytes)
+2026-02-12 01:53:38,337 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/openscad.py', 'wb')
+2026-02-12 01:53:38,338 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/openscad.py', 'wb') -> 00000000
+2026-02-12 01:53:38,338 - DEBUG - Progress: 3700/3700 bytes
+2026-02-12 01:53:38,338 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,340 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/openscad.py')
+2026-02-12 01:53:38,342 - INFO - Upload completed successfully: openscad.py
+2026-02-12 01:53:38,342 - INFO - Starting upload: lib/pygments/lexers/ooc.py -> /home/kevin/test/lib/pygments/lexers/ooc.py (3002 bytes)
+2026-02-12 01:53:38,344 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ooc.py', 'wb')
+2026-02-12 01:53:38,345 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ooc.py', 'wb') -> 00000000
+2026-02-12 01:53:38,345 - DEBUG - Progress: 3002/3002 bytes
+2026-02-12 01:53:38,345 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,346 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ooc.py')
+2026-02-12 01:53:38,348 - INFO - Upload completed successfully: ooc.py
+2026-02-12 01:53:38,349 - INFO - Starting upload: lib/pygments/lexers/objective.py -> /home/kevin/test/lib/pygments/lexers/objective.py (23297 bytes)
+2026-02-12 01:53:38,351 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/objective.py', 'wb')
+2026-02-12 01:53:38,351 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/objective.py', 'wb') -> 00000000
+2026-02-12 01:53:38,351 - DEBUG - Progress: 23297/23297 bytes
+2026-02-12 01:53:38,351 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,354 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/objective.py')
+2026-02-12 01:53:38,357 - INFO - Upload completed successfully: objective.py
+2026-02-12 01:53:38,357 - INFO - Starting upload: lib/pygments/lexers/oberon.py -> /home/kevin/test/lib/pygments/lexers/oberon.py (4210 bytes)
+2026-02-12 01:53:38,359 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/oberon.py', 'wb')
+2026-02-12 01:53:38,359 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/oberon.py', 'wb') -> 00000000
+2026-02-12 01:53:38,360 - DEBUG - Progress: 4210/4210 bytes
+2026-02-12 01:53:38,360 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,361 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/oberon.py')
+2026-02-12 01:53:38,363 - INFO - Upload completed successfully: oberon.py
+2026-02-12 01:53:38,363 - INFO - Starting upload: lib/pygments/lexers/numbair.py -> /home/kevin/test/lib/pygments/lexers/numbair.py (1758 bytes)
+2026-02-12 01:53:38,365 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/numbair.py', 'wb')
+2026-02-12 01:53:38,366 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/numbair.py', 'wb') -> 00000000
+2026-02-12 01:53:38,366 - DEBUG - Progress: 1758/1758 bytes
+2026-02-12 01:53:38,366 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,368 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/numbair.py')
+2026-02-12 01:53:38,370 - INFO - Upload completed successfully: numbair.py
+2026-02-12 01:53:38,370 - INFO - Starting upload: lib/pygments/lexers/nix.py -> /home/kevin/test/lib/pygments/lexers/nix.py (4421 bytes)
+2026-02-12 01:53:38,372 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/nix.py', 'wb')
+2026-02-12 01:53:38,372 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/nix.py', 'wb') -> 00000000
+2026-02-12 01:53:38,372 - DEBUG - Progress: 4421/4421 bytes
+2026-02-12 01:53:38,372 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,374 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/nix.py')
+2026-02-12 01:53:38,376 - INFO - Upload completed successfully: nix.py
+2026-02-12 01:53:38,376 - INFO - Starting upload: lib/pygments/lexers/nit.py -> /home/kevin/test/lib/pygments/lexers/nit.py (2725 bytes)
+2026-02-12 01:53:38,378 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/nit.py', 'wb')
+2026-02-12 01:53:38,379 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/nit.py', 'wb') -> 00000000
+2026-02-12 01:53:38,379 - DEBUG - Progress: 2725/2725 bytes
+2026-02-12 01:53:38,379 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,380 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/nit.py')
+2026-02-12 01:53:38,383 - INFO - Upload completed successfully: nit.py
+2026-02-12 01:53:38,383 - INFO - Starting upload: lib/pygments/lexers/nimrod.py -> /home/kevin/test/lib/pygments/lexers/nimrod.py (6413 bytes)
+2026-02-12 01:53:38,385 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/nimrod.py', 'wb')
+2026-02-12 01:53:38,385 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/nimrod.py', 'wb') -> 00000000
+2026-02-12 01:53:38,386 - DEBUG - Progress: 6413/6413 bytes
+2026-02-12 01:53:38,386 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,387 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/nimrod.py')
+2026-02-12 01:53:38,389 - INFO - Upload completed successfully: nimrod.py
+2026-02-12 01:53:38,389 - INFO - Starting upload: lib/pygments/lexers/ncl.py -> /home/kevin/test/lib/pygments/lexers/ncl.py (63999 bytes)
+2026-02-12 01:53:38,392 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ncl.py', 'wb')
+2026-02-12 01:53:38,393 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ncl.py', 'wb') -> 00000000
+2026-02-12 01:53:38,393 - DEBUG - Progress: 32768/63999 bytes
+2026-02-12 01:53:38,393 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,400 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ncl.py')
+2026-02-12 01:53:38,402 - INFO - Upload completed successfully: ncl.py
+2026-02-12 01:53:38,402 - INFO - Starting upload: lib/pygments/lexers/mosel.py -> /home/kevin/test/lib/pygments/lexers/mosel.py (9297 bytes)
+2026-02-12 01:53:38,403 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mosel.py', 'wb')
+2026-02-12 01:53:38,404 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mosel.py', 'wb') -> 00000000
+2026-02-12 01:53:38,404 - DEBUG - Progress: 9297/9297 bytes
+2026-02-12 01:53:38,404 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,406 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/mosel.py')
+2026-02-12 01:53:38,408 - INFO - Upload completed successfully: mosel.py
+2026-02-12 01:53:38,408 - INFO - Starting upload: lib/pygments/lexers/monte.py -> /home/kevin/test/lib/pygments/lexers/monte.py (6289 bytes)
+2026-02-12 01:53:38,410 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/monte.py', 'wb')
+2026-02-12 01:53:38,410 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/monte.py', 'wb') -> 00000000
+2026-02-12 01:53:38,411 - DEBUG - Progress: 6289/6289 bytes
+2026-02-12 01:53:38,411 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,412 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/monte.py')
+2026-02-12 01:53:38,414 - INFO - Upload completed successfully: monte.py
+2026-02-12 01:53:38,414 - INFO - Starting upload: lib/pygments/lexers/mojo.py -> /home/kevin/test/lib/pygments/lexers/mojo.py (24233 bytes)
+2026-02-12 01:53:38,415 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mojo.py', 'wb')
+2026-02-12 01:53:38,416 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mojo.py', 'wb') -> 00000000
+2026-02-12 01:53:38,416 - DEBUG - Progress: 24233/24233 bytes
+2026-02-12 01:53:38,416 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,419 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/mojo.py')
+2026-02-12 01:53:38,421 - INFO - Upload completed successfully: mojo.py
+2026-02-12 01:53:38,421 - INFO - Starting upload: lib/pygments/lexers/modula2.py -> /home/kevin/test/lib/pygments/lexers/modula2.py (53072 bytes)
+2026-02-12 01:53:38,422 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/modula2.py', 'wb')
+2026-02-12 01:53:38,423 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/modula2.py', 'wb') -> 00000000
+2026-02-12 01:53:38,423 - DEBUG - Progress: 32768/53072 bytes
+2026-02-12 01:53:38,423 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,429 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/modula2.py')
+2026-02-12 01:53:38,430 - INFO - Upload completed successfully: modula2.py
+2026-02-12 01:53:38,430 - INFO - Starting upload: lib/pygments/lexers/modeling.py -> /home/kevin/test/lib/pygments/lexers/modeling.py (13683 bytes)
+2026-02-12 01:53:38,432 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/modeling.py', 'wb')
+2026-02-12 01:53:38,433 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/modeling.py', 'wb') -> 00000000
+2026-02-12 01:53:38,433 - DEBUG - Progress: 13683/13683 bytes
+2026-02-12 01:53:38,433 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,435 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/modeling.py')
+2026-02-12 01:53:38,437 - INFO - Upload completed successfully: modeling.py
+2026-02-12 01:53:38,437 - INFO - Starting upload: lib/pygments/lexers/ml.py -> /home/kevin/test/lib/pygments/lexers/ml.py (35390 bytes)
+2026-02-12 01:53:38,439 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ml.py', 'wb')
+2026-02-12 01:53:38,440 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ml.py', 'wb') -> 00000000
+2026-02-12 01:53:38,440 - DEBUG - Progress: 32768/35390 bytes
+2026-02-12 01:53:38,441 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,445 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ml.py')
+2026-02-12 01:53:38,447 - INFO - Upload completed successfully: ml.py
+2026-02-12 01:53:38,447 - INFO - Starting upload: lib/pygments/lexers/mips.py -> /home/kevin/test/lib/pygments/lexers/mips.py (4656 bytes)
+2026-02-12 01:53:38,449 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mips.py', 'wb')
+2026-02-12 01:53:38,450 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mips.py', 'wb') -> 00000000
+2026-02-12 01:53:38,450 - DEBUG - Progress: 4656/4656 bytes
+2026-02-12 01:53:38,450 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,451 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/mips.py')
+2026-02-12 01:53:38,454 - INFO - Upload completed successfully: mips.py
+2026-02-12 01:53:38,454 - INFO - Starting upload: lib/pygments/lexers/minecraft.py -> /home/kevin/test/lib/pygments/lexers/minecraft.py (13696 bytes)
+2026-02-12 01:53:38,456 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/minecraft.py', 'wb')
+2026-02-12 01:53:38,456 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/minecraft.py', 'wb') -> 00000000
+2026-02-12 01:53:38,457 - DEBUG - Progress: 13696/13696 bytes
+2026-02-12 01:53:38,457 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,461 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/minecraft.py')
+2026-02-12 01:53:38,463 - INFO - Upload completed successfully: minecraft.py
+2026-02-12 01:53:38,463 - INFO - Starting upload: lib/pygments/lexers/mime.py -> /home/kevin/test/lib/pygments/lexers/mime.py (7582 bytes)
+2026-02-12 01:53:38,465 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mime.py', 'wb')
+2026-02-12 01:53:38,466 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/mime.py', 'wb') -> 00000000
+2026-02-12 01:53:38,466 - DEBUG - Progress: 7582/7582 bytes
+2026-02-12 01:53:38,466 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,468 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/mime.py')
+2026-02-12 01:53:38,471 - INFO - Upload completed successfully: mime.py
+2026-02-12 01:53:38,471 - INFO - Starting upload: lib/pygments/lexers/meson.py -> /home/kevin/test/lib/pygments/lexers/meson.py (4336 bytes)
+2026-02-12 01:53:38,473 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/meson.py', 'wb')
+2026-02-12 01:53:38,474 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/meson.py', 'wb') -> 00000000
+2026-02-12 01:53:38,474 - DEBUG - Progress: 4336/4336 bytes
+2026-02-12 01:53:38,474 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,476 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/meson.py')
+2026-02-12 01:53:38,478 - INFO - Upload completed successfully: meson.py
+2026-02-12 01:53:38,478 - INFO - Starting upload: lib/pygments/lexers/maxima.py -> /home/kevin/test/lib/pygments/lexers/maxima.py (2715 bytes)
+2026-02-12 01:53:38,480 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/maxima.py', 'wb')
+2026-02-12 01:53:38,481 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/maxima.py', 'wb') -> 00000000
+2026-02-12 01:53:38,481 - DEBUG - Progress: 2715/2715 bytes
+2026-02-12 01:53:38,481 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,482 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/maxima.py')
+2026-02-12 01:53:38,484 - INFO - Upload completed successfully: maxima.py
+2026-02-12 01:53:38,485 - INFO - Starting upload: lib/pygments/lexers/matlab.py -> /home/kevin/test/lib/pygments/lexers/matlab.py (133027 bytes)
+2026-02-12 01:53:38,487 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/matlab.py', 'wb')
+2026-02-12 01:53:38,487 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/matlab.py', 'wb') -> 00000000
+2026-02-12 01:53:38,488 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,501 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/matlab.py')
+2026-02-12 01:53:38,503 - INFO - Upload completed successfully: matlab.py
+2026-02-12 01:53:38,503 - INFO - Starting upload: lib/pygments/lexers/math.py -> /home/kevin/test/lib/pygments/lexers/math.py (695 bytes)
+2026-02-12 01:53:38,505 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/math.py', 'wb')
+2026-02-12 01:53:38,506 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/math.py', 'wb') -> 00000000
+2026-02-12 01:53:38,506 - DEBUG - Progress: 695/695 bytes
+2026-02-12 01:53:38,506 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,507 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/math.py')
+2026-02-12 01:53:38,509 - INFO - Upload completed successfully: math.py
+2026-02-12 01:53:38,509 - INFO - Starting upload: lib/pygments/lexers/markup.py -> /home/kevin/test/lib/pygments/lexers/markup.py (65088 bytes)
+2026-02-12 01:53:38,511 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/markup.py', 'wb')
+2026-02-12 01:53:38,512 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/markup.py', 'wb') -> 00000000
+2026-02-12 01:53:38,512 - DEBUG - Progress: 32768/65088 bytes
+2026-02-12 01:53:38,512 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,520 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/markup.py')
+2026-02-12 01:53:38,522 - INFO - Upload completed successfully: markup.py
+2026-02-12 01:53:38,522 - INFO - Starting upload: lib/pygments/lexers/maple.py -> /home/kevin/test/lib/pygments/lexers/maple.py (7960 bytes)
+2026-02-12 01:53:38,524 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/maple.py', 'wb')
+2026-02-12 01:53:38,525 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/maple.py', 'wb') -> 00000000
+2026-02-12 01:53:38,525 - DEBUG - Progress: 7960/7960 bytes
+2026-02-12 01:53:38,525 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,527 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/maple.py')
+2026-02-12 01:53:38,529 - INFO - Upload completed successfully: maple.py
+2026-02-12 01:53:38,529 - INFO - Starting upload: lib/pygments/lexers/make.py -> /home/kevin/test/lib/pygments/lexers/make.py (7831 bytes)
+2026-02-12 01:53:38,531 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/make.py', 'wb')
+2026-02-12 01:53:38,532 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/make.py', 'wb') -> 00000000
+2026-02-12 01:53:38,532 - DEBUG - Progress: 7831/7831 bytes
+2026-02-12 01:53:38,532 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,534 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/make.py')
+2026-02-12 01:53:38,536 - INFO - Upload completed successfully: make.py
+2026-02-12 01:53:38,536 - INFO - Starting upload: lib/pygments/lexers/macaulay2.py -> /home/kevin/test/lib/pygments/lexers/macaulay2.py (33366 bytes)
+2026-02-12 01:53:38,538 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/macaulay2.py', 'wb')
+2026-02-12 01:53:38,539 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/macaulay2.py', 'wb') -> 00000000
+2026-02-12 01:53:38,539 - DEBUG - Progress: 32768/33366 bytes
+2026-02-12 01:53:38,539 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,543 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/macaulay2.py')
+2026-02-12 01:53:38,545 - INFO - Upload completed successfully: macaulay2.py
+2026-02-12 01:53:38,545 - INFO - Starting upload: lib/pygments/lexers/lisp.py -> /home/kevin/test/lib/pygments/lexers/lisp.py (157668 bytes)
+2026-02-12 01:53:38,547 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/lisp.py', 'wb')
+2026-02-12 01:53:38,547 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/lisp.py', 'wb') -> 00000000
+2026-02-12 01:53:38,548 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,565 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/lisp.py')
+2026-02-12 01:53:38,567 - INFO - Upload completed successfully: lisp.py
+2026-02-12 01:53:38,567 - INFO - Starting upload: lib/pygments/lexers/lilypond.py -> /home/kevin/test/lib/pygments/lexers/lilypond.py (9752 bytes)
+2026-02-12 01:53:38,569 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/lilypond.py', 'wb')
+2026-02-12 01:53:38,570 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/lilypond.py', 'wb') -> 00000000
+2026-02-12 01:53:38,570 - DEBUG - Progress: 9752/9752 bytes
+2026-02-12 01:53:38,570 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,572 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/lilypond.py')
+2026-02-12 01:53:38,574 - INFO - Upload completed successfully: lilypond.py
+2026-02-12 01:53:38,574 - INFO - Starting upload: lib/pygments/lexers/lean.py -> /home/kevin/test/lib/pygments/lexers/lean.py (8570 bytes)
+2026-02-12 01:53:38,576 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/lean.py', 'wb')
+2026-02-12 01:53:38,577 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/lean.py', 'wb') -> 00000000
+2026-02-12 01:53:38,577 - DEBUG - Progress: 8570/8570 bytes
+2026-02-12 01:53:38,577 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,579 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/lean.py')
+2026-02-12 01:53:38,581 - INFO - Upload completed successfully: lean.py
+2026-02-12 01:53:38,581 - INFO - Starting upload: lib/pygments/lexers/ldap.py -> /home/kevin/test/lib/pygments/lexers/ldap.py (6551 bytes)
+2026-02-12 01:53:38,583 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ldap.py', 'wb')
+2026-02-12 01:53:38,584 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ldap.py', 'wb') -> 00000000
+2026-02-12 01:53:38,584 - DEBUG - Progress: 6551/6551 bytes
+2026-02-12 01:53:38,584 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,585 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ldap.py')
+2026-02-12 01:53:38,587 - INFO - Upload completed successfully: ldap.py
+2026-02-12 01:53:38,587 - INFO - Starting upload: lib/pygments/lexers/kusto.py -> /home/kevin/test/lib/pygments/lexers/kusto.py (3477 bytes)
+2026-02-12 01:53:38,589 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/kusto.py', 'wb')
+2026-02-12 01:53:38,590 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/kusto.py', 'wb') -> 00000000
+2026-02-12 01:53:38,590 - DEBUG - Progress: 3477/3477 bytes
+2026-02-12 01:53:38,590 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,592 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/kusto.py')
+2026-02-12 01:53:38,594 - INFO - Upload completed successfully: kusto.py
+2026-02-12 01:53:38,594 - INFO - Starting upload: lib/pygments/lexers/kuin.py -> /home/kevin/test/lib/pygments/lexers/kuin.py (11405 bytes)
+2026-02-12 01:53:38,596 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/kuin.py', 'wb')
+2026-02-12 01:53:38,596 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/kuin.py', 'wb') -> 00000000
+2026-02-12 01:53:38,597 - DEBUG - Progress: 11405/11405 bytes
+2026-02-12 01:53:38,597 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,598 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/kuin.py')
+2026-02-12 01:53:38,600 - INFO - Upload completed successfully: kuin.py
+2026-02-12 01:53:38,601 - INFO - Starting upload: lib/pygments/lexers/jvm.py -> /home/kevin/test/lib/pygments/lexers/jvm.py (72667 bytes)
+2026-02-12 01:53:38,603 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jvm.py', 'wb')
+2026-02-12 01:53:38,603 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jvm.py', 'wb') -> 00000000
+2026-02-12 01:53:38,604 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,613 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/jvm.py')
+2026-02-12 01:53:38,615 - INFO - Upload completed successfully: jvm.py
+2026-02-12 01:53:38,616 - INFO - Starting upload: lib/pygments/lexers/julia.py -> /home/kevin/test/lib/pygments/lexers/julia.py (11710 bytes)
+2026-02-12 01:53:38,617 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/julia.py', 'wb')
+2026-02-12 01:53:38,618 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/julia.py', 'wb') -> 00000000
+2026-02-12 01:53:38,618 - DEBUG - Progress: 11710/11710 bytes
+2026-02-12 01:53:38,618 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,621 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/julia.py')
+2026-02-12 01:53:38,622 - INFO - Upload completed successfully: julia.py
+2026-02-12 01:53:38,623 - INFO - Starting upload: lib/pygments/lexers/jsx.py -> /home/kevin/test/lib/pygments/lexers/jsx.py (2693 bytes)
+2026-02-12 01:53:38,624 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jsx.py', 'wb')
+2026-02-12 01:53:38,625 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jsx.py', 'wb') -> 00000000
+2026-02-12 01:53:38,625 - DEBUG - Progress: 2693/2693 bytes
+2026-02-12 01:53:38,625 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,627 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/jsx.py')
+2026-02-12 01:53:38,628 - INFO - Upload completed successfully: jsx.py
+2026-02-12 01:53:38,628 - INFO - Starting upload: lib/pygments/lexers/jsonnet.py -> /home/kevin/test/lib/pygments/lexers/jsonnet.py (5636 bytes)
+2026-02-12 01:53:38,630 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jsonnet.py', 'wb')
+2026-02-12 01:53:38,631 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jsonnet.py', 'wb') -> 00000000
+2026-02-12 01:53:38,631 - DEBUG - Progress: 5636/5636 bytes
+2026-02-12 01:53:38,631 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,632 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/jsonnet.py')
+2026-02-12 01:53:38,634 - INFO - Upload completed successfully: jsonnet.py
+2026-02-12 01:53:38,634 - INFO - Starting upload: lib/pygments/lexers/json5.py -> /home/kevin/test/lib/pygments/lexers/json5.py (2502 bytes)
+2026-02-12 01:53:38,636 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/json5.py', 'wb')
+2026-02-12 01:53:38,637 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/json5.py', 'wb') -> 00000000
+2026-02-12 01:53:38,637 - DEBUG - Progress: 2502/2502 bytes
+2026-02-12 01:53:38,637 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,638 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/json5.py')
+2026-02-12 01:53:38,641 - INFO - Upload completed successfully: json5.py
+2026-02-12 01:53:38,641 - INFO - Starting upload: lib/pygments/lexers/jslt.py -> /home/kevin/test/lib/pygments/lexers/jslt.py (3700 bytes)
+2026-02-12 01:53:38,643 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jslt.py', 'wb')
+2026-02-12 01:53:38,643 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jslt.py', 'wb') -> 00000000
+2026-02-12 01:53:38,643 - DEBUG - Progress: 3700/3700 bytes
+2026-02-12 01:53:38,643 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,645 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/jslt.py')
+2026-02-12 01:53:38,647 - INFO - Upload completed successfully: jslt.py
+2026-02-12 01:53:38,647 - INFO - Starting upload: lib/pygments/lexers/jmespath.py -> /home/kevin/test/lib/pygments/lexers/jmespath.py (2082 bytes)
+2026-02-12 01:53:38,649 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jmespath.py', 'wb')
+2026-02-12 01:53:38,650 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/jmespath.py', 'wb') -> 00000000
+2026-02-12 01:53:38,650 - DEBUG - Progress: 2082/2082 bytes
+2026-02-12 01:53:38,650 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,651 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/jmespath.py')
+2026-02-12 01:53:38,653 - INFO - Upload completed successfully: jmespath.py
+2026-02-12 01:53:38,653 - INFO - Starting upload: lib/pygments/lexers/javascript.py -> /home/kevin/test/lib/pygments/lexers/javascript.py (63243 bytes)
+2026-02-12 01:53:38,654 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/javascript.py', 'wb')
+2026-02-12 01:53:38,655 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/javascript.py', 'wb') -> 00000000
+2026-02-12 01:53:38,655 - DEBUG - Progress: 32768/63243 bytes
+2026-02-12 01:53:38,655 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,664 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/javascript.py')
+2026-02-12 01:53:38,666 - INFO - Upload completed successfully: javascript.py
+2026-02-12 01:53:38,666 - INFO - Starting upload: lib/pygments/lexers/j.py -> /home/kevin/test/lib/pygments/lexers/j.py (4853 bytes)
+2026-02-12 01:53:38,667 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/j.py', 'wb')
+2026-02-12 01:53:38,668 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/j.py', 'wb') -> 00000000
+2026-02-12 01:53:38,668 - DEBUG - Progress: 4853/4853 bytes
+2026-02-12 01:53:38,668 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,669 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/j.py')
+2026-02-12 01:53:38,672 - INFO - Upload completed successfully: j.py
+2026-02-12 01:53:38,672 - INFO - Starting upload: lib/pygments/lexers/iolang.py -> /home/kevin/test/lib/pygments/lexers/iolang.py (1905 bytes)
+2026-02-12 01:53:38,674 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/iolang.py', 'wb')
+2026-02-12 01:53:38,675 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/iolang.py', 'wb') -> 00000000
+2026-02-12 01:53:38,675 - DEBUG - Progress: 1905/1905 bytes
+2026-02-12 01:53:38,675 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,676 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/iolang.py')
+2026-02-12 01:53:38,678 - INFO - Upload completed successfully: iolang.py
+2026-02-12 01:53:38,678 - INFO - Starting upload: lib/pygments/lexers/int_fiction.py -> /home/kevin/test/lib/pygments/lexers/int_fiction.py (56544 bytes)
+2026-02-12 01:53:38,680 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/int_fiction.py', 'wb')
+2026-02-12 01:53:38,681 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/int_fiction.py', 'wb') -> 00000000
+2026-02-12 01:53:38,681 - DEBUG - Progress: 32768/56544 bytes
+2026-02-12 01:53:38,681 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,688 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/int_fiction.py')
+2026-02-12 01:53:38,690 - INFO - Upload completed successfully: int_fiction.py
+2026-02-12 01:53:38,690 - INFO - Starting upload: lib/pygments/lexers/installers.py -> /home/kevin/test/lib/pygments/lexers/installers.py (14435 bytes)
+2026-02-12 01:53:38,692 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/installers.py', 'wb')
+2026-02-12 01:53:38,692 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/installers.py', 'wb') -> 00000000
+2026-02-12 01:53:38,692 - DEBUG - Progress: 14435/14435 bytes
+2026-02-12 01:53:38,693 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,695 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/installers.py')
+2026-02-12 01:53:38,696 - INFO - Upload completed successfully: installers.py
+2026-02-12 01:53:38,696 - INFO - Starting upload: lib/pygments/lexers/inferno.py -> /home/kevin/test/lib/pygments/lexers/inferno.py (3135 bytes)
+2026-02-12 01:53:38,699 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/inferno.py', 'wb')
+2026-02-12 01:53:38,699 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/inferno.py', 'wb') -> 00000000
+2026-02-12 01:53:38,699 - DEBUG - Progress: 3135/3135 bytes
+2026-02-12 01:53:38,699 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,701 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/inferno.py')
+2026-02-12 01:53:38,703 - INFO - Upload completed successfully: inferno.py
+2026-02-12 01:53:38,703 - INFO - Starting upload: lib/pygments/lexers/igor.py -> /home/kevin/test/lib/pygments/lexers/igor.py (31633 bytes)
+2026-02-12 01:53:38,704 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/igor.py', 'wb')
+2026-02-12 01:53:38,705 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/igor.py', 'wb') -> 00000000
+2026-02-12 01:53:38,705 - DEBUG - Progress: 31633/31633 bytes
+2026-02-12 01:53:38,705 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,709 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/igor.py')
+2026-02-12 01:53:38,711 - INFO - Upload completed successfully: igor.py
+2026-02-12 01:53:38,711 - INFO - Starting upload: lib/pygments/lexers/idl.py -> /home/kevin/test/lib/pygments/lexers/idl.py (15449 bytes)
+2026-02-12 01:53:38,713 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/idl.py', 'wb')
+2026-02-12 01:53:38,714 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/idl.py', 'wb') -> 00000000
+2026-02-12 01:53:38,714 - DEBUG - Progress: 15449/15449 bytes
+2026-02-12 01:53:38,714 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,716 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/idl.py')
+2026-02-12 01:53:38,718 - INFO - Upload completed successfully: idl.py
+2026-02-12 01:53:38,719 - INFO - Starting upload: lib/pygments/lexers/html.py -> /home/kevin/test/lib/pygments/lexers/html.py (21977 bytes)
+2026-02-12 01:53:38,721 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/html.py', 'wb')
+2026-02-12 01:53:38,721 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/html.py', 'wb') -> 00000000
+2026-02-12 01:53:38,721 - DEBUG - Progress: 21977/21977 bytes
+2026-02-12 01:53:38,721 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,724 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/html.py')
+2026-02-12 01:53:38,727 - INFO - Upload completed successfully: html.py
+2026-02-12 01:53:38,727 - INFO - Starting upload: lib/pygments/lexers/hexdump.py -> /home/kevin/test/lib/pygments/lexers/hexdump.py (3653 bytes)
+2026-02-12 01:53:38,729 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/hexdump.py', 'wb')
+2026-02-12 01:53:38,729 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/hexdump.py', 'wb') -> 00000000
+2026-02-12 01:53:38,730 - DEBUG - Progress: 3653/3653 bytes
+2026-02-12 01:53:38,730 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,731 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/hexdump.py')
+2026-02-12 01:53:38,733 - INFO - Upload completed successfully: hexdump.py
+2026-02-12 01:53:38,733 - INFO - Starting upload: lib/pygments/lexers/hdl.py -> /home/kevin/test/lib/pygments/lexers/hdl.py (22738 bytes)
+2026-02-12 01:53:38,735 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/hdl.py', 'wb')
+2026-02-12 01:53:38,735 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/hdl.py', 'wb') -> 00000000
+2026-02-12 01:53:38,735 - DEBUG - Progress: 22738/22738 bytes
+2026-02-12 01:53:38,735 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,738 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/hdl.py')
+2026-02-12 01:53:38,740 - INFO - Upload completed successfully: hdl.py
+2026-02-12 01:53:38,741 - INFO - Starting upload: lib/pygments/lexers/haxe.py -> /home/kevin/test/lib/pygments/lexers/haxe.py (30974 bytes)
+2026-02-12 01:53:38,742 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/haxe.py', 'wb')
+2026-02-12 01:53:38,743 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/haxe.py', 'wb') -> 00000000
+2026-02-12 01:53:38,743 - DEBUG - Progress: 30974/30974 bytes
+2026-02-12 01:53:38,743 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,747 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/haxe.py')
+2026-02-12 01:53:38,749 - INFO - Upload completed successfully: haxe.py
+2026-02-12 01:53:38,749 - INFO - Starting upload: lib/pygments/lexers/haskell.py -> /home/kevin/test/lib/pygments/lexers/haskell.py (33262 bytes)
+2026-02-12 01:53:38,751 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/haskell.py', 'wb')
+2026-02-12 01:53:38,752 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/haskell.py', 'wb') -> 00000000
+2026-02-12 01:53:38,752 - DEBUG - Progress: 32768/33262 bytes
+2026-02-12 01:53:38,752 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,757 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/haskell.py')
+2026-02-12 01:53:38,760 - INFO - Upload completed successfully: haskell.py
+2026-02-12 01:53:38,760 - INFO - Starting upload: lib/pygments/lexers/hare.py -> /home/kevin/test/lib/pygments/lexers/hare.py (2649 bytes)
+2026-02-12 01:53:38,762 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/hare.py', 'wb')
+2026-02-12 01:53:38,762 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/hare.py', 'wb') -> 00000000
+2026-02-12 01:53:38,763 - DEBUG - Progress: 2649/2649 bytes
+2026-02-12 01:53:38,763 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,764 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/hare.py')
+2026-02-12 01:53:38,766 - INFO - Upload completed successfully: hare.py
+2026-02-12 01:53:38,766 - INFO - Starting upload: lib/pygments/lexers/gsql.py -> /home/kevin/test/lib/pygments/lexers/gsql.py (3990 bytes)
+2026-02-12 01:53:38,768 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gsql.py', 'wb')
+2026-02-12 01:53:38,768 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gsql.py', 'wb') -> 00000000
+2026-02-12 01:53:38,768 - DEBUG - Progress: 3990/3990 bytes
+2026-02-12 01:53:38,769 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,770 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/gsql.py')
+2026-02-12 01:53:38,771 - INFO - Upload completed successfully: gsql.py
+2026-02-12 01:53:38,772 - INFO - Starting upload: lib/pygments/lexers/graphviz.py -> /home/kevin/test/lib/pygments/lexers/graphviz.py (1934 bytes)
+2026-02-12 01:53:38,774 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graphviz.py', 'wb')
+2026-02-12 01:53:38,774 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graphviz.py', 'wb') -> 00000000
+2026-02-12 01:53:38,774 - DEBUG - Progress: 1934/1934 bytes
+2026-02-12 01:53:38,774 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,776 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/graphviz.py')
+2026-02-12 01:53:38,778 - INFO - Upload completed successfully: graphviz.py
+2026-02-12 01:53:38,778 - INFO - Starting upload: lib/pygments/lexers/graphql.py -> /home/kevin/test/lib/pygments/lexers/graphql.py (5601 bytes)
+2026-02-12 01:53:38,780 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graphql.py', 'wb')
+2026-02-12 01:53:38,781 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graphql.py', 'wb') -> 00000000
+2026-02-12 01:53:38,781 - DEBUG - Progress: 5601/5601 bytes
+2026-02-12 01:53:38,781 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,782 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/graphql.py')
+2026-02-12 01:53:38,784 - INFO - Upload completed successfully: graphql.py
+2026-02-12 01:53:38,784 - INFO - Starting upload: lib/pygments/lexers/graphics.py -> /home/kevin/test/lib/pygments/lexers/graphics.py (39145 bytes)
+2026-02-12 01:53:38,786 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graphics.py', 'wb')
+2026-02-12 01:53:38,787 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graphics.py', 'wb') -> 00000000
+2026-02-12 01:53:38,787 - DEBUG - Progress: 32768/39145 bytes
+2026-02-12 01:53:38,787 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,791 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/graphics.py')
+2026-02-12 01:53:38,794 - INFO - Upload completed successfully: graphics.py
+2026-02-12 01:53:38,794 - INFO - Starting upload: lib/pygments/lexers/graph.py -> /home/kevin/test/lib/pygments/lexers/graph.py (4108 bytes)
+2026-02-12 01:53:38,796 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graph.py', 'wb')
+2026-02-12 01:53:38,797 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/graph.py', 'wb') -> 00000000
+2026-02-12 01:53:38,797 - DEBUG - Progress: 4108/4108 bytes
+2026-02-12 01:53:38,797 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,798 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/graph.py')
+2026-02-12 01:53:38,800 - INFO - Upload completed successfully: graph.py
+2026-02-12 01:53:38,800 - INFO - Starting upload: lib/pygments/lexers/grammar_notation.py -> /home/kevin/test/lib/pygments/lexers/grammar_notation.py (8043 bytes)
+2026-02-12 01:53:38,802 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/grammar_notation.py', 'wb')
+2026-02-12 01:53:38,802 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/grammar_notation.py', 'wb') -> 00000000
+2026-02-12 01:53:38,803 - DEBUG - Progress: 8043/8043 bytes
+2026-02-12 01:53:38,803 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,804 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/grammar_notation.py')
+2026-02-12 01:53:38,807 - INFO - Upload completed successfully: grammar_notation.py
+2026-02-12 01:53:38,807 - INFO - Starting upload: lib/pygments/lexers/go.py -> /home/kevin/test/lib/pygments/lexers/go.py (3783 bytes)
+2026-02-12 01:53:38,809 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/go.py', 'wb')
+2026-02-12 01:53:38,810 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/go.py', 'wb') -> 00000000
+2026-02-12 01:53:38,810 - DEBUG - Progress: 3783/3783 bytes
+2026-02-12 01:53:38,810 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,811 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/go.py')
+2026-02-12 01:53:38,813 - INFO - Upload completed successfully: go.py
+2026-02-12 01:53:38,814 - INFO - Starting upload: lib/pygments/lexers/gleam.py -> /home/kevin/test/lib/pygments/lexers/gleam.py (2392 bytes)
+2026-02-12 01:53:38,815 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gleam.py', 'wb')
+2026-02-12 01:53:38,816 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gleam.py', 'wb') -> 00000000
+2026-02-12 01:53:38,816 - DEBUG - Progress: 2392/2392 bytes
+2026-02-12 01:53:38,816 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,818 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/gleam.py')
+2026-02-12 01:53:38,820 - INFO - Upload completed successfully: gleam.py
+2026-02-12 01:53:38,820 - INFO - Starting upload: lib/pygments/lexers/gdscript.py -> /home/kevin/test/lib/pygments/lexers/gdscript.py (7566 bytes)
+2026-02-12 01:53:38,822 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gdscript.py', 'wb')
+2026-02-12 01:53:38,823 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gdscript.py', 'wb') -> 00000000
+2026-02-12 01:53:38,823 - DEBUG - Progress: 7566/7566 bytes
+2026-02-12 01:53:38,823 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,825 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/gdscript.py')
+2026-02-12 01:53:38,827 - INFO - Upload completed successfully: gdscript.py
+2026-02-12 01:53:38,827 - INFO - Starting upload: lib/pygments/lexers/gcodelexer.py -> /home/kevin/test/lib/pygments/lexers/gcodelexer.py (874 bytes)
+2026-02-12 01:53:38,829 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gcodelexer.py', 'wb')
+2026-02-12 01:53:38,830 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/gcodelexer.py', 'wb') -> 00000000
+2026-02-12 01:53:38,830 - DEBUG - Progress: 874/874 bytes
+2026-02-12 01:53:38,830 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,831 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/gcodelexer.py')
+2026-02-12 01:53:38,833 - INFO - Upload completed successfully: gcodelexer.py
+2026-02-12 01:53:38,833 - INFO - Starting upload: lib/pygments/lexers/futhark.py -> /home/kevin/test/lib/pygments/lexers/futhark.py (3743 bytes)
+2026-02-12 01:53:38,835 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/futhark.py', 'wb')
+2026-02-12 01:53:38,836 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/futhark.py', 'wb') -> 00000000
+2026-02-12 01:53:38,836 - DEBUG - Progress: 3743/3743 bytes
+2026-02-12 01:53:38,836 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,837 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/futhark.py')
+2026-02-12 01:53:38,839 - INFO - Upload completed successfully: futhark.py
+2026-02-12 01:53:38,839 - INFO - Starting upload: lib/pygments/lexers/functional.py -> /home/kevin/test/lib/pygments/lexers/functional.py (693 bytes)
+2026-02-12 01:53:38,841 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/functional.py', 'wb')
+2026-02-12 01:53:38,842 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/functional.py', 'wb') -> 00000000
+2026-02-12 01:53:38,842 - DEBUG - Progress: 693/693 bytes
+2026-02-12 01:53:38,842 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,843 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/functional.py')
+2026-02-12 01:53:38,844 - INFO - Upload completed successfully: functional.py
+2026-02-12 01:53:38,844 - INFO - Starting upload: lib/pygments/lexers/func.py -> /home/kevin/test/lib/pygments/lexers/func.py (3700 bytes)
+2026-02-12 01:53:38,846 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/func.py', 'wb')
+2026-02-12 01:53:38,846 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/func.py', 'wb') -> 00000000
+2026-02-12 01:53:38,847 - DEBUG - Progress: 3700/3700 bytes
+2026-02-12 01:53:38,847 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,848 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/func.py')
+2026-02-12 01:53:38,850 - INFO - Upload completed successfully: func.py
+2026-02-12 01:53:38,850 - INFO - Starting upload: lib/pygments/lexers/freefem.py -> /home/kevin/test/lib/pygments/lexers/freefem.py (26913 bytes)
+2026-02-12 01:53:38,851 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/freefem.py', 'wb')
+2026-02-12 01:53:38,852 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/freefem.py', 'wb') -> 00000000
+2026-02-12 01:53:38,852 - DEBUG - Progress: 26913/26913 bytes
+2026-02-12 01:53:38,852 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,856 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/freefem.py')
+2026-02-12 01:53:38,857 - INFO - Upload completed successfully: freefem.py
+2026-02-12 01:53:38,857 - INFO - Starting upload: lib/pygments/lexers/foxpro.py -> /home/kevin/test/lib/pygments/lexers/foxpro.py (26295 bytes)
+2026-02-12 01:53:38,859 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/foxpro.py', 'wb')
+2026-02-12 01:53:38,859 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/foxpro.py', 'wb') -> 00000000
+2026-02-12 01:53:38,859 - DEBUG - Progress: 26295/26295 bytes
+2026-02-12 01:53:38,860 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,863 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/foxpro.py')
+2026-02-12 01:53:38,865 - INFO - Upload completed successfully: foxpro.py
+2026-02-12 01:53:38,865 - INFO - Starting upload: lib/pygments/lexers/fortran.py -> /home/kevin/test/lib/pygments/lexers/fortran.py (10382 bytes)
+2026-02-12 01:53:38,866 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/fortran.py', 'wb')
+2026-02-12 01:53:38,867 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/fortran.py', 'wb') -> 00000000
+2026-02-12 01:53:38,867 - DEBUG - Progress: 10382/10382 bytes
+2026-02-12 01:53:38,867 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,869 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/fortran.py')
+2026-02-12 01:53:38,871 - INFO - Upload completed successfully: fortran.py
+2026-02-12 01:53:38,871 - INFO - Starting upload: lib/pygments/lexers/forth.py -> /home/kevin/test/lib/pygments/lexers/forth.py (7193 bytes)
+2026-02-12 01:53:38,872 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/forth.py', 'wb')
+2026-02-12 01:53:38,873 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/forth.py', 'wb') -> 00000000
+2026-02-12 01:53:38,873 - DEBUG - Progress: 7193/7193 bytes
+2026-02-12 01:53:38,873 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,875 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/forth.py')
+2026-02-12 01:53:38,877 - INFO - Upload completed successfully: forth.py
+2026-02-12 01:53:38,877 - INFO - Starting upload: lib/pygments/lexers/floscript.py -> /home/kevin/test/lib/pygments/lexers/floscript.py (2667 bytes)
+2026-02-12 01:53:38,878 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/floscript.py', 'wb')
+2026-02-12 01:53:38,879 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/floscript.py', 'wb') -> 00000000
+2026-02-12 01:53:38,879 - DEBUG - Progress: 2667/2667 bytes
+2026-02-12 01:53:38,879 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,880 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/floscript.py')
+2026-02-12 01:53:38,882 - INFO - Upload completed successfully: floscript.py
+2026-02-12 01:53:38,882 - INFO - Starting upload: lib/pygments/lexers/fift.py -> /home/kevin/test/lib/pygments/lexers/fift.py (1644 bytes)
+2026-02-12 01:53:38,884 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/fift.py', 'wb')
+2026-02-12 01:53:38,884 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/fift.py', 'wb') -> 00000000
+2026-02-12 01:53:38,884 - DEBUG - Progress: 1644/1644 bytes
+2026-02-12 01:53:38,884 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,885 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/fift.py')
+2026-02-12 01:53:38,887 - INFO - Upload completed successfully: fift.py
+2026-02-12 01:53:38,887 - INFO - Starting upload: lib/pygments/lexers/felix.py -> /home/kevin/test/lib/pygments/lexers/felix.py (9655 bytes)
+2026-02-12 01:53:38,888 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/felix.py', 'wb')
+2026-02-12 01:53:38,889 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/felix.py', 'wb') -> 00000000
+2026-02-12 01:53:38,889 - DEBUG - Progress: 9655/9655 bytes
+2026-02-12 01:53:38,889 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,891 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/felix.py')
+2026-02-12 01:53:38,893 - INFO - Upload completed successfully: felix.py
+2026-02-12 01:53:38,893 - INFO - Starting upload: lib/pygments/lexers/fantom.py -> /home/kevin/test/lib/pygments/lexers/fantom.py (10231 bytes)
+2026-02-12 01:53:38,894 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/fantom.py', 'wb')
+2026-02-12 01:53:38,895 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/fantom.py', 'wb') -> 00000000
+2026-02-12 01:53:38,895 - DEBUG - Progress: 10231/10231 bytes
+2026-02-12 01:53:38,895 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,897 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/fantom.py')
+2026-02-12 01:53:38,899 - INFO - Upload completed successfully: fantom.py
+2026-02-12 01:53:38,899 - INFO - Starting upload: lib/pygments/lexers/factor.py -> /home/kevin/test/lib/pygments/lexers/factor.py (19530 bytes)
+2026-02-12 01:53:38,900 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/factor.py', 'wb')
+2026-02-12 01:53:38,901 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/factor.py', 'wb') -> 00000000
+2026-02-12 01:53:38,901 - DEBUG - Progress: 19530/19530 bytes
+2026-02-12 01:53:38,901 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,904 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/factor.py')
+2026-02-12 01:53:38,905 - INFO - Upload completed successfully: factor.py
+2026-02-12 01:53:38,905 - INFO - Starting upload: lib/pygments/lexers/ezhil.py -> /home/kevin/test/lib/pygments/lexers/ezhil.py (3272 bytes)
+2026-02-12 01:53:38,907 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ezhil.py', 'wb')
+2026-02-12 01:53:38,907 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ezhil.py', 'wb') -> 00000000
+2026-02-12 01:53:38,908 - DEBUG - Progress: 3272/3272 bytes
+2026-02-12 01:53:38,908 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,909 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ezhil.py')
+2026-02-12 01:53:38,911 - INFO - Upload completed successfully: ezhil.py
+2026-02-12 01:53:38,911 - INFO - Starting upload: lib/pygments/lexers/esoteric.py -> /home/kevin/test/lib/pygments/lexers/esoteric.py (10500 bytes)
+2026-02-12 01:53:38,912 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/esoteric.py', 'wb')
+2026-02-12 01:53:38,913 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/esoteric.py', 'wb') -> 00000000
+2026-02-12 01:53:38,913 - DEBUG - Progress: 10500/10500 bytes
+2026-02-12 01:53:38,913 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,915 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/esoteric.py')
+2026-02-12 01:53:38,916 - INFO - Upload completed successfully: esoteric.py
+2026-02-12 01:53:38,917 - INFO - Starting upload: lib/pygments/lexers/erlang.py -> /home/kevin/test/lib/pygments/lexers/erlang.py (19147 bytes)
+2026-02-12 01:53:38,918 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/erlang.py', 'wb')
+2026-02-12 01:53:38,919 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/erlang.py', 'wb') -> 00000000
+2026-02-12 01:53:38,919 - DEBUG - Progress: 19147/19147 bytes
+2026-02-12 01:53:38,919 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,923 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/erlang.py')
+2026-02-12 01:53:38,925 - INFO - Upload completed successfully: erlang.py
+2026-02-12 01:53:38,925 - INFO - Starting upload: lib/pygments/lexers/email.py -> /home/kevin/test/lib/pygments/lexers/email.py (4804 bytes)
+2026-02-12 01:53:38,926 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/email.py', 'wb')
+2026-02-12 01:53:38,927 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/email.py', 'wb') -> 00000000
+2026-02-12 01:53:38,927 - DEBUG - Progress: 4804/4804 bytes
+2026-02-12 01:53:38,927 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,929 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/email.py')
+2026-02-12 01:53:38,931 - INFO - Upload completed successfully: email.py
+2026-02-12 01:53:38,931 - INFO - Starting upload: lib/pygments/lexers/elpi.py -> /home/kevin/test/lib/pygments/lexers/elpi.py (6877 bytes)
+2026-02-12 01:53:38,933 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/elpi.py', 'wb')
+2026-02-12 01:53:38,934 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/elpi.py', 'wb') -> 00000000
+2026-02-12 01:53:38,934 - DEBUG - Progress: 6877/6877 bytes
+2026-02-12 01:53:38,934 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,935 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/elpi.py')
+2026-02-12 01:53:38,937 - INFO - Upload completed successfully: elpi.py
+2026-02-12 01:53:38,938 - INFO - Starting upload: lib/pygments/lexers/elm.py -> /home/kevin/test/lib/pygments/lexers/elm.py (3152 bytes)
+2026-02-12 01:53:38,940 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/elm.py', 'wb')
+2026-02-12 01:53:38,941 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/elm.py', 'wb') -> 00000000
+2026-02-12 01:53:38,941 - DEBUG - Progress: 3152/3152 bytes
+2026-02-12 01:53:38,941 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,942 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/elm.py')
+2026-02-12 01:53:38,944 - INFO - Upload completed successfully: elm.py
+2026-02-12 01:53:38,944 - INFO - Starting upload: lib/pygments/lexers/eiffel.py -> /home/kevin/test/lib/pygments/lexers/eiffel.py (2690 bytes)
+2026-02-12 01:53:38,946 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/eiffel.py', 'wb')
+2026-02-12 01:53:38,947 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/eiffel.py', 'wb') -> 00000000
+2026-02-12 01:53:38,947 - DEBUG - Progress: 2690/2690 bytes
+2026-02-12 01:53:38,947 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,949 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/eiffel.py')
+2026-02-12 01:53:38,951 - INFO - Upload completed successfully: eiffel.py
+2026-02-12 01:53:38,951 - INFO - Starting upload: lib/pygments/lexers/ecl.py -> /home/kevin/test/lib/pygments/lexers/ecl.py (6371 bytes)
+2026-02-12 01:53:38,954 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ecl.py', 'wb')
+2026-02-12 01:53:38,954 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ecl.py', 'wb') -> 00000000
+2026-02-12 01:53:38,955 - DEBUG - Progress: 6371/6371 bytes
+2026-02-12 01:53:38,955 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,956 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ecl.py')
+2026-02-12 01:53:38,958 - INFO - Upload completed successfully: ecl.py
+2026-02-12 01:53:38,958 - INFO - Starting upload: lib/pygments/lexers/dylan.py -> /home/kevin/test/lib/pygments/lexers/dylan.py (10409 bytes)
+2026-02-12 01:53:38,960 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dylan.py', 'wb')
+2026-02-12 01:53:38,961 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dylan.py', 'wb') -> 00000000
+2026-02-12 01:53:38,961 - DEBUG - Progress: 10409/10409 bytes
+2026-02-12 01:53:38,961 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,963 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/dylan.py')
+2026-02-12 01:53:38,965 - INFO - Upload completed successfully: dylan.py
+2026-02-12 01:53:38,966 - INFO - Starting upload: lib/pygments/lexers/dsls.py -> /home/kevin/test/lib/pygments/lexers/dsls.py (36746 bytes)
+2026-02-12 01:53:38,968 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dsls.py', 'wb')
+2026-02-12 01:53:38,969 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dsls.py', 'wb') -> 00000000
+2026-02-12 01:53:38,969 - DEBUG - Progress: 32768/36746 bytes
+2026-02-12 01:53:38,970 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,974 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/dsls.py')
+2026-02-12 01:53:38,975 - INFO - Upload completed successfully: dsls.py
+2026-02-12 01:53:38,976 - INFO - Starting upload: lib/pygments/lexers/dotnet.py -> /home/kevin/test/lib/pygments/lexers/dotnet.py (39441 bytes)
+2026-02-12 01:53:38,977 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dotnet.py', 'wb')
+2026-02-12 01:53:38,978 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dotnet.py', 'wb') -> 00000000
+2026-02-12 01:53:38,978 - DEBUG - Progress: 32768/39441 bytes
+2026-02-12 01:53:38,978 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,982 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/dotnet.py')
+2026-02-12 01:53:38,984 - INFO - Upload completed successfully: dotnet.py
+2026-02-12 01:53:38,984 - INFO - Starting upload: lib/pygments/lexers/dns.py -> /home/kevin/test/lib/pygments/lexers/dns.py (3891 bytes)
+2026-02-12 01:53:38,985 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dns.py', 'wb')
+2026-02-12 01:53:38,986 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dns.py', 'wb') -> 00000000
+2026-02-12 01:53:38,986 - DEBUG - Progress: 3891/3891 bytes
+2026-02-12 01:53:38,986 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,988 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/dns.py')
+2026-02-12 01:53:38,989 - INFO - Upload completed successfully: dns.py
+2026-02-12 01:53:38,989 - INFO - Starting upload: lib/pygments/lexers/diff.py -> /home/kevin/test/lib/pygments/lexers/diff.py (5382 bytes)
+2026-02-12 01:53:38,991 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/diff.py', 'wb')
+2026-02-12 01:53:38,992 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/diff.py', 'wb') -> 00000000
+2026-02-12 01:53:38,992 - DEBUG - Progress: 5382/5382 bytes
+2026-02-12 01:53:38,992 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:38,993 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/diff.py')
+2026-02-12 01:53:38,996 - INFO - Upload completed successfully: diff.py
+2026-02-12 01:53:38,997 - INFO - Starting upload: lib/pygments/lexers/devicetree.py -> /home/kevin/test/lib/pygments/lexers/devicetree.py (4019 bytes)
+2026-02-12 01:53:39,000 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/devicetree.py', 'wb')
+2026-02-12 01:53:39,001 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/devicetree.py', 'wb') -> 00000000
+2026-02-12 01:53:39,001 - DEBUG - Progress: 4019/4019 bytes
+2026-02-12 01:53:39,001 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,003 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/devicetree.py')
+2026-02-12 01:53:39,006 - INFO - Upload completed successfully: devicetree.py
+2026-02-12 01:53:39,006 - INFO - Starting upload: lib/pygments/lexers/dax.py -> /home/kevin/test/lib/pygments/lexers/dax.py (8098 bytes)
+2026-02-12 01:53:39,010 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dax.py', 'wb')
+2026-02-12 01:53:39,011 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dax.py', 'wb') -> 00000000
+2026-02-12 01:53:39,011 - DEBUG - Progress: 8098/8098 bytes
+2026-02-12 01:53:39,011 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,013 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/dax.py')
+2026-02-12 01:53:39,017 - INFO - Upload completed successfully: dax.py
+2026-02-12 01:53:39,017 - INFO - Starting upload: lib/pygments/lexers/data.py -> /home/kevin/test/lib/pygments/lexers/data.py (27026 bytes)
+2026-02-12 01:53:39,019 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/data.py', 'wb')
+2026-02-12 01:53:39,020 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/data.py', 'wb') -> 00000000
+2026-02-12 01:53:39,020 - DEBUG - Progress: 27026/27026 bytes
+2026-02-12 01:53:39,020 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,024 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/data.py')
+2026-02-12 01:53:39,025 - INFO - Upload completed successfully: data.py
+2026-02-12 01:53:39,026 - INFO - Starting upload: lib/pygments/lexers/dalvik.py -> /home/kevin/test/lib/pygments/lexers/dalvik.py (4606 bytes)
+2026-02-12 01:53:39,027 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dalvik.py', 'wb')
+2026-02-12 01:53:39,028 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/dalvik.py', 'wb') -> 00000000
+2026-02-12 01:53:39,028 - DEBUG - Progress: 4606/4606 bytes
+2026-02-12 01:53:39,028 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,029 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/dalvik.py')
+2026-02-12 01:53:39,031 - INFO - Upload completed successfully: dalvik.py
+2026-02-12 01:53:39,031 - INFO - Starting upload: lib/pygments/lexers/d.py -> /home/kevin/test/lib/pygments/lexers/d.py (9920 bytes)
+2026-02-12 01:53:39,032 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/d.py', 'wb')
+2026-02-12 01:53:39,033 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/d.py', 'wb') -> 00000000
+2026-02-12 01:53:39,033 - DEBUG - Progress: 9920/9920 bytes
+2026-02-12 01:53:39,033 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,035 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/d.py')
+2026-02-12 01:53:39,037 - INFO - Upload completed successfully: d.py
+2026-02-12 01:53:39,037 - INFO - Starting upload: lib/pygments/lexers/css.py -> /home/kevin/test/lib/pygments/lexers/css.py (25376 bytes)
+2026-02-12 01:53:39,038 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/css.py', 'wb')
+2026-02-12 01:53:39,039 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/css.py', 'wb') -> 00000000
+2026-02-12 01:53:39,039 - DEBUG - Progress: 25376/25376 bytes
+2026-02-12 01:53:39,039 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,043 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/css.py')
+2026-02-12 01:53:39,045 - INFO - Upload completed successfully: css.py
+2026-02-12 01:53:39,045 - INFO - Starting upload: lib/pygments/lexers/csound.py -> /home/kevin/test/lib/pygments/lexers/csound.py (16998 bytes)
+2026-02-12 01:53:39,046 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/csound.py', 'wb')
+2026-02-12 01:53:39,047 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/csound.py', 'wb') -> 00000000
+2026-02-12 01:53:39,047 - DEBUG - Progress: 16998/16998 bytes
+2026-02-12 01:53:39,047 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,050 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/csound.py')
+2026-02-12 01:53:39,051 - INFO - Upload completed successfully: csound.py
+2026-02-12 01:53:39,052 - INFO - Starting upload: lib/pygments/lexers/crystal.py -> /home/kevin/test/lib/pygments/lexers/crystal.py (15754 bytes)
+2026-02-12 01:53:39,053 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/crystal.py', 'wb')
+2026-02-12 01:53:39,054 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/crystal.py', 'wb') -> 00000000
+2026-02-12 01:53:39,054 - DEBUG - Progress: 15754/15754 bytes
+2026-02-12 01:53:39,054 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,056 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/crystal.py')
+2026-02-12 01:53:39,057 - INFO - Upload completed successfully: crystal.py
+2026-02-12 01:53:39,057 - INFO - Starting upload: lib/pygments/lexers/cplint.py -> /home/kevin/test/lib/pygments/lexers/cplint.py (1389 bytes)
+2026-02-12 01:53:39,059 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/cplint.py', 'wb')
+2026-02-12 01:53:39,059 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/cplint.py', 'wb') -> 00000000
+2026-02-12 01:53:39,060 - DEBUG - Progress: 1389/1389 bytes
+2026-02-12 01:53:39,060 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,061 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/cplint.py')
+2026-02-12 01:53:39,062 - INFO - Upload completed successfully: cplint.py
+2026-02-12 01:53:39,062 - INFO - Starting upload: lib/pygments/lexers/console.py -> /home/kevin/test/lib/pygments/lexers/console.py (4180 bytes)
+2026-02-12 01:53:39,064 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/console.py', 'wb')
+2026-02-12 01:53:39,065 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/console.py', 'wb') -> 00000000
+2026-02-12 01:53:39,065 - DEBUG - Progress: 4180/4180 bytes
+2026-02-12 01:53:39,065 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,066 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/console.py')
+2026-02-12 01:53:39,068 - INFO - Upload completed successfully: console.py
+2026-02-12 01:53:39,068 - INFO - Starting upload: lib/pygments/lexers/configs.py -> /home/kevin/test/lib/pygments/lexers/configs.py (50913 bytes)
+2026-02-12 01:53:39,069 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/configs.py', 'wb')
+2026-02-12 01:53:39,070 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/configs.py', 'wb') -> 00000000
+2026-02-12 01:53:39,070 - DEBUG - Progress: 32768/50913 bytes
+2026-02-12 01:53:39,071 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,079 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/configs.py')
+2026-02-12 01:53:39,081 - INFO - Upload completed successfully: configs.py
+2026-02-12 01:53:39,081 - INFO - Starting upload: lib/pygments/lexers/compiled.py -> /home/kevin/test/lib/pygments/lexers/compiled.py (1426 bytes)
+2026-02-12 01:53:39,083 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/compiled.py', 'wb')
+2026-02-12 01:53:39,084 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/compiled.py', 'wb') -> 00000000
+2026-02-12 01:53:39,084 - DEBUG - Progress: 1426/1426 bytes
+2026-02-12 01:53:39,084 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,085 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/compiled.py')
+2026-02-12 01:53:39,087 - INFO - Upload completed successfully: compiled.py
+2026-02-12 01:53:39,087 - INFO - Starting upload: lib/pygments/lexers/comal.py -> /home/kevin/test/lib/pygments/lexers/comal.py (3179 bytes)
+2026-02-12 01:53:39,089 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/comal.py', 'wb')
+2026-02-12 01:53:39,090 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/comal.py', 'wb') -> 00000000
+2026-02-12 01:53:39,090 - DEBUG - Progress: 3179/3179 bytes
+2026-02-12 01:53:39,090 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,092 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/comal.py')
+2026-02-12 01:53:39,094 - INFO - Upload completed successfully: comal.py
+2026-02-12 01:53:39,094 - INFO - Starting upload: lib/pygments/lexers/codeql.py -> /home/kevin/test/lib/pygments/lexers/codeql.py (2576 bytes)
+2026-02-12 01:53:39,096 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/codeql.py', 'wb')
+2026-02-12 01:53:39,096 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/codeql.py', 'wb') -> 00000000
+2026-02-12 01:53:39,097 - DEBUG - Progress: 2576/2576 bytes
+2026-02-12 01:53:39,097 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,098 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/codeql.py')
+2026-02-12 01:53:39,100 - INFO - Upload completed successfully: codeql.py
+2026-02-12 01:53:39,100 - INFO - Starting upload: lib/pygments/lexers/clean.py -> /home/kevin/test/lib/pygments/lexers/clean.py (6418 bytes)
+2026-02-12 01:53:39,102 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/clean.py', 'wb')
+2026-02-12 01:53:39,103 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/clean.py', 'wb') -> 00000000
+2026-02-12 01:53:39,103 - DEBUG - Progress: 6418/6418 bytes
+2026-02-12 01:53:39,103 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,104 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/clean.py')
+2026-02-12 01:53:39,106 - INFO - Upload completed successfully: clean.py
+2026-02-12 01:53:39,107 - INFO - Starting upload: lib/pygments/lexers/chapel.py -> /home/kevin/test/lib/pygments/lexers/chapel.py (5156 bytes)
+2026-02-12 01:53:39,109 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/chapel.py', 'wb')
+2026-02-12 01:53:39,109 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/chapel.py', 'wb') -> 00000000
+2026-02-12 01:53:39,109 - DEBUG - Progress: 5156/5156 bytes
+2026-02-12 01:53:39,110 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,111 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/chapel.py')
+2026-02-12 01:53:39,113 - INFO - Upload completed successfully: chapel.py
+2026-02-12 01:53:39,113 - INFO - Starting upload: lib/pygments/lexers/cddl.py -> /home/kevin/test/lib/pygments/lexers/cddl.py (5076 bytes)
+2026-02-12 01:53:39,115 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/cddl.py', 'wb')
+2026-02-12 01:53:39,116 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/cddl.py', 'wb') -> 00000000
+2026-02-12 01:53:39,116 - DEBUG - Progress: 5076/5076 bytes
+2026-02-12 01:53:39,116 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,117 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/cddl.py')
+2026-02-12 01:53:39,119 - INFO - Upload completed successfully: cddl.py
+2026-02-12 01:53:39,120 - INFO - Starting upload: lib/pygments/lexers/carbon.py -> /home/kevin/test/lib/pygments/lexers/carbon.py (3211 bytes)
+2026-02-12 01:53:39,122 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/carbon.py', 'wb')
+2026-02-12 01:53:39,122 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/carbon.py', 'wb') -> 00000000
+2026-02-12 01:53:39,122 - DEBUG - Progress: 3211/3211 bytes
+2026-02-12 01:53:39,123 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,124 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/carbon.py')
+2026-02-12 01:53:39,126 - INFO - Upload completed successfully: carbon.py
+2026-02-12 01:53:39,126 - INFO - Starting upload: lib/pygments/lexers/capnproto.py -> /home/kevin/test/lib/pygments/lexers/capnproto.py (2174 bytes)
+2026-02-12 01:53:39,128 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/capnproto.py', 'wb')
+2026-02-12 01:53:39,129 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/capnproto.py', 'wb') -> 00000000
+2026-02-12 01:53:39,129 - DEBUG - Progress: 2174/2174 bytes
+2026-02-12 01:53:39,129 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,130 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/capnproto.py')
+2026-02-12 01:53:39,133 - INFO - Upload completed successfully: capnproto.py
+2026-02-12 01:53:39,133 - INFO - Starting upload: lib/pygments/lexers/c_like.py -> /home/kevin/test/lib/pygments/lexers/c_like.py (32021 bytes)
+2026-02-12 01:53:39,134 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/c_like.py', 'wb')
+2026-02-12 01:53:39,135 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/c_like.py', 'wb') -> 00000000
+2026-02-12 01:53:39,135 - DEBUG - Progress: 32021/32021 bytes
+2026-02-12 01:53:39,135 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,139 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/c_like.py')
+2026-02-12 01:53:39,140 - INFO - Upload completed successfully: c_like.py
+2026-02-12 01:53:39,141 - INFO - Starting upload: lib/pygments/lexers/c_cpp.py -> /home/kevin/test/lib/pygments/lexers/c_cpp.py (18059 bytes)
+2026-02-12 01:53:39,142 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/c_cpp.py', 'wb')
+2026-02-12 01:53:39,143 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/c_cpp.py', 'wb') -> 00000000
+2026-02-12 01:53:39,143 - DEBUG - Progress: 18059/18059 bytes
+2026-02-12 01:53:39,143 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,147 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/c_cpp.py')
+2026-02-12 01:53:39,149 - INFO - Upload completed successfully: c_cpp.py
+2026-02-12 01:53:39,150 - INFO - Starting upload: lib/pygments/lexers/business.py -> /home/kevin/test/lib/pygments/lexers/business.py (28345 bytes)
+2026-02-12 01:53:39,152 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/business.py', 'wb')
+2026-02-12 01:53:39,152 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/business.py', 'wb') -> 00000000
+2026-02-12 01:53:39,153 - DEBUG - Progress: 28345/28345 bytes
+2026-02-12 01:53:39,153 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,157 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/business.py')
+2026-02-12 01:53:39,159 - INFO - Upload completed successfully: business.py
+2026-02-12 01:53:39,160 - INFO - Starting upload: lib/pygments/lexers/bqn.py -> /home/kevin/test/lib/pygments/lexers/bqn.py (3671 bytes)
+2026-02-12 01:53:39,162 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bqn.py', 'wb')
+2026-02-12 01:53:39,163 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bqn.py', 'wb') -> 00000000
+2026-02-12 01:53:39,163 - DEBUG - Progress: 3671/3671 bytes
+2026-02-12 01:53:39,163 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,164 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/bqn.py')
+2026-02-12 01:53:39,165 - INFO - Upload completed successfully: bqn.py
+2026-02-12 01:53:39,165 - INFO - Starting upload: lib/pygments/lexers/boa.py -> /home/kevin/test/lib/pygments/lexers/boa.py (3921 bytes)
+2026-02-12 01:53:39,167 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/boa.py', 'wb')
+2026-02-12 01:53:39,167 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/boa.py', 'wb') -> 00000000
+2026-02-12 01:53:39,168 - DEBUG - Progress: 3921/3921 bytes
+2026-02-12 01:53:39,168 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,169 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/boa.py')
+2026-02-12 01:53:39,171 - INFO - Upload completed successfully: boa.py
+2026-02-12 01:53:39,171 - INFO - Starting upload: lib/pygments/lexers/blueprint.py -> /home/kevin/test/lib/pygments/lexers/blueprint.py (6188 bytes)
+2026-02-12 01:53:39,172 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/blueprint.py', 'wb')
+2026-02-12 01:53:39,173 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/blueprint.py', 'wb') -> 00000000
+2026-02-12 01:53:39,173 - DEBUG - Progress: 6188/6188 bytes
+2026-02-12 01:53:39,173 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,174 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/blueprint.py')
+2026-02-12 01:53:39,176 - INFO - Upload completed successfully: blueprint.py
+2026-02-12 01:53:39,176 - INFO - Starting upload: lib/pygments/lexers/bibtex.py -> /home/kevin/test/lib/pygments/lexers/bibtex.py (4811 bytes)
+2026-02-12 01:53:39,178 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bibtex.py', 'wb')
+2026-02-12 01:53:39,179 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bibtex.py', 'wb') -> 00000000
+2026-02-12 01:53:39,179 - DEBUG - Progress: 4811/4811 bytes
+2026-02-12 01:53:39,179 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,180 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/bibtex.py')
+2026-02-12 01:53:39,182 - INFO - Upload completed successfully: bibtex.py
+2026-02-12 01:53:39,183 - INFO - Starting upload: lib/pygments/lexers/berry.py -> /home/kevin/test/lib/pygments/lexers/berry.py (3209 bytes)
+2026-02-12 01:53:39,184 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/berry.py', 'wb')
+2026-02-12 01:53:39,185 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/berry.py', 'wb') -> 00000000
+2026-02-12 01:53:39,185 - DEBUG - Progress: 3209/3209 bytes
+2026-02-12 01:53:39,185 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,186 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/berry.py')
+2026-02-12 01:53:39,188 - INFO - Upload completed successfully: berry.py
+2026-02-12 01:53:39,189 - INFO - Starting upload: lib/pygments/lexers/bdd.py -> /home/kevin/test/lib/pygments/lexers/bdd.py (1641 bytes)
+2026-02-12 01:53:39,191 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bdd.py', 'wb')
+2026-02-12 01:53:39,191 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bdd.py', 'wb') -> 00000000
+2026-02-12 01:53:39,192 - DEBUG - Progress: 1641/1641 bytes
+2026-02-12 01:53:39,192 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,193 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/bdd.py')
+2026-02-12 01:53:39,195 - INFO - Upload completed successfully: bdd.py
+2026-02-12 01:53:39,195 - INFO - Starting upload: lib/pygments/lexers/basic.py -> /home/kevin/test/lib/pygments/lexers/basic.py (27989 bytes)
+2026-02-12 01:53:39,197 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/basic.py', 'wb')
+2026-02-12 01:53:39,198 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/basic.py', 'wb') -> 00000000
+2026-02-12 01:53:39,198 - DEBUG - Progress: 27989/27989 bytes
+2026-02-12 01:53:39,198 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,202 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/basic.py')
+2026-02-12 01:53:39,204 - INFO - Upload completed successfully: basic.py
+2026-02-12 01:53:39,204 - INFO - Starting upload: lib/pygments/lexers/bare.py -> /home/kevin/test/lib/pygments/lexers/bare.py (3020 bytes)
+2026-02-12 01:53:39,240 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bare.py', 'wb')
+2026-02-12 01:53:39,241 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/bare.py', 'wb') -> 00000000
+2026-02-12 01:53:39,241 - DEBUG - Progress: 3020/3020 bytes
+2026-02-12 01:53:39,241 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,242 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/bare.py')
+2026-02-12 01:53:39,244 - INFO - Upload completed successfully: bare.py
+2026-02-12 01:53:39,244 - INFO - Starting upload: lib/pygments/lexers/automation.py -> /home/kevin/test/lib/pygments/lexers/automation.py (19831 bytes)
+2026-02-12 01:53:39,246 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/automation.py', 'wb')
+2026-02-12 01:53:39,246 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/automation.py', 'wb') -> 00000000
+2026-02-12 01:53:39,246 - DEBUG - Progress: 19831/19831 bytes
+2026-02-12 01:53:39,247 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,249 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/automation.py')
+2026-02-12 01:53:39,251 - INFO - Upload completed successfully: automation.py
+2026-02-12 01:53:39,251 - INFO - Starting upload: lib/pygments/lexers/asn1.py -> /home/kevin/test/lib/pygments/lexers/asn1.py (4262 bytes)
+2026-02-12 01:53:39,252 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/asn1.py', 'wb')
+2026-02-12 01:53:39,253 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/asn1.py', 'wb') -> 00000000
+2026-02-12 01:53:39,253 - DEBUG - Progress: 4262/4262 bytes
+2026-02-12 01:53:39,253 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,254 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/asn1.py')
+2026-02-12 01:53:39,256 - INFO - Upload completed successfully: asn1.py
+2026-02-12 01:53:39,256 - INFO - Starting upload: lib/pygments/lexers/asm.py -> /home/kevin/test/lib/pygments/lexers/asm.py (41967 bytes)
+2026-02-12 01:53:39,257 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/asm.py', 'wb')
+2026-02-12 01:53:39,258 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/asm.py', 'wb') -> 00000000
+2026-02-12 01:53:39,258 - DEBUG - Progress: 32768/41967 bytes
+2026-02-12 01:53:39,258 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,264 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/asm.py')
+2026-02-12 01:53:39,266 - INFO - Upload completed successfully: asm.py
+2026-02-12 01:53:39,266 - INFO - Starting upload: lib/pygments/lexers/asc.py -> /home/kevin/test/lib/pygments/lexers/asc.py (1693 bytes)
+2026-02-12 01:53:39,267 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/asc.py', 'wb')
+2026-02-12 01:53:39,268 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/asc.py', 'wb') -> 00000000
+2026-02-12 01:53:39,268 - DEBUG - Progress: 1693/1693 bytes
+2026-02-12 01:53:39,268 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,269 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/asc.py')
+2026-02-12 01:53:39,270 - INFO - Upload completed successfully: asc.py
+2026-02-12 01:53:39,270 - INFO - Starting upload: lib/pygments/lexers/arturo.py -> /home/kevin/test/lib/pygments/lexers/arturo.py (11414 bytes)
+2026-02-12 01:53:39,272 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/arturo.py', 'wb')
+2026-02-12 01:53:39,272 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/arturo.py', 'wb') -> 00000000
+2026-02-12 01:53:39,272 - DEBUG - Progress: 11414/11414 bytes
+2026-02-12 01:53:39,272 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,274 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/arturo.py')
+2026-02-12 01:53:39,276 - INFO - Upload completed successfully: arturo.py
+2026-02-12 01:53:39,276 - INFO - Starting upload: lib/pygments/lexers/arrow.py -> /home/kevin/test/lib/pygments/lexers/arrow.py (3564 bytes)
+2026-02-12 01:53:39,277 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/arrow.py', 'wb')
+2026-02-12 01:53:39,278 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/arrow.py', 'wb') -> 00000000
+2026-02-12 01:53:39,278 - DEBUG - Progress: 3564/3564 bytes
+2026-02-12 01:53:39,278 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,279 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/arrow.py')
+2026-02-12 01:53:39,281 - INFO - Upload completed successfully: arrow.py
+2026-02-12 01:53:39,281 - INFO - Starting upload: lib/pygments/lexers/archetype.py -> /home/kevin/test/lib/pygments/lexers/archetype.py (11538 bytes)
+2026-02-12 01:53:39,283 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/archetype.py', 'wb')
+2026-02-12 01:53:39,284 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/archetype.py', 'wb') -> 00000000
+2026-02-12 01:53:39,284 - DEBUG - Progress: 11538/11538 bytes
+2026-02-12 01:53:39,284 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,286 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/archetype.py')
+2026-02-12 01:53:39,287 - INFO - Upload completed successfully: archetype.py
+2026-02-12 01:53:39,287 - INFO - Starting upload: lib/pygments/lexers/apl.py -> /home/kevin/test/lib/pygments/lexers/apl.py (3404 bytes)
+2026-02-12 01:53:39,289 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/apl.py', 'wb')
+2026-02-12 01:53:39,289 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/apl.py', 'wb') -> 00000000
+2026-02-12 01:53:39,290 - DEBUG - Progress: 3404/3404 bytes
+2026-02-12 01:53:39,290 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,291 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/apl.py')
+2026-02-12 01:53:39,292 - INFO - Upload completed successfully: apl.py
+2026-02-12 01:53:39,292 - INFO - Starting upload: lib/pygments/lexers/apdlexer.py -> /home/kevin/test/lib/pygments/lexers/apdlexer.py (30800 bytes)
+2026-02-12 01:53:39,294 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/apdlexer.py', 'wb')
+2026-02-12 01:53:39,294 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/apdlexer.py', 'wb') -> 00000000
+2026-02-12 01:53:39,294 - DEBUG - Progress: 30800/30800 bytes
+2026-02-12 01:53:39,294 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,298 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/apdlexer.py')
+2026-02-12 01:53:39,300 - INFO - Upload completed successfully: apdlexer.py
+2026-02-12 01:53:39,300 - INFO - Starting upload: lib/pygments/lexers/ampl.py -> /home/kevin/test/lib/pygments/lexers/ampl.py (4176 bytes)
+2026-02-12 01:53:39,302 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ampl.py', 'wb')
+2026-02-12 01:53:39,302 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ampl.py', 'wb') -> 00000000
+2026-02-12 01:53:39,303 - DEBUG - Progress: 4176/4176 bytes
+2026-02-12 01:53:39,303 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,304 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ampl.py')
+2026-02-12 01:53:39,306 - INFO - Upload completed successfully: ampl.py
+2026-02-12 01:53:39,306 - INFO - Starting upload: lib/pygments/lexers/amdgpu.py -> /home/kevin/test/lib/pygments/lexers/amdgpu.py (1723 bytes)
+2026-02-12 01:53:39,307 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/amdgpu.py', 'wb')
+2026-02-12 01:53:39,308 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/amdgpu.py', 'wb') -> 00000000
+2026-02-12 01:53:39,308 - DEBUG - Progress: 1723/1723 bytes
+2026-02-12 01:53:39,308 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,309 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/amdgpu.py')
+2026-02-12 01:53:39,311 - INFO - Upload completed successfully: amdgpu.py
+2026-02-12 01:53:39,311 - INFO - Starting upload: lib/pygments/lexers/ambient.py -> /home/kevin/test/lib/pygments/lexers/ambient.py (2605 bytes)
+2026-02-12 01:53:39,312 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ambient.py', 'wb')
+2026-02-12 01:53:39,313 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ambient.py', 'wb') -> 00000000
+2026-02-12 01:53:39,313 - DEBUG - Progress: 2605/2605 bytes
+2026-02-12 01:53:39,313 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,314 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ambient.py')
+2026-02-12 01:53:39,316 - INFO - Upload completed successfully: ambient.py
+2026-02-12 01:53:39,316 - INFO - Starting upload: lib/pygments/lexers/algebra.py -> /home/kevin/test/lib/pygments/lexers/algebra.py (9952 bytes)
+2026-02-12 01:53:39,318 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/algebra.py', 'wb')
+2026-02-12 01:53:39,318 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/algebra.py', 'wb') -> 00000000
+2026-02-12 01:53:39,319 - DEBUG - Progress: 9952/9952 bytes
+2026-02-12 01:53:39,319 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,320 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/algebra.py')
+2026-02-12 01:53:39,323 - INFO - Upload completed successfully: algebra.py
+2026-02-12 01:53:39,323 - INFO - Starting upload: lib/pygments/lexers/agile.py -> /home/kevin/test/lib/pygments/lexers/agile.py (896 bytes)
+2026-02-12 01:53:39,324 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/agile.py', 'wb')
+2026-02-12 01:53:39,325 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/agile.py', 'wb') -> 00000000
+2026-02-12 01:53:39,325 - DEBUG - Progress: 896/896 bytes
+2026-02-12 01:53:39,325 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,326 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/agile.py')
+2026-02-12 01:53:39,327 - INFO - Upload completed successfully: agile.py
+2026-02-12 01:53:39,327 - INFO - Starting upload: lib/pygments/lexers/ada.py -> /home/kevin/test/lib/pygments/lexers/ada.py (5353 bytes)
+2026-02-12 01:53:39,329 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ada.py', 'wb')
+2026-02-12 01:53:39,329 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/ada.py', 'wb') -> 00000000
+2026-02-12 01:53:39,329 - DEBUG - Progress: 5353/5353 bytes
+2026-02-12 01:53:39,329 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,331 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/ada.py')
+2026-02-12 01:53:39,332 - INFO - Upload completed successfully: ada.py
+2026-02-12 01:53:39,332 - INFO - Starting upload: lib/pygments/lexers/actionscript.py -> /home/kevin/test/lib/pygments/lexers/actionscript.py (11727 bytes)
+2026-02-12 01:53:39,334 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/actionscript.py', 'wb')
+2026-02-12 01:53:39,334 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/actionscript.py', 'wb') -> 00000000
+2026-02-12 01:53:39,334 - DEBUG - Progress: 11727/11727 bytes
+2026-02-12 01:53:39,334 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,336 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/actionscript.py')
+2026-02-12 01:53:39,338 - INFO - Upload completed successfully: actionscript.py
+2026-02-12 01:53:39,338 - INFO - Starting upload: lib/pygments/lexers/_vim_builtins.py -> /home/kevin/test/lib/pygments/lexers/_vim_builtins.py (57066 bytes)
+2026-02-12 01:53:39,339 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_vim_builtins.py', 'wb')
+2026-02-12 01:53:39,340 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_vim_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,340 - DEBUG - Progress: 32768/57066 bytes
+2026-02-12 01:53:39,340 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,350 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_vim_builtins.py')
+2026-02-12 01:53:39,352 - INFO - Upload completed successfully: _vim_builtins.py
+2026-02-12 01:53:39,353 - INFO - Starting upload: lib/pygments/lexers/_vbscript_builtins.py -> /home/kevin/test/lib/pygments/lexers/_vbscript_builtins.py (4225 bytes)
+2026-02-12 01:53:39,354 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_vbscript_builtins.py', 'wb')
+2026-02-12 01:53:39,355 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_vbscript_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,355 - DEBUG - Progress: 4225/4225 bytes
+2026-02-12 01:53:39,355 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,357 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_vbscript_builtins.py')
+2026-02-12 01:53:39,359 - INFO - Upload completed successfully: _vbscript_builtins.py
+2026-02-12 01:53:39,359 - INFO - Starting upload: lib/pygments/lexers/_usd_builtins.py -> /home/kevin/test/lib/pygments/lexers/_usd_builtins.py (1658 bytes)
+2026-02-12 01:53:39,361 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_usd_builtins.py', 'wb')
+2026-02-12 01:53:39,362 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_usd_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,362 - DEBUG - Progress: 1658/1658 bytes
+2026-02-12 01:53:39,362 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,363 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_usd_builtins.py')
+2026-02-12 01:53:39,365 - INFO - Upload completed successfully: _usd_builtins.py
+2026-02-12 01:53:39,365 - INFO - Starting upload: lib/pygments/lexers/_tsql_builtins.py -> /home/kevin/test/lib/pygments/lexers/_tsql_builtins.py (15460 bytes)
+2026-02-12 01:53:39,367 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_tsql_builtins.py', 'wb')
+2026-02-12 01:53:39,368 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_tsql_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,368 - DEBUG - Progress: 15460/15460 bytes
+2026-02-12 01:53:39,368 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,370 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_tsql_builtins.py')
+2026-02-12 01:53:39,372 - INFO - Upload completed successfully: _tsql_builtins.py
+2026-02-12 01:53:39,372 - INFO - Starting upload: lib/pygments/lexers/_stata_builtins.py -> /home/kevin/test/lib/pygments/lexers/_stata_builtins.py (27227 bytes)
+2026-02-12 01:53:39,374 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_stata_builtins.py', 'wb')
+2026-02-12 01:53:39,375 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_stata_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,375 - DEBUG - Progress: 27227/27227 bytes
+2026-02-12 01:53:39,375 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,378 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_stata_builtins.py')
+2026-02-12 01:53:39,380 - INFO - Upload completed successfully: _stata_builtins.py
+2026-02-12 01:53:39,381 - INFO - Starting upload: lib/pygments/lexers/_stan_builtins.py -> /home/kevin/test/lib/pygments/lexers/_stan_builtins.py (13445 bytes)
+2026-02-12 01:53:39,383 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_stan_builtins.py', 'wb')
+2026-02-12 01:53:39,383 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_stan_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,384 - DEBUG - Progress: 13445/13445 bytes
+2026-02-12 01:53:39,384 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,386 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_stan_builtins.py')
+2026-02-12 01:53:39,388 - INFO - Upload completed successfully: _stan_builtins.py
+2026-02-12 01:53:39,388 - INFO - Starting upload: lib/pygments/lexers/_sql_builtins.py -> /home/kevin/test/lib/pygments/lexers/_sql_builtins.py (6767 bytes)
+2026-02-12 01:53:39,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_sql_builtins.py', 'wb')
+2026-02-12 01:53:39,390 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_sql_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,390 - DEBUG - Progress: 6767/6767 bytes
+2026-02-12 01:53:39,391 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,392 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_sql_builtins.py')
+2026-02-12 01:53:39,394 - INFO - Upload completed successfully: _sql_builtins.py
+2026-02-12 01:53:39,394 - INFO - Starting upload: lib/pygments/lexers/_sourcemod_builtins.py -> /home/kevin/test/lib/pygments/lexers/_sourcemod_builtins.py (26777 bytes)
+2026-02-12 01:53:39,396 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_sourcemod_builtins.py', 'wb')
+2026-02-12 01:53:39,397 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_sourcemod_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,397 - DEBUG - Progress: 26777/26777 bytes
+2026-02-12 01:53:39,397 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,401 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_sourcemod_builtins.py')
+2026-02-12 01:53:39,403 - INFO - Upload completed successfully: _sourcemod_builtins.py
+2026-02-12 01:53:39,403 - INFO - Starting upload: lib/pygments/lexers/_scilab_builtins.py -> /home/kevin/test/lib/pygments/lexers/_scilab_builtins.py (52411 bytes)
+2026-02-12 01:53:39,405 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_scilab_builtins.py', 'wb')
+2026-02-12 01:53:39,406 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_scilab_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,406 - DEBUG - Progress: 32768/52411 bytes
+2026-02-12 01:53:39,406 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,413 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_scilab_builtins.py')
+2026-02-12 01:53:39,415 - INFO - Upload completed successfully: _scilab_builtins.py
+2026-02-12 01:53:39,415 - INFO - Starting upload: lib/pygments/lexers/_scheme_builtins.py -> /home/kevin/test/lib/pygments/lexers/_scheme_builtins.py (32564 bytes)
+2026-02-12 01:53:39,417 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_scheme_builtins.py', 'wb')
+2026-02-12 01:53:39,417 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_scheme_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,417 - DEBUG - Progress: 32564/32564 bytes
+2026-02-12 01:53:39,417 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,421 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_scheme_builtins.py')
+2026-02-12 01:53:39,424 - INFO - Upload completed successfully: _scheme_builtins.py
+2026-02-12 01:53:39,424 - INFO - Starting upload: lib/pygments/lexers/_qlik_builtins.py -> /home/kevin/test/lib/pygments/lexers/_qlik_builtins.py (12595 bytes)
+2026-02-12 01:53:39,426 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_qlik_builtins.py', 'wb')
+2026-02-12 01:53:39,427 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_qlik_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,427 - DEBUG - Progress: 12595/12595 bytes
+2026-02-12 01:53:39,427 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,430 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_qlik_builtins.py')
+2026-02-12 01:53:39,431 - INFO - Upload completed successfully: _qlik_builtins.py
+2026-02-12 01:53:39,432 - INFO - Starting upload: lib/pygments/lexers/_postgres_builtins.py -> /home/kevin/test/lib/pygments/lexers/_postgres_builtins.py (13343 bytes)
+2026-02-12 01:53:39,434 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_postgres_builtins.py', 'wb')
+2026-02-12 01:53:39,434 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_postgres_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,435 - DEBUG - Progress: 13343/13343 bytes
+2026-02-12 01:53:39,435 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,437 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_postgres_builtins.py')
+2026-02-12 01:53:39,438 - INFO - Upload completed successfully: _postgres_builtins.py
+2026-02-12 01:53:39,439 - INFO - Starting upload: lib/pygments/lexers/_php_builtins.py -> /home/kevin/test/lib/pygments/lexers/_php_builtins.py (107922 bytes)
+2026-02-12 01:53:39,440 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_php_builtins.py', 'wb')
+2026-02-12 01:53:39,441 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_php_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,441 - DEBUG - Progress: 32768/107922 bytes
+2026-02-12 01:53:39,442 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,454 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_php_builtins.py')
+2026-02-12 01:53:39,456 - INFO - Upload completed successfully: _php_builtins.py
+2026-02-12 01:53:39,456 - INFO - Starting upload: lib/pygments/lexers/_openedge_builtins.py -> /home/kevin/test/lib/pygments/lexers/_openedge_builtins.py (49398 bytes)
+2026-02-12 01:53:39,458 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_openedge_builtins.py', 'wb')
+2026-02-12 01:53:39,458 - DEBUG - [chan 0] open(b'/home/kevin/test/lib/pygments/lexers/_openedge_builtins.py', 'wb') -> 00000000
+2026-02-12 01:53:39,459 - DEBUG - Progress: 32768/49398 bytes
+2026-02-12 01:53:39,459 - DEBUG - [chan 0] close(00000000)
+2026-02-12 01:53:39,467 - DEBUG - [chan 0] stat(b'/home/kevin/test/lib/pygments/lexers/_openedge_builtins.py')
+2026-02-12 01:54:08,328 - DEBUG - Using selector: EpollSelector
+2026-02-12 02:03:01,343 - DEBUG - Using selector: EpollSelector
+2026-02-12 02:03:14,257 - INFO - Initiating SSH connection to 192.168.1.5:22
+2026-02-12 02:03:14,257 - INFO - Attempting login for user: kevin
+2026-02-12 02:03:14,259 - DEBUG - starting thread (client mode): 0x74e1eba0
+2026-02-12 02:03:14,259 - DEBUG - Local version/idstring: SSH-2.0-paramiko_4.0.0
+2026-02-12 02:03:14,273 - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.11
+2026-02-12 02:03:14,274 - INFO - Connected (version 2.0, client OpenSSH_9.6p1)
+2026-02-12 02:03:14,276 - DEBUG - === Key exchange possibilities ===
+2026-02-12 02:03:14,276 - DEBUG - kex algos: sntrup761x25519-sha512@openssh.com, curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, ext-info-s, kex-strict-s-v00@openssh.com
+2026-02-12 02:03:14,276 - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519
+2026-02-12 02:03:14,276 - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 02:03:14,276 - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 02:03:14,276 - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 02:03:14,276 - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 02:03:14,276 - DEBUG - client compress: none, zlib@openssh.com
+2026-02-12 02:03:14,276 - DEBUG - server compress: none, zlib@openssh.com
+2026-02-12 02:03:14,276 - DEBUG - client lang: 
+2026-02-12 02:03:14,276 - DEBUG - server lang: 
+2026-02-12 02:03:14,276 - DEBUG - kex follows: False
+2026-02-12 02:03:14,276 - DEBUG - === Key exchange agreements ===
+2026-02-12 02:03:14,276 - DEBUG - Strict kex mode: True
+2026-02-12 02:03:14,276 - DEBUG - Kex: curve25519-sha256@libssh.org
+2026-02-12 02:03:14,276 - DEBUG - HostKey: ssh-ed25519
+2026-02-12 02:03:14,276 - DEBUG - Cipher: aes128-ctr
+2026-02-12 02:03:14,277 - DEBUG - MAC: hmac-sha2-256
+2026-02-12 02:03:14,277 - DEBUG - Compression: none
+2026-02-12 02:03:14,277 - DEBUG - === End of kex handshake ===
+2026-02-12 02:03:14,286 - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode
+2026-02-12 02:03:14,286 - DEBUG - kex engine KexCurve25519 specified hash_algo 
+2026-02-12 02:03:14,286 - DEBUG - Switch to new keys ...
+2026-02-12 02:03:14,286 - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode
+2026-02-12 02:03:14,287 - DEBUG - Adding ssh-ed25519 host key for 192.168.1.5: b'05b4c881009e843d53172fd7a680af53'
+2026-02-12 02:03:14,287 - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256', 'publickey-hostbound@openssh.com': b'0', 'ping@openssh.com': b'0'}
+2026-02-12 02:03:14,327 - DEBUG - userauth is OK
+2026-02-12 02:03:14,348 - INFO - Authentication (password) successful!
+2026-02-12 02:03:14,349 - INFO - SSH connection established. Opening SFTP session...
+2026-02-12 02:03:14,349 - DEBUG - [chan 0] Max packet in: 32768 bytes
+2026-02-12 02:03:14,784 - DEBUG - Received global request "hostkeys-00@openssh.com"
+2026-02-12 02:03:14,784 - DEBUG - Rejecting "hostkeys-00@openssh.com" global request from server.
+2026-02-12 02:03:14,824 - DEBUG - [chan 0] Max packet out: 32768 bytes
+2026-02-12 02:03:14,824 - DEBUG - Secsh channel 0 opened.
+2026-02-12 02:03:14,826 - DEBUG - [chan 0] Sesch channel 0 request ok
+2026-02-12 02:03:14,829 - INFO - [chan 0] Opened sftp connection (server version 3)
+2026-02-12 02:03:14,829 - INFO - SFTP session opened successfully.
+2026-02-12 02:03:14,829 - INFO - Scanning directory: core
+2026-02-12 02:03:14,829 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/core', 511)
+2026-02-12 02:03:14,830 - INFO - Created remote folder: /home/kevin/test/core
+2026-02-12 02:03:14,831 - INFO - Closing SFTP and SSH connections.
+2026-02-12 02:03:14,831 - INFO - [chan 0] sftp session closed.
+2026-02-12 02:03:14,831 - DEBUG - [chan 0] EOF sent (0)
+2026-02-12 02:03:52,971 - DEBUG - Using selector: EpollSelector
+2026-02-12 02:03:54,763 - INFO - Initiating SSH connection to 192.168.1.5:22
+2026-02-12 02:03:54,763 - INFO - Attempting login for user: kevin
+2026-02-12 02:03:54,767 - DEBUG - starting thread (client mode): 0x7adc6ba0
+2026-02-12 02:03:54,767 - DEBUG - Local version/idstring: SSH-2.0-paramiko_4.0.0
+2026-02-12 02:03:54,780 - DEBUG - Remote version/idstring: SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.11
+2026-02-12 02:03:54,781 - INFO - Connected (version 2.0, client OpenSSH_9.6p1)
+2026-02-12 02:03:54,782 - DEBUG - === Key exchange possibilities ===
+2026-02-12 02:03:54,783 - DEBUG - kex algos: sntrup761x25519-sha512@openssh.com, curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group14-sha256, ext-info-s, kex-strict-s-v00@openssh.com
+2026-02-12 02:03:54,783 - DEBUG - server key: rsa-sha2-512, rsa-sha2-256, ecdsa-sha2-nistp256, ssh-ed25519
+2026-02-12 02:03:54,783 - DEBUG - client encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 02:03:54,783 - DEBUG - server encrypt: chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com
+2026-02-12 02:03:54,783 - DEBUG - client mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 02:03:54,783 - DEBUG - server mac: umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1
+2026-02-12 02:03:54,783 - DEBUG - client compress: none, zlib@openssh.com
+2026-02-12 02:03:54,783 - DEBUG - server compress: none, zlib@openssh.com
+2026-02-12 02:03:54,783 - DEBUG - client lang: 
+2026-02-12 02:03:54,783 - DEBUG - server lang: 
+2026-02-12 02:03:54,783 - DEBUG - kex follows: False
+2026-02-12 02:03:54,783 - DEBUG - === Key exchange agreements ===
+2026-02-12 02:03:54,783 - DEBUG - Strict kex mode: True
+2026-02-12 02:03:54,783 - DEBUG - Kex: curve25519-sha256@libssh.org
+2026-02-12 02:03:54,783 - DEBUG - HostKey: ssh-ed25519
+2026-02-12 02:03:54,783 - DEBUG - Cipher: aes128-ctr
+2026-02-12 02:03:54,783 - DEBUG - MAC: hmac-sha2-256
+2026-02-12 02:03:54,783 - DEBUG - Compression: none
+2026-02-12 02:03:54,783 - DEBUG - === End of kex handshake ===
+2026-02-12 02:03:54,792 - DEBUG - Resetting outbound seqno after NEWKEYS due to strict mode
+2026-02-12 02:03:54,793 - DEBUG - kex engine KexCurve25519 specified hash_algo 
+2026-02-12 02:03:54,793 - DEBUG - Switch to new keys ...
+2026-02-12 02:03:54,793 - DEBUG - Resetting inbound seqno after NEWKEYS due to strict mode
+2026-02-12 02:03:54,794 - DEBUG - Adding ssh-ed25519 host key for 192.168.1.5: b'05b4c881009e843d53172fd7a680af53'
+2026-02-12 02:03:54,794 - DEBUG - Got EXT_INFO: {'server-sig-algs': b'ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256', 'publickey-hostbound@openssh.com': b'0', 'ping@openssh.com': b'0'}
+2026-02-12 02:03:54,834 - DEBUG - userauth is OK
+2026-02-12 02:03:54,855 - INFO - Authentication (password) successful!
+2026-02-12 02:03:54,856 - INFO - SSH connection established. Opening SFTP session...
+2026-02-12 02:03:54,856 - DEBUG - [chan 0] Max packet in: 32768 bytes
+2026-02-12 02:03:54,922 - DEBUG - Received global request "hostkeys-00@openssh.com"
+2026-02-12 02:03:54,922 - DEBUG - Rejecting "hostkeys-00@openssh.com" global request from server.
+2026-02-12 02:03:54,963 - DEBUG - [chan 0] Max packet out: 32768 bytes
+2026-02-12 02:03:54,963 - DEBUG - Secsh channel 0 opened.
+2026-02-12 02:03:54,965 - DEBUG - [chan 0] Sesch channel 0 request ok
+2026-02-12 02:03:54,969 - INFO - [chan 0] Opened sftp connection (server version 3)
+2026-02-12 02:03:54,969 - INFO - SFTP session opened successfully.
+2026-02-12 02:03:54,969 - INFO - Scanning directory: __pycache__
+2026-02-12 02:03:54,969 - DEBUG - [chan 0] mkdir(b'/home/kevin/test/__pycache__', 511)
+2026-02-12 02:03:54,970 - INFO - Created remote folder: /home/kevin/test/__pycache__
+2026-02-12 02:03:54,970 - INFO - Starting upload: __pycache__/setup.cpython-314.pyc -> /home/kevin/test/__pycache__/setup.cpython-314.pyc (1219 bytes)
+2026-02-12 02:03:54,975 - DEBUG - [chan 0] open(b'/home/kevin/test/__pycache__/setup.cpython-314.pyc', 'wb')
+2026-02-12 02:03:54,976 - DEBUG - [chan 0] open(b'/home/kevin/test/__pycache__/setup.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 02:03:54,977 - DEBUG - Progress: 1219/1219 bytes
+2026-02-12 02:03:54,977 - DEBUG - [chan 0] close(00000000)
+2026-02-12 02:03:54,978 - DEBUG - [chan 0] stat(b'/home/kevin/test/__pycache__/setup.cpython-314.pyc')
+2026-02-12 02:03:54,981 - INFO - Upload completed successfully: setup.cpython-314.pyc
+2026-02-12 02:03:54,982 - INFO - Starting upload: __pycache__/initSync.cpython-314.pyc -> /home/kevin/test/__pycache__/initSync.cpython-314.pyc (4463 bytes)
+2026-02-12 02:03:54,985 - DEBUG - [chan 0] open(b'/home/kevin/test/__pycache__/initSync.cpython-314.pyc', 'wb')
+2026-02-12 02:03:54,986 - DEBUG - [chan 0] open(b'/home/kevin/test/__pycache__/initSync.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 02:03:54,986 - DEBUG - Progress: 4463/4463 bytes
+2026-02-12 02:03:54,986 - DEBUG - [chan 0] close(00000000)
+2026-02-12 02:03:54,988 - DEBUG - [chan 0] stat(b'/home/kevin/test/__pycache__/initSync.cpython-314.pyc')
+2026-02-12 02:03:54,991 - INFO - Upload completed successfully: initSync.cpython-314.pyc
+2026-02-12 02:03:54,991 - INFO - Starting upload: __pycache__/sync.cpython-314.pyc -> /home/kevin/test/__pycache__/sync.cpython-314.pyc (2675 bytes)
+2026-02-12 02:03:54,994 - DEBUG - [chan 0] open(b'/home/kevin/test/__pycache__/sync.cpython-314.pyc', 'wb')
+2026-02-12 02:03:54,995 - DEBUG - [chan 0] open(b'/home/kevin/test/__pycache__/sync.cpython-314.pyc', 'wb') -> 00000000
+2026-02-12 02:03:54,996 - DEBUG - Progress: 2675/2675 bytes
+2026-02-12 02:03:54,996 - DEBUG - [chan 0] close(00000000)
+2026-02-12 02:03:54,997 - DEBUG - [chan 0] stat(b'/home/kevin/test/__pycache__/sync.cpython-314.pyc')
+2026-02-12 02:03:55,000 - INFO - Upload completed successfully: sync.cpython-314.pyc
+2026-02-12 02:03:55,000 - INFO - Closing SFTP and SSH connections.
+2026-02-12 02:03:55,000 - INFO - [chan 0] sftp session closed.
+2026-02-12 02:03:55,001 - DEBUG - [chan 0] EOF sent (0)
+2026-02-12 02:05:35,235 - DEBUG - Using selector: EpollSelector
+2026-02-12 02:07:35,655 - DEBUG - Using selector: EpollSelector
+2026-02-12 02:08:45,662 - DEBUG - Using selector: EpollSelector
diff --git a/sync.py b/sync.py
new file mode 100644
index 0000000..1801694
--- /dev/null
+++ b/sync.py
@@ -0,0 +1,60 @@
+import os
+import lib.questionary as qs
+import json
+from os import stat
+from pathlib import Path
+from lib.ftp import SFTPSync 
+
+
+def load_config(filename=".SyncNode.config"):
+    path = Path(filename)
+    if path.exists():
+        with open(path, "r") as f:
+            return json.load(f)
+    return None
+
+def get_smart_staged_paths(root_dir, config):
+    blacklist = config.get("blacklist", [])
+    auto_sync = config.get("autoSync", [])
+    
+    choices = []
+    
+    # Use .iterdir() to stay in the ROOT only
+    for p in Path(root_dir).iterdir():
+        rel_path = p.name
+        
+        if rel_path in blacklist or rel_path.startswith('.'):
+            continue
+            
+        is_auto = rel_path in auto_sync
+        choices.append(qs.Choice(rel_path, checked=is_auto))
+
+    return qs.checkbox("Select top-level folders/files to sync:", choices=choices).ask()
+config = load_config()
+def handle_upload():
+    current_dir = Path.cwd()
+    config = load_config()
+
+    if config is None:
+        print("\n[!] Configuration Missing")
+        print(f"[*] No '.SyncNode.config' found in: {current_dir}")
+        print("[*] Please run 'ServerSync init' (or setup) in this directory first.\n")
+        return    
+    print(f"[*] Scanning for files in: {current_dir}")
+    
+    selected = get_smart_staged_paths(current_dir, config)
+    
+    if not selected:
+        print("[~] No files selected for sync.")
+        return
+    client = SFTPSync(config['ip'], config['user'], config['password'], config['port'])
+    if client.connect() is True:
+        for item in selected:
+            local_path = current_dir / item
+            
+            if local_path.is_dir():
+                client.upload_directory(str(local_path), config['directory'])
+            else:
+                client.upload_with_progress(str(local_path), config['directory'])
+        client.disconnect()
+        print("\n[✔] Staged Sync Complete!")

' : '\U0001d4ab', + '\\' : '\U0001d4ac', + '\\' : '\U0000211b', + '\\' : '\U0001d4ae', + '\\' : '\U0001d4af', + '\\' : '\U0001d4b0', + '\\' : '\U0001d4b1', + '\\' : '\U0001d4b2', + '\\' : '\U0001d4b3', + '\\' : '\U0001d4b4', + '\\' : '\U0001d4b5', + '\\' : '\U0001d5ba', + '\\' : '\U0001d5bb', + '\\' : '\U0001d5bc', + '\\' : '\U0001d5bd', + '\\' : '\U0001d5be', + '\\' : '\U0001d5bf', + '\\' : '\U0001d5c0', + '\\' : '\U0001d5c1', + '\\' : '\U0001d5c2', + '\\' : '\U0001d5c3', + '\\' : '\U0001d5c4', + '\\' : '\U0001d5c5', + '\\' : '\U0001d5c6', + '\\' : '\U0001d5c7', + '\\' : '\U0001d5c8', + '\\